10659 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE3; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX4; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX25; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FP6; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FCP7; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX5128; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512-FCP9; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ10; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ-FCP11; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW12; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW-FCP13; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512DQ-BW14; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ-BW-FCP15 16; These patterns are produced by LoopVectorizer for interleaved stores.17 18define void @store_i16_stride6_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %in.vecptr3, ptr %in.vecptr4, ptr %in.vecptr5, ptr %out.vec) nounwind {19; SSE-LABEL: store_i16_stride6_vf2:20; SSE: # %bb.0:21; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax22; SSE-NEXT: movdqa (%rdi), %xmm023; SSE-NEXT: movdqa (%rdx), %xmm124; SSE-NEXT: movdqa (%r8), %xmm225; SSE-NEXT: movdqa (%r9), %xmm326; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]27; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]28; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]29; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]30; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm0[0,1,2,3,4,5,4,6]31; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]32; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]33; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm4[3,3]34; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,1,3,4,5,6,7]35; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0,2]36; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]37; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,0,3,3,4,5,6,7]38; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]39; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,1,2,0]40; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]41; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]42; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,0,3,4,5,6,7]43; SSE-NEXT: movaps %xmm0, (%rax)44; SSE-NEXT: movq %xmm1, 16(%rax)45; SSE-NEXT: retq46;47; AVX-LABEL: store_i16_stride6_vf2:48; AVX: # %bb.0:49; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax50; AVX-NEXT: vmovdqa (%rdi), %xmm051; AVX-NEXT: vmovdqa (%rdx), %xmm152; AVX-NEXT: vmovdqa (%r8), %xmm253; AVX-NEXT: vmovdqa (%r9), %xmm354; AVX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]55; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]56; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]57; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]58; AVX-NEXT: vpshufhw {{.*#+}} xmm4 = xmm0[0,1,2,3,5,7,6,7]59; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[2,2,2,2]60; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]61; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]62; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,u,u,u,u,2,3,6,7]63; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]64; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]65; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5],xmm0[6,7]66; AVX-NEXT: vmovdqa %xmm0, (%rax)67; AVX-NEXT: vmovq %xmm1, 16(%rax)68; AVX-NEXT: retq69;70; AVX2-LABEL: store_i16_stride6_vf2:71; AVX2: # %bb.0:72; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax73; AVX2-NEXT: vmovdqa (%rdi), %xmm074; AVX2-NEXT: vmovdqa (%rdx), %xmm175; AVX2-NEXT: vmovdqa (%r8), %xmm276; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]77; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]78; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]79; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],mem[0],xmm2[1],mem[1]80; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm081; AVX2-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,2,3,6,7,u,u,u,u,18,19,22,23,u,u,u,u,u,u,u,u]82; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]83; AVX2-NEXT: vpshuflw {{.*#+}} ymm0 = ymm0[1,3,2,3,4,5,6,7,9,11,10,11,12,13,14,15]84; AVX2-NEXT: vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]85; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6,7]86; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm187; AVX2-NEXT: vmovq %xmm1, 16(%rax)88; AVX2-NEXT: vmovdqa %xmm0, (%rax)89; AVX2-NEXT: vzeroupper90; AVX2-NEXT: retq91;92; AVX2-FP-LABEL: store_i16_stride6_vf2:93; AVX2-FP: # %bb.0:94; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %rax95; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm096; AVX2-FP-NEXT: vmovdqa (%rdx), %xmm197; AVX2-FP-NEXT: vmovdqa (%r8), %xmm298; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]99; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]100; AVX2-FP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]101; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],mem[0],xmm2[1],mem[1]102; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0103; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,4,5,8,9,12,13,8,9,12,13,2,3,6,7,18,19,22,23,2,3,6,7,u,u,u,u,u,u,u,u]104; AVX2-FP-NEXT: vpshufb %ymm1, %ymm0, %ymm2105; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]106; AVX2-FP-NEXT: vpshufb %ymm1, %ymm0, %ymm0107; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm0[2],ymm2[3],ymm0[4],ymm2[5,6,7]108; AVX2-FP-NEXT: vextracti128 $1, %ymm0, %xmm1109; AVX2-FP-NEXT: vmovq %xmm1, 16(%rax)110; AVX2-FP-NEXT: vmovdqa %xmm0, (%rax)111; AVX2-FP-NEXT: vzeroupper112; AVX2-FP-NEXT: retq113;114; AVX2-FCP-LABEL: store_i16_stride6_vf2:115; AVX2-FCP: # %bb.0:116; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax117; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm0118; AVX2-FCP-NEXT: vmovdqa (%rdx), %xmm1119; AVX2-FCP-NEXT: vmovdqa (%r8), %xmm2120; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]121; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]122; AVX2-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]123; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],mem[0],xmm2[1],mem[1]124; AVX2-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0125; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,4,5,8,9,12,13,8,9,12,13,2,3,6,7,18,19,22,23,2,3,6,7,u,u,u,u,u,u,u,u]126; AVX2-FCP-NEXT: vpshufb %ymm1, %ymm0, %ymm2127; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]128; AVX2-FCP-NEXT: vpshufb %ymm1, %ymm0, %ymm0129; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm0[2],ymm2[3],ymm0[4],ymm2[5,6,7]130; AVX2-FCP-NEXT: vextracti128 $1, %ymm0, %xmm1131; AVX2-FCP-NEXT: vmovq %xmm1, 16(%rax)132; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rax)133; AVX2-FCP-NEXT: vzeroupper134; AVX2-FCP-NEXT: retq135;136; AVX512-LABEL: store_i16_stride6_vf2:137; AVX512: # %bb.0:138; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax139; AVX512-NEXT: vmovdqa (%rdi), %xmm0140; AVX512-NEXT: vmovdqa (%rdx), %xmm1141; AVX512-NEXT: vmovdqa (%r8), %xmm2142; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]143; AVX512-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]144; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]145; AVX512-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],mem[0],xmm2[1],mem[1]146; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0147; AVX512-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,2,3,6,7,u,u,u,u,18,19,22,23,u,u,u,u,u,u,u,u]148; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]149; AVX512-NEXT: vpshuflw {{.*#+}} ymm0 = ymm0[1,3,2,3,4,5,6,7,9,11,10,11,12,13,14,15]150; AVX512-NEXT: vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]151; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6,7]152; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1153; AVX512-NEXT: vmovq %xmm1, 16(%rax)154; AVX512-NEXT: vmovdqa %xmm0, (%rax)155; AVX512-NEXT: vzeroupper156; AVX512-NEXT: retq157;158; AVX512-FCP-LABEL: store_i16_stride6_vf2:159; AVX512-FCP: # %bb.0:160; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax161; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm0162; AVX512-FCP-NEXT: vmovdqa (%rdx), %xmm1163; AVX512-FCP-NEXT: vmovdqa (%r8), %xmm2164; AVX512-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]165; AVX512-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]166; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]167; AVX512-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],mem[0],xmm2[1],mem[1]168; AVX512-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0169; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,4,5,8,9,12,13,8,9,12,13,2,3,6,7,18,19,22,23,2,3,6,7,u,u,u,u,u,u,u,u]170; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm0, %ymm2171; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]172; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm0, %ymm0173; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm0[2],ymm2[3],ymm0[4],ymm2[5,6,7]174; AVX512-FCP-NEXT: vextracti128 $1, %ymm0, %xmm1175; AVX512-FCP-NEXT: vmovq %xmm1, 16(%rax)176; AVX512-FCP-NEXT: vmovdqa %xmm0, (%rax)177; AVX512-FCP-NEXT: vzeroupper178; AVX512-FCP-NEXT: retq179;180; AVX512DQ-LABEL: store_i16_stride6_vf2:181; AVX512DQ: # %bb.0:182; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax183; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0184; AVX512DQ-NEXT: vmovdqa (%rdx), %xmm1185; AVX512DQ-NEXT: vmovdqa (%r8), %xmm2186; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]187; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]188; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]189; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],mem[0],xmm2[1],mem[1]190; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0191; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,2,3,6,7,u,u,u,u,18,19,22,23,u,u,u,u,u,u,u,u]192; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]193; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm0 = ymm0[1,3,2,3,4,5,6,7,9,11,10,11,12,13,14,15]194; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]195; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6,7]196; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm1197; AVX512DQ-NEXT: vmovq %xmm1, 16(%rax)198; AVX512DQ-NEXT: vmovdqa %xmm0, (%rax)199; AVX512DQ-NEXT: vzeroupper200; AVX512DQ-NEXT: retq201;202; AVX512DQ-FCP-LABEL: store_i16_stride6_vf2:203; AVX512DQ-FCP: # %bb.0:204; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax205; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm0206; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %xmm1207; AVX512DQ-FCP-NEXT: vmovdqa (%r8), %xmm2208; AVX512DQ-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]209; AVX512DQ-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]210; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]211; AVX512DQ-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],mem[0],xmm2[1],mem[1]212; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0213; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [0,1,4,5,8,9,12,13,8,9,12,13,2,3,6,7,18,19,22,23,2,3,6,7,u,u,u,u,u,u,u,u]214; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm0, %ymm2215; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]216; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm0, %ymm0217; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm0[2],ymm2[3],ymm0[4],ymm2[5,6,7]218; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm0, %xmm1219; AVX512DQ-FCP-NEXT: vmovq %xmm1, 16(%rax)220; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, (%rax)221; AVX512DQ-FCP-NEXT: vzeroupper222; AVX512DQ-FCP-NEXT: retq223;224; AVX512BW-LABEL: store_i16_stride6_vf2:225; AVX512BW: # %bb.0:226; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax227; AVX512BW-NEXT: vmovdqa (%rdi), %xmm0228; AVX512BW-NEXT: vmovdqa (%rdx), %xmm1229; AVX512BW-NEXT: vmovdqa (%r8), %xmm2230; AVX512BW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]231; AVX512BW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]232; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]233; AVX512BW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],mem[0],xmm2[1],mem[1]234; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [0,2,4,6,16,18,1,3,5,7,17,19,0,0,0,0]235; AVX512BW-NEXT: vpermi2w %ymm1, %ymm0, %ymm2236; AVX512BW-NEXT: vextracti128 $1, %ymm2, %xmm0237; AVX512BW-NEXT: vmovq %xmm0, 16(%rax)238; AVX512BW-NEXT: vmovdqa %xmm2, (%rax)239; AVX512BW-NEXT: vzeroupper240; AVX512BW-NEXT: retq241;242; AVX512BW-FCP-LABEL: store_i16_stride6_vf2:243; AVX512BW-FCP: # %bb.0:244; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax245; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %xmm0246; AVX512BW-FCP-NEXT: vmovdqa (%rdx), %xmm1247; AVX512BW-FCP-NEXT: vmovdqa (%r8), %xmm2248; AVX512BW-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]249; AVX512BW-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]250; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]251; AVX512BW-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],mem[0],xmm2[1],mem[1]252; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [0,2,4,6,16,18,1,3,5,7,17,19,0,0,0,0]253; AVX512BW-FCP-NEXT: vpermi2w %ymm1, %ymm0, %ymm2254; AVX512BW-FCP-NEXT: vextracti128 $1, %ymm2, %xmm0255; AVX512BW-FCP-NEXT: vmovq %xmm0, 16(%rax)256; AVX512BW-FCP-NEXT: vmovdqa %xmm2, (%rax)257; AVX512BW-FCP-NEXT: vzeroupper258; AVX512BW-FCP-NEXT: retq259;260; AVX512DQ-BW-LABEL: store_i16_stride6_vf2:261; AVX512DQ-BW: # %bb.0:262; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %rax263; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %xmm0264; AVX512DQ-BW-NEXT: vmovdqa (%rdx), %xmm1265; AVX512DQ-BW-NEXT: vmovdqa (%r8), %xmm2266; AVX512DQ-BW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]267; AVX512DQ-BW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]268; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]269; AVX512DQ-BW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],mem[0],xmm2[1],mem[1]270; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [0,2,4,6,16,18,1,3,5,7,17,19,0,0,0,0]271; AVX512DQ-BW-NEXT: vpermi2w %ymm1, %ymm0, %ymm2272; AVX512DQ-BW-NEXT: vextracti128 $1, %ymm2, %xmm0273; AVX512DQ-BW-NEXT: vmovq %xmm0, 16(%rax)274; AVX512DQ-BW-NEXT: vmovdqa %xmm2, (%rax)275; AVX512DQ-BW-NEXT: vzeroupper276; AVX512DQ-BW-NEXT: retq277;278; AVX512DQ-BW-FCP-LABEL: store_i16_stride6_vf2:279; AVX512DQ-BW-FCP: # %bb.0:280; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax281; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %xmm0282; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdx), %xmm1283; AVX512DQ-BW-FCP-NEXT: vmovdqa (%r8), %xmm2284; AVX512DQ-BW-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]285; AVX512DQ-BW-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]286; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]287; AVX512DQ-BW-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],mem[0],xmm2[1],mem[1]288; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [0,2,4,6,16,18,1,3,5,7,17,19,0,0,0,0]289; AVX512DQ-BW-FCP-NEXT: vpermi2w %ymm1, %ymm0, %ymm2290; AVX512DQ-BW-FCP-NEXT: vextracti128 $1, %ymm2, %xmm0291; AVX512DQ-BW-FCP-NEXT: vmovq %xmm0, 16(%rax)292; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm2, (%rax)293; AVX512DQ-BW-FCP-NEXT: vzeroupper294; AVX512DQ-BW-FCP-NEXT: retq295 %in.vec0 = load <2 x i16>, ptr %in.vecptr0, align 64296 %in.vec1 = load <2 x i16>, ptr %in.vecptr1, align 64297 %in.vec2 = load <2 x i16>, ptr %in.vecptr2, align 64298 %in.vec3 = load <2 x i16>, ptr %in.vecptr3, align 64299 %in.vec4 = load <2 x i16>, ptr %in.vecptr4, align 64300 %in.vec5 = load <2 x i16>, ptr %in.vecptr5, align 64301 %1 = shufflevector <2 x i16> %in.vec0, <2 x i16> %in.vec1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>302 %2 = shufflevector <2 x i16> %in.vec2, <2 x i16> %in.vec3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>303 %3 = shufflevector <2 x i16> %in.vec4, <2 x i16> %in.vec5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>304 %4 = shufflevector <4 x i16> %1, <4 x i16> %2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>305 %5 = shufflevector <4 x i16> %3, <4 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>306 %6 = shufflevector <8 x i16> %4, <8 x i16> %5, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>307 %interleaved.vec = shufflevector <12 x i16> %6, <12 x i16> poison, <12 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11>308 store <12 x i16> %interleaved.vec, ptr %out.vec, align 64309 ret void310}311 312define void @store_i16_stride6_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %in.vecptr3, ptr %in.vecptr4, ptr %in.vecptr5, ptr %out.vec) nounwind {313; SSE-LABEL: store_i16_stride6_vf4:314; SSE: # %bb.0:315; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax316; SSE-NEXT: movq {{.*#+}} xmm0 = mem[0],zero317; SSE-NEXT: movq {{.*#+}} xmm1 = mem[0],zero318; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]319; SSE-NEXT: movq {{.*#+}} xmm1 = mem[0],zero320; SSE-NEXT: movq {{.*#+}} xmm2 = mem[0],zero321; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]322; SSE-NEXT: movq {{.*#+}} xmm2 = mem[0],zero323; SSE-NEXT: movq {{.*#+}} xmm3 = mem[0],zero324; SSE-NEXT: movdqa %xmm3, %xmm4325; SSE-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]326; SSE-NEXT: movdqa %xmm1, %xmm5327; SSE-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm0[0]328; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,1,2,0]329; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,6,4,6,7]330; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[2,1],xmm0[1,3]331; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[2,0],xmm6[0,2]332; SSE-NEXT: movdqa %xmm0, %xmm6333; SSE-NEXT: punpckhqdq {{.*#+}} xmm6 = xmm6[1],xmm1[1]334; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]335; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[1,1]336; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm6[0,2]337; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,3],xmm1[3,3]338; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm4[3,1,1,3]339; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]340; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,5,7]341; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,2],xmm1[0,3]342; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0,1,3]343; SSE-NEXT: movaps %xmm3, 16(%rax)344; SSE-NEXT: movaps %xmm5, (%rax)345; SSE-NEXT: movaps %xmm0, 32(%rax)346; SSE-NEXT: retq347;348; AVX-LABEL: store_i16_stride6_vf4:349; AVX: # %bb.0:350; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax351; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero352; AVX-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero353; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]354; AVX-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero355; AVX-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero356; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm2[0],xmm1[0]357; AVX-NEXT: vmovq {{.*#+}} xmm4 = mem[0],zero358; AVX-NEXT: vmovq {{.*#+}} xmm5 = mem[0],zero359; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm5[0],xmm4[0]360; AVX-NEXT: vpshufb {{.*#+}} xmm7 = xmm3[2,3,10,11,u,u,u,u,u,u,u,u,4,5,12,13]361; AVX-NEXT: vpshufd {{.*#+}} xmm8 = xmm0[0,1,1,3]362; AVX-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,6,6,7]363; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3],xmm8[4,5],xmm7[6,7]364; AVX-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]365; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm7[0,1],xmm4[2,3],xmm7[4,5,6,7]366; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]367; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,0,2,4,5,6,7]368; AVX-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[0,1,8,9,u,u,u,u,u,u,u,u,2,3,10,11]369; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5,6,7]370; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm6[0,1,2,0]371; AVX-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,4,6,7]372; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5],xmm1[6,7]373; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm1, %ymm1374; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[3,1,2,3]375; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]376; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]377; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7]378; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]379; AVX-NEXT: vpshufb {{.*#+}} xmm2 = xmm6[4,5,12,13,u,u,u,u,u,u,u,u,6,7,14,15]380; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3,4,5],xmm2[6,7]381; AVX-NEXT: vmovdqa %xmm0, 32(%rax)382; AVX-NEXT: vmovaps %ymm1, (%rax)383; AVX-NEXT: vzeroupper384; AVX-NEXT: retq385;386; AVX2-LABEL: store_i16_stride6_vf4:387; AVX2: # %bb.0:388; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax389; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero390; AVX2-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero391; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]392; AVX2-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero393; AVX2-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero394; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]395; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm2396; AVX2-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero397; AVX2-NEXT: vmovq {{.*#+}} xmm4 = mem[0],zero398; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm5 = xmm4[0],xmm3[0]399; AVX2-NEXT: vmovdqa {{.*#+}} ymm6 = [0,1,8,9,0,1,8,9,u,u,u,u,2,3,10,11,2,3,10,11,u,u,u,u,20,21,28,29,4,5,12,13]400; AVX2-NEXT: vpshufb %ymm6, %ymm2, %ymm7401; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,3,0,1]402; AVX2-NEXT: vpshufb %ymm6, %ymm2, %ymm2403; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0],ymm2[1],ymm7[2,3,4,5],ymm2[6],ymm7[7]404; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]405; AVX2-NEXT: vpbroadcastq %xmm3, %ymm3406; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]407; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]408; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]409; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]410; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7]411; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]412; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = xmm5[4,5,12,13,u,u,u,u,u,u,u,u,6,7,14,15]413; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3]414; AVX2-NEXT: vmovdqa %xmm0, 32(%rax)415; AVX2-NEXT: vmovdqa %ymm2, (%rax)416; AVX2-NEXT: vzeroupper417; AVX2-NEXT: retq418;419; AVX2-FP-LABEL: store_i16_stride6_vf4:420; AVX2-FP: # %bb.0:421; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %rax422; AVX2-FP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero423; AVX2-FP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero424; AVX2-FP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]425; AVX2-FP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero426; AVX2-FP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero427; AVX2-FP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]428; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm2429; AVX2-FP-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero430; AVX2-FP-NEXT: vmovq {{.*#+}} xmm4 = mem[0],zero431; AVX2-FP-NEXT: vpunpcklqdq {{.*#+}} xmm5 = xmm4[0],xmm3[0]432; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm6 = [0,1,8,9,0,1,8,9,u,u,u,u,2,3,10,11,2,3,10,11,u,u,u,u,20,21,28,29,4,5,12,13]433; AVX2-FP-NEXT: vpshufb %ymm6, %ymm2, %ymm7434; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,3,0,1]435; AVX2-FP-NEXT: vpshufb %ymm6, %ymm2, %ymm2436; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0],ymm2[1],ymm7[2,3,4,5],ymm2[6],ymm7[7]437; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]438; AVX2-FP-NEXT: vpbroadcastq %xmm3, %ymm3439; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]440; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[6,7,14,15,4,5,6,7,u,u,u,u,u,u,u,u]441; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[12,13,14,15,6,7,14,15,u,u,u,u,u,u,u,u]442; AVX2-FP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]443; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm1 = xmm5[4,5,12,13,u,u,u,u,u,u,u,u,6,7,14,15]444; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3]445; AVX2-FP-NEXT: vmovdqa %xmm0, 32(%rax)446; AVX2-FP-NEXT: vmovdqa %ymm2, (%rax)447; AVX2-FP-NEXT: vzeroupper448; AVX2-FP-NEXT: retq449;450; AVX2-FCP-LABEL: store_i16_stride6_vf4:451; AVX2-FCP: # %bb.0:452; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax453; AVX2-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero454; AVX2-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero455; AVX2-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]456; AVX2-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero457; AVX2-FCP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero458; AVX2-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]459; AVX2-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm2460; AVX2-FCP-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero461; AVX2-FCP-NEXT: vmovq {{.*#+}} xmm4 = mem[0],zero462; AVX2-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm5 = xmm4[0],xmm3[0]463; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [0,1,8,9,0,1,4,5,u,u,u,u,2,3,10,11,2,3,10,11,u,u,u,u,24,25,28,29,4,5,12,13]464; AVX2-FCP-NEXT: vpshufb %ymm6, %ymm2, %ymm7465; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm8 = [4,6,1,3,4,6,1,3]466; AVX2-FCP-NEXT: # ymm8 = mem[0,1,0,1]467; AVX2-FCP-NEXT: vpermd %ymm2, %ymm8, %ymm2468; AVX2-FCP-NEXT: vpshufb %ymm6, %ymm2, %ymm2469; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0],ymm2[1],ymm7[2,3,4,5],ymm2[6],ymm7[7]470; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]471; AVX2-FCP-NEXT: vpbroadcastq %xmm3, %ymm3472; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]473; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[6,7,14,15,4,5,6,7,u,u,u,u,u,u,u,u]474; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[12,13,14,15,6,7,14,15,u,u,u,u,u,u,u,u]475; AVX2-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]476; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm5[4,5,12,13,u,u,u,u,u,u,u,u,6,7,14,15]477; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3]478; AVX2-FCP-NEXT: vmovdqa %xmm0, 32(%rax)479; AVX2-FCP-NEXT: vmovdqa %ymm2, (%rax)480; AVX2-FCP-NEXT: vzeroupper481; AVX2-FCP-NEXT: retq482;483; AVX512-LABEL: store_i16_stride6_vf4:484; AVX512: # %bb.0:485; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax486; AVX512-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero487; AVX512-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero488; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]489; AVX512-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero490; AVX512-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero491; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]492; AVX512-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero493; AVX512-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero494; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm3[0],xmm2[0]495; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm5496; AVX512-NEXT: vmovdqa {{.*#+}} ymm6 = [0,1,8,9,0,1,8,9,u,u,u,u,2,3,10,11,2,3,10,11,u,u,u,u,20,21,28,29,4,5,12,13]497; AVX512-NEXT: vpshufb %ymm6, %ymm5, %ymm7498; AVX512-NEXT: vpermq {{.*#+}} ymm5 = ymm5[2,3,0,1]499; AVX512-NEXT: vpshufb %ymm6, %ymm5, %ymm5500; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0],ymm5[1],ymm7[2,3,4,5],ymm5[6],ymm7[7]501; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]502; AVX512-NEXT: vpbroadcastq %xmm2, %ymm2503; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1],ymm2[2],ymm5[3,4],ymm2[5],ymm5[6,7]504; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]505; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]506; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]507; AVX512-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7]508; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]509; AVX512-NEXT: vpshufb {{.*#+}} xmm1 = xmm4[4,5,12,13,u,u,u,u,u,u,u,u,6,7,14,15]510; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3]511; AVX512-NEXT: vinserti32x4 $2, %xmm0, %zmm2, %zmm1512; AVX512-NEXT: vmovdqa %xmm0, 32(%rax)513; AVX512-NEXT: vmovdqa %ymm1, (%rax)514; AVX512-NEXT: vzeroupper515; AVX512-NEXT: retq516;517; AVX512-FCP-LABEL: store_i16_stride6_vf4:518; AVX512-FCP: # %bb.0:519; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax520; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero521; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero522; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]523; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero524; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero525; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]526; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero527; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero528; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm3[0],xmm2[0]529; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [8,10,1,3,8,10,1,3]530; AVX512-FCP-NEXT: # ymm5 = mem[0,1,0,1]531; AVX512-FCP-NEXT: vpermi2d %ymm1, %ymm0, %ymm5532; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [0,1,8,9,0,1,4,5,u,u,u,u,2,3,10,11,2,3,10,11,u,u,u,u,24,25,28,29,4,5,12,13]533; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm5, %ymm5534; AVX512-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm7535; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm7, %ymm6536; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2,3,4,5],ymm5[6],ymm6[7]537; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]538; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [0,1,8,3,4,9,6,7]539; AVX512-FCP-NEXT: vpermi2d %ymm2, %ymm5, %ymm3540; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[6,7,14,15,4,5,6,7,u,u,u,u,u,u,u,u]541; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[12,13,14,15,6,7,14,15,u,u,u,u,u,u,u,u]542; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]543; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm4[4,5,12,13,u,u,u,u,u,u,u,u,6,7,14,15]544; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3]545; AVX512-FCP-NEXT: vinserti32x4 $2, %xmm0, %zmm3, %zmm1546; AVX512-FCP-NEXT: vmovdqa %xmm0, 32(%rax)547; AVX512-FCP-NEXT: vmovdqa %ymm1, (%rax)548; AVX512-FCP-NEXT: vzeroupper549; AVX512-FCP-NEXT: retq550;551; AVX512DQ-LABEL: store_i16_stride6_vf4:552; AVX512DQ: # %bb.0:553; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax554; AVX512DQ-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero555; AVX512DQ-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero556; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]557; AVX512DQ-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero558; AVX512DQ-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero559; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]560; AVX512DQ-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero561; AVX512DQ-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero562; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm3[0],xmm2[0]563; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm5564; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm6 = [0,1,8,9,0,1,8,9,u,u,u,u,2,3,10,11,2,3,10,11,u,u,u,u,20,21,28,29,4,5,12,13]565; AVX512DQ-NEXT: vpshufb %ymm6, %ymm5, %ymm7566; AVX512DQ-NEXT: vpermq {{.*#+}} ymm5 = ymm5[2,3,0,1]567; AVX512DQ-NEXT: vpshufb %ymm6, %ymm5, %ymm5568; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0],ymm5[1],ymm7[2,3,4,5],ymm5[6],ymm7[7]569; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]570; AVX512DQ-NEXT: vpbroadcastq %xmm2, %ymm2571; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm5[0,1],ymm2[2],ymm5[3,4],ymm2[5],ymm5[6,7]572; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]573; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]574; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]575; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7]576; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]577; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm1 = xmm4[4,5,12,13,u,u,u,u,u,u,u,u,6,7,14,15]578; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3]579; AVX512DQ-NEXT: vinserti32x4 $2, %xmm0, %zmm2, %zmm1580; AVX512DQ-NEXT: vmovdqa %xmm0, 32(%rax)581; AVX512DQ-NEXT: vmovdqa %ymm1, (%rax)582; AVX512DQ-NEXT: vzeroupper583; AVX512DQ-NEXT: retq584;585; AVX512DQ-FCP-LABEL: store_i16_stride6_vf4:586; AVX512DQ-FCP: # %bb.0:587; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax588; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero589; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero590; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]591; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero592; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero593; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]594; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero595; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero596; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm3[0],xmm2[0]597; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [8,10,1,3,8,10,1,3]598; AVX512DQ-FCP-NEXT: # ymm5 = mem[0,1,0,1]599; AVX512DQ-FCP-NEXT: vpermi2d %ymm1, %ymm0, %ymm5600; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [0,1,8,9,0,1,4,5,u,u,u,u,2,3,10,11,2,3,10,11,u,u,u,u,24,25,28,29,4,5,12,13]601; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm5, %ymm5602; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm7603; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm7, %ymm6604; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2,3,4,5],ymm5[6],ymm6[7]605; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]606; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [0,1,8,3,4,9,6,7]607; AVX512DQ-FCP-NEXT: vpermi2d %ymm2, %ymm5, %ymm3608; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[6,7,14,15,4,5,6,7,u,u,u,u,u,u,u,u]609; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[12,13,14,15,6,7,14,15,u,u,u,u,u,u,u,u]610; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]611; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm4[4,5,12,13,u,u,u,u,u,u,u,u,6,7,14,15]612; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3]613; AVX512DQ-FCP-NEXT: vinserti32x4 $2, %xmm0, %zmm3, %zmm1614; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, 32(%rax)615; AVX512DQ-FCP-NEXT: vmovdqa %ymm1, (%rax)616; AVX512DQ-FCP-NEXT: vzeroupper617; AVX512DQ-FCP-NEXT: retq618;619; AVX512BW-LABEL: store_i16_stride6_vf4:620; AVX512BW: # %bb.0:621; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax622; AVX512BW-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero623; AVX512BW-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero624; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]625; AVX512BW-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero626; AVX512BW-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero627; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]628; AVX512BW-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero629; AVX512BW-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero630; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]631; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0632; AVX512BW-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm0633; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm1 = [0,4,8,12,16,20,1,5,9,13,17,21,2,6,10,14,18,22,3,7,11,15,19,23,0,0,0,0,0,0,0,0]634; AVX512BW-NEXT: vpermw %zmm0, %zmm1, %zmm0635; AVX512BW-NEXT: vextracti32x4 $2, %zmm0, 32(%rax)636; AVX512BW-NEXT: vmovdqa %ymm0, (%rax)637; AVX512BW-NEXT: vzeroupper638; AVX512BW-NEXT: retq639;640; AVX512BW-FCP-LABEL: store_i16_stride6_vf4:641; AVX512BW-FCP: # %bb.0:642; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax643; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero644; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero645; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]646; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero647; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero648; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]649; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero650; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero651; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]652; AVX512BW-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0653; AVX512BW-FCP-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm0654; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm1 = [0,4,8,12,16,20,1,5,9,13,17,21,2,6,10,14,18,22,3,7,11,15,19,23,0,0,0,0,0,0,0,0]655; AVX512BW-FCP-NEXT: vpermw %zmm0, %zmm1, %zmm0656; AVX512BW-FCP-NEXT: vextracti32x4 $2, %zmm0, 32(%rax)657; AVX512BW-FCP-NEXT: vmovdqa %ymm0, (%rax)658; AVX512BW-FCP-NEXT: vzeroupper659; AVX512BW-FCP-NEXT: retq660;661; AVX512DQ-BW-LABEL: store_i16_stride6_vf4:662; AVX512DQ-BW: # %bb.0:663; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %rax664; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero665; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero666; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]667; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero668; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero669; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]670; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero671; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero672; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]673; AVX512DQ-BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0674; AVX512DQ-BW-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm0675; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm1 = [0,4,8,12,16,20,1,5,9,13,17,21,2,6,10,14,18,22,3,7,11,15,19,23,0,0,0,0,0,0,0,0]676; AVX512DQ-BW-NEXT: vpermw %zmm0, %zmm1, %zmm0677; AVX512DQ-BW-NEXT: vextracti32x4 $2, %zmm0, 32(%rax)678; AVX512DQ-BW-NEXT: vmovdqa %ymm0, (%rax)679; AVX512DQ-BW-NEXT: vzeroupper680; AVX512DQ-BW-NEXT: retq681;682; AVX512DQ-BW-FCP-LABEL: store_i16_stride6_vf4:683; AVX512DQ-BW-FCP: # %bb.0:684; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax685; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero686; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero687; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]688; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero689; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero690; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]691; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero692; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero693; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]694; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0695; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $2, %xmm2, %zmm0, %zmm0696; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm1 = [0,4,8,12,16,20,1,5,9,13,17,21,2,6,10,14,18,22,3,7,11,15,19,23,0,0,0,0,0,0,0,0]697; AVX512DQ-BW-FCP-NEXT: vpermw %zmm0, %zmm1, %zmm0698; AVX512DQ-BW-FCP-NEXT: vextracti32x4 $2, %zmm0, 32(%rax)699; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm0, (%rax)700; AVX512DQ-BW-FCP-NEXT: vzeroupper701; AVX512DQ-BW-FCP-NEXT: retq702 %in.vec0 = load <4 x i16>, ptr %in.vecptr0, align 64703 %in.vec1 = load <4 x i16>, ptr %in.vecptr1, align 64704 %in.vec2 = load <4 x i16>, ptr %in.vecptr2, align 64705 %in.vec3 = load <4 x i16>, ptr %in.vecptr3, align 64706 %in.vec4 = load <4 x i16>, ptr %in.vecptr4, align 64707 %in.vec5 = load <4 x i16>, ptr %in.vecptr5, align 64708 %1 = shufflevector <4 x i16> %in.vec0, <4 x i16> %in.vec1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>709 %2 = shufflevector <4 x i16> %in.vec2, <4 x i16> %in.vec3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>710 %3 = shufflevector <4 x i16> %in.vec4, <4 x i16> %in.vec5, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>711 %4 = shufflevector <8 x i16> %1, <8 x i16> %2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>712 %5 = shufflevector <8 x i16> %3, <8 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>713 %6 = shufflevector <16 x i16> %4, <16 x i16> %5, <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>714 %interleaved.vec = shufflevector <24 x i16> %6, <24 x i16> poison, <24 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 3, i32 7, i32 11, i32 15, i32 19, i32 23>715 store <24 x i16> %interleaved.vec, ptr %out.vec, align 64716 ret void717}718 719define void @store_i16_stride6_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %in.vecptr3, ptr %in.vecptr4, ptr %in.vecptr5, ptr %out.vec) nounwind {720; SSE-LABEL: store_i16_stride6_vf8:721; SSE: # %bb.0:722; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax723; SSE-NEXT: movdqa (%rdi), %xmm0724; SSE-NEXT: movdqa (%rsi), %xmm8725; SSE-NEXT: movdqa (%rdx), %xmm1726; SSE-NEXT: movdqa (%rcx), %xmm9727; SSE-NEXT: movdqa (%r8), %xmm6728; SSE-NEXT: movdqa (%r9), %xmm5729; SSE-NEXT: movdqa %xmm1, %xmm4730; SSE-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm9[4],xmm4[5],xmm9[5],xmm4[6],xmm9[6],xmm4[7],xmm9[7]731; SSE-NEXT: movdqa %xmm0, %xmm7732; SSE-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm8[4],xmm7[5],xmm8[5],xmm7[6],xmm8[6],xmm7[7],xmm8[7]733; SSE-NEXT: movdqa %xmm7, %xmm10734; SSE-NEXT: shufps {{.*#+}} xmm10 = xmm10[2,3],xmm4[3,3]735; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm6[0,1,2,3,6,5,7,7]736; SSE-NEXT: shufps {{.*#+}} xmm10 = xmm10[1,2],xmm2[2,3]737; SSE-NEXT: shufps {{.*#+}} xmm10 = xmm10[2,0,1,3]738; SSE-NEXT: movaps {{.*#+}} xmm2 = [65535,0,65535,65535,65535,65535,65535,0]739; SSE-NEXT: andps %xmm2, %xmm10740; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm5[0,1,2,3,4,6,6,7]741; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm3[2,1,2,3]742; SSE-NEXT: movaps %xmm2, %xmm3743; SSE-NEXT: andnps %xmm11, %xmm3744; SSE-NEXT: orps %xmm10, %xmm3745; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm9[0],xmm1[1],xmm9[1],xmm1[2],xmm9[2],xmm1[3],xmm9[3]746; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1],xmm0[2],xmm8[2],xmm0[3],xmm8[3]747; SSE-NEXT: movdqa %xmm0, %xmm8748; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[2,3],xmm1[3,3]749; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm6[2,1,3,3,4,5,6,7]750; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[1,2],xmm9[0,1]751; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[2,0,1,3]752; SSE-NEXT: andps %xmm2, %xmm8753; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm5[0,2,2,3,4,5,6,7]754; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,1,2,1]755; SSE-NEXT: andnps %xmm9, %xmm2756; SSE-NEXT: orps %xmm8, %xmm2757; SSE-NEXT: movdqa %xmm1, %xmm10758; SSE-NEXT: punpcklqdq {{.*#+}} xmm10 = xmm10[0],xmm0[0]759; SSE-NEXT: movdqa %xmm6, %xmm8760; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[0,1],xmm0[1,3]761; SSE-NEXT: shufps {{.*#+}} xmm10 = xmm10[2,0],xmm8[0,2]762; SSE-NEXT: movaps {{.*#+}} xmm8 = [65535,65535,65535,65535,65535,0,65535,65535]763; SSE-NEXT: andps %xmm8, %xmm10764; SSE-NEXT: movdqa %xmm5, %xmm11765; SSE-NEXT: pslldq {{.*#+}} xmm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm11[0,1,2,3,4,5]766; SSE-NEXT: movaps %xmm8, %xmm9767; SSE-NEXT: andnps %xmm11, %xmm9768; SSE-NEXT: orps %xmm10, %xmm9769; SSE-NEXT: movdqa %xmm7, %xmm10770; SSE-NEXT: punpckhqdq {{.*#+}} xmm10 = xmm10[1],xmm4[1]771; SSE-NEXT: movdqa %xmm6, %xmm12772; SSE-NEXT: psrldq {{.*#+}} xmm12 = xmm12[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero773; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[1,1],xmm4[1,1]774; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[2,0],xmm10[0,2]775; SSE-NEXT: movaps {{.*#+}} xmm10 = [65535,65535,65535,0,65535,65535,65535,65535]776; SSE-NEXT: andps %xmm10, %xmm12777; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm5[2,2,3,3]778; SSE-NEXT: movaps %xmm10, %xmm11779; SSE-NEXT: andnps %xmm13, %xmm11780; SSE-NEXT: orps %xmm12, %xmm11781; SSE-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm7[0]782; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm6[1,1,1,1,4,5,6,7]783; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[2,1],xmm7[1,3]784; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm6[0,2]785; SSE-NEXT: andps %xmm8, %xmm4786; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,1,1]787; SSE-NEXT: pslld $16, %xmm5788; SSE-NEXT: andnps %xmm5, %xmm8789; SSE-NEXT: orps %xmm4, %xmm8790; SSE-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]791; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[1,1],xmm1[1,1]792; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[2,0],xmm0[0,2]793; SSE-NEXT: andps %xmm10, %xmm12794; SSE-NEXT: andnps %xmm6, %xmm10795; SSE-NEXT: orps %xmm12, %xmm10796; SSE-NEXT: movaps %xmm10, 16(%rax)797; SSE-NEXT: movaps %xmm8, 48(%rax)798; SSE-NEXT: movaps %xmm11, 64(%rax)799; SSE-NEXT: movaps %xmm9, (%rax)800; SSE-NEXT: movaps %xmm2, 32(%rax)801; SSE-NEXT: movaps %xmm3, 80(%rax)802; SSE-NEXT: retq803;804; AVX-LABEL: store_i16_stride6_vf8:805; AVX: # %bb.0:806; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax807; AVX-NEXT: vmovdqa (%rdi), %xmm0808; AVX-NEXT: vmovdqa (%rsi), %xmm1809; AVX-NEXT: vmovdqa (%rdx), %xmm2810; AVX-NEXT: vmovdqa (%rcx), %xmm3811; AVX-NEXT: vmovdqa (%r8), %xmm4812; AVX-NEXT: vmovdqa (%r9), %xmm5813; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]814; AVX-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]815; AVX-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[1,1,2,2]816; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2,3],xmm6[4,5],xmm8[6,7]817; AVX-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]818; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1],xmm9[2,3],xmm8[4,5,6,7]819; AVX-NEXT: vpshufd {{.*#+}} xmm10 = xmm7[0,0,1,1]820; AVX-NEXT: vpshufd {{.*#+}} xmm11 = xmm6[0,1,0,1]821; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3],xmm11[4,5,6,7]822; AVX-NEXT: vpshufd {{.*#+}} xmm11 = xmm9[0,1,0,1]823; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1,2,3],xmm11[4,5],xmm10[6,7]824; AVX-NEXT: vinsertf128 $1, %xmm8, %ymm10, %ymm8825; AVX-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]826; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,0,1,1]827; AVX-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]828; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,1,0,1]829; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5,6,7]830; AVX-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]831; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[0,1,0,1]832; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm4[4,5],xmm1[6,7]833; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm7[2,2,3,3]834; AVX-NEXT: vpunpckhqdq {{.*#+}} xmm4 = xmm6[1],xmm4[1]835; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm9[2,3,2,3]836; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3,4,5],xmm5[6,7]837; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm4, %ymm1838; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,2,3,3]839; AVX-NEXT: vpunpckhqdq {{.*#+}} xmm4 = xmm0[1],xmm4[1]840; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[2,3,2,3]841; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3,4,5],xmm5[6,7]842; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,2]843; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5],xmm2[6,7]844; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5,6,7]845; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0846; AVX-NEXT: vmovaps %ymm0, 64(%rax)847; AVX-NEXT: vmovaps %ymm1, 32(%rax)848; AVX-NEXT: vmovaps %ymm8, (%rax)849; AVX-NEXT: vzeroupper850; AVX-NEXT: retq851;852; AVX2-LABEL: store_i16_stride6_vf8:853; AVX2: # %bb.0:854; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax855; AVX2-NEXT: vmovdqa (%rdi), %xmm0856; AVX2-NEXT: vmovdqa (%rdx), %xmm1857; AVX2-NEXT: vmovdqa (%r8), %xmm2858; AVX2-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0859; AVX2-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1860; AVX2-NEXT: vinserti128 $1, (%r9), %ymm2, %ymm2861; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm1[0,2,0,2]862; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,8,9,0,1,8,9,4,5,12,13,2,3,10,11,18,19,26,27,24,25,30,31,20,21,28,29,20,21,28,29]863; AVX2-NEXT: vpshufb %ymm4, %ymm3, %ymm3864; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm0[0,2,0,2]865; AVX2-NEXT: vpshufb %ymm4, %ymm5, %ymm4866; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5,6],ymm3[7]867; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm2[0,2,0,2]868; AVX2-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,0,1,8,9,u,u,u,u,u,u,u,u,18,19,26,27,u,u,u,u,u,u,u,u]869; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]870; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm1[0,2,1,3]871; AVX2-NEXT: vmovdqa {{.*#+}} ymm5 = [0,1,8,9,6,7,14,15,6,7,14,15,2,3,10,11,16,17,24,25,16,17,24,25,24,25,26,27,18,19,26,27]872; AVX2-NEXT: vpshufb %ymm5, %ymm4, %ymm4873; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm0[0,2,1,3]874; AVX2-NEXT: vpshufb %ymm5, %ymm6, %ymm5875; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1],ymm4[2],ymm5[3,4],ymm4[5],ymm5[6,7]876; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm2[0,2,1,3]877; AVX2-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[4,5,12,13,u,u,u,u,u,u,u,u,6,7,14,15,u,u,u,u,u,u,u,u,16,17,24,25,u,u,u,u]878; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1,2],ymm5[3],ymm4[4,5],ymm5[6],ymm4[7]879; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,3,1,3]880; AVX2-NEXT: vmovdqa {{.*#+}} ymm5 = [2,3,10,11,8,9,10,11,4,5,12,13,4,5,12,13,18,19,26,27,22,23,30,31,22,23,30,31,20,21,28,29]881; AVX2-NEXT: vpshufb %ymm5, %ymm0, %ymm0882; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[1,3,1,3]883; AVX2-NEXT: vpshufb %ymm5, %ymm1, %ymm1884; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]885; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm2[1,3,1,3]886; AVX2-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,2,3,10,11,u,u,u,u,u,u,u,u,20,21,28,29,u,u,u,u,u,u,u,u,22,23,30,31]887; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]888; AVX2-NEXT: vmovdqa %ymm0, 64(%rax)889; AVX2-NEXT: vmovdqa %ymm4, 32(%rax)890; AVX2-NEXT: vmovdqa %ymm3, (%rax)891; AVX2-NEXT: vzeroupper892; AVX2-NEXT: retq893;894; AVX2-FP-LABEL: store_i16_stride6_vf8:895; AVX2-FP: # %bb.0:896; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %rax897; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm0898; AVX2-FP-NEXT: vmovdqa (%rdx), %xmm1899; AVX2-FP-NEXT: vmovdqa (%r8), %xmm2900; AVX2-FP-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0901; AVX2-FP-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1902; AVX2-FP-NEXT: vinserti128 $1, (%r9), %ymm2, %ymm2903; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm1[0,2,0,2]904; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,8,9,0,1,8,9,4,5,12,13,2,3,10,11,18,19,26,27,24,25,30,31,20,21,28,29,20,21,28,29]905; AVX2-FP-NEXT: vpshufb %ymm4, %ymm3, %ymm3906; AVX2-FP-NEXT: vpermq {{.*#+}} ymm5 = ymm0[0,2,0,2]907; AVX2-FP-NEXT: vpshufb %ymm4, %ymm5, %ymm4908; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5,6],ymm3[7]909; AVX2-FP-NEXT: vpermq {{.*#+}} ymm4 = ymm2[0,2,0,2]910; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,0,1,8,9,u,u,u,u,u,u,u,u,18,19,26,27,u,u,u,u,u,u,u,u]911; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]912; AVX2-FP-NEXT: vpermq {{.*#+}} ymm4 = ymm1[0,2,1,3]913; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm5 = [0,1,8,9,6,7,14,15,6,7,14,15,2,3,10,11,16,17,24,25,16,17,24,25,24,25,26,27,18,19,26,27]914; AVX2-FP-NEXT: vpshufb %ymm5, %ymm4, %ymm4915; AVX2-FP-NEXT: vpermq {{.*#+}} ymm6 = ymm0[0,2,1,3]916; AVX2-FP-NEXT: vpshufb %ymm5, %ymm6, %ymm5917; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1],ymm4[2],ymm5[3,4],ymm4[5],ymm5[6,7]918; AVX2-FP-NEXT: vpermq {{.*#+}} ymm5 = ymm2[0,2,1,3]919; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[4,5,12,13,u,u,u,u,u,u,u,u,6,7,14,15,u,u,u,u,u,u,u,u,16,17,24,25,u,u,u,u]920; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1,2],ymm5[3],ymm4[4,5],ymm5[6],ymm4[7]921; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,3,1,3]922; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm5 = [2,3,10,11,8,9,10,11,4,5,12,13,4,5,12,13,18,19,26,27,22,23,30,31,22,23,30,31,20,21,28,29]923; AVX2-FP-NEXT: vpshufb %ymm5, %ymm0, %ymm0924; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[1,3,1,3]925; AVX2-FP-NEXT: vpshufb %ymm5, %ymm1, %ymm1926; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]927; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm2[1,3,1,3]928; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,2,3,10,11,u,u,u,u,u,u,u,u,20,21,28,29,u,u,u,u,u,u,u,u,22,23,30,31]929; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]930; AVX2-FP-NEXT: vmovdqa %ymm0, 64(%rax)931; AVX2-FP-NEXT: vmovdqa %ymm4, 32(%rax)932; AVX2-FP-NEXT: vmovdqa %ymm3, (%rax)933; AVX2-FP-NEXT: vzeroupper934; AVX2-FP-NEXT: retq935;936; AVX2-FCP-LABEL: store_i16_stride6_vf8:937; AVX2-FCP: # %bb.0:938; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax939; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm0940; AVX2-FCP-NEXT: vmovdqa (%rdx), %xmm1941; AVX2-FCP-NEXT: vmovdqa (%r8), %xmm2942; AVX2-FCP-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0943; AVX2-FCP-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1944; AVX2-FCP-NEXT: vinserti128 $1, (%r9), %ymm2, %ymm2945; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,4,1,5,0,4,1,5]946; AVX2-FCP-NEXT: # ymm3 = mem[0,1,0,1]947; AVX2-FCP-NEXT: vpermd %ymm1, %ymm3, %ymm3948; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,8,9,0,1,4,5,4,5,12,13,2,3,10,11,18,19,22,23,24,25,30,31,20,21,28,29,24,25,28,29]949; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm3, %ymm3950; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm5 = ymm0[0,2,0,2]951; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm5, %ymm4952; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5,6],ymm3[7]953; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [0,4,4,0,0,4,4,0]954; AVX2-FCP-NEXT: # ymm4 = mem[0,1,0,1]955; AVX2-FCP-NEXT: vpermd %ymm2, %ymm4, %ymm4956; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,12,13,8,9,u,u,u,u,u,u,u,u,18,19,22,23,u,u,u,u,u,u,u,u]957; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]958; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [2,6,1,5,2,6,1,5]959; AVX2-FCP-NEXT: # ymm4 = mem[0,1,0,1]960; AVX2-FCP-NEXT: vpermd %ymm1, %ymm4, %ymm4961; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm5 = [0,1,8,9,6,7,14,15,10,11,14,15,2,3,10,11,16,17,24,25,16,17,20,21,24,25,26,27,18,19,26,27]962; AVX2-FCP-NEXT: vpshufb %ymm5, %ymm4, %ymm4963; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm6 = ymm0[0,2,1,3]964; AVX2-FCP-NEXT: vpshufb %ymm5, %ymm6, %ymm5965; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0,1],ymm4[2],ymm5[3,4],ymm4[5],ymm5[6,7]966; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm5 = ymm2[0,2,1,3]967; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[4,5,12,13,u,u,u,u,u,u,u,u,6,7,14,15,u,u,u,u,u,u,u,u,16,17,24,25,u,u,u,u]968; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1,2],ymm5[3],ymm4[4,5],ymm5[6],ymm4[7]969; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [7,3,3,7,7,3,3,7]970; AVX2-FCP-NEXT: # ymm5 = mem[0,1,0,1]971; AVX2-FCP-NEXT: vpermd %ymm0, %ymm5, %ymm0972; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm5 = [2,3,10,11,8,9,10,11,8,9,12,13,4,5,12,13,18,19,26,27,22,23,18,19,22,23,30,31,20,21,28,29]973; AVX2-FCP-NEXT: vpshufb %ymm5, %ymm0, %ymm0974; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[1,3,1,3]975; AVX2-FCP-NEXT: vpshufb %ymm5, %ymm1, %ymm1976; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]977; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm2[1,3,1,3]978; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,2,3,10,11,u,u,u,u,u,u,u,u,20,21,28,29,u,u,u,u,u,u,u,u,22,23,30,31]979; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]980; AVX2-FCP-NEXT: vmovdqa %ymm0, 64(%rax)981; AVX2-FCP-NEXT: vmovdqa %ymm4, 32(%rax)982; AVX2-FCP-NEXT: vmovdqa %ymm3, (%rax)983; AVX2-FCP-NEXT: vzeroupper984; AVX2-FCP-NEXT: retq985;986; AVX512-LABEL: store_i16_stride6_vf8:987; AVX512: # %bb.0:988; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax989; AVX512-NEXT: vmovdqa (%rdi), %xmm0990; AVX512-NEXT: vmovdqa (%rdx), %xmm1991; AVX512-NEXT: vmovdqa (%r8), %xmm2992; AVX512-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1993; AVX512-NEXT: vpermq {{.*#+}} ymm3 = ymm1[0,2,1,3]994; AVX512-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,8,9,6,7,14,15,6,7,14,15,2,3,10,11,16,17,24,25,16,17,24,25,24,25,26,27,18,19,26,27]995; AVX512-NEXT: vpshufb %ymm4, %ymm3, %ymm3996; AVX512-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0997; AVX512-NEXT: vpermq {{.*#+}} ymm5 = ymm0[0,2,1,3]998; AVX512-NEXT: vpshufb %ymm4, %ymm5, %ymm4999; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7]1000; AVX512-NEXT: vinserti128 $1, (%r9), %ymm2, %ymm21001; AVX512-NEXT: vpermq {{.*#+}} ymm4 = ymm2[0,2,1,3]1002; AVX512-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[4,5,12,13,u,u,u,u,u,u,u,u,6,7,14,15,u,u,u,u,u,u,u,u,16,17,24,25,u,u,u,u]1003; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0],ymm3[1,2],ymm4[3],ymm3[4,5],ymm4[6],ymm3[7]1004; AVX512-NEXT: vpermq {{.*#+}} ymm4 = ymm1[0,2,0,2]1005; AVX512-NEXT: vmovdqa {{.*#+}} ymm5 = [0,1,8,9,0,1,8,9,4,5,12,13,2,3,10,11,18,19,26,27,24,25,30,31,20,21,28,29,20,21,28,29]1006; AVX512-NEXT: vpshufb %ymm5, %ymm4, %ymm41007; AVX512-NEXT: vpermq {{.*#+}} ymm6 = ymm0[0,2,0,2]1008; AVX512-NEXT: vpshufb %ymm5, %ymm6, %ymm51009; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5,6],ymm4[7]1010; AVX512-NEXT: vpermq {{.*#+}} ymm5 = ymm2[0,2,0,2]1011; AVX512-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,u,u,u,u,u,0,1,8,9,u,u,u,u,u,u,u,u,18,19,26,27,u,u,u,u,u,u,u,u]1012; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]1013; AVX512-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm31014; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,3,1,3]1015; AVX512-NEXT: vmovdqa {{.*#+}} ymm4 = [2,3,10,11,8,9,10,11,4,5,12,13,4,5,12,13,18,19,26,27,22,23,30,31,22,23,30,31,20,21,28,29]1016; AVX512-NEXT: vpshufb %ymm4, %ymm0, %ymm01017; AVX512-NEXT: vpermq {{.*#+}} ymm1 = ymm1[1,3,1,3]1018; AVX512-NEXT: vpshufb %ymm4, %ymm1, %ymm11019; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]1020; AVX512-NEXT: vpermq {{.*#+}} ymm1 = ymm2[1,3,1,3]1021; AVX512-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,2,3,10,11,u,u,u,u,u,u,u,u,20,21,28,29,u,u,u,u,u,u,u,u,22,23,30,31]1022; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]1023; AVX512-NEXT: vmovdqa %ymm0, 64(%rax)1024; AVX512-NEXT: vmovdqa64 %zmm3, (%rax)1025; AVX512-NEXT: vzeroupper1026; AVX512-NEXT: retq1027;1028; AVX512-FCP-LABEL: store_i16_stride6_vf8:1029; AVX512-FCP: # %bb.0:1030; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax1031; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm01032; AVX512-FCP-NEXT: vmovdqa (%rdx), %xmm11033; AVX512-FCP-NEXT: vmovdqa (%r8), %xmm21034; AVX512-FCP-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm11035; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [2,6,1,5,2,6,1,5]1036; AVX512-FCP-NEXT: # ymm3 = mem[0,1,0,1]1037; AVX512-FCP-NEXT: vpermd %ymm1, %ymm3, %ymm31038; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,8,9,6,7,14,15,10,11,14,15,2,3,10,11,16,17,24,25,16,17,20,21,24,25,26,27,18,19,26,27]1039; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm3, %ymm31040; AVX512-FCP-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm01041; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm5 = ymm0[0,2,1,3]1042; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm5, %ymm41043; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7]1044; AVX512-FCP-NEXT: vinserti128 $1, (%r9), %ymm2, %ymm21045; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm2[0,2,1,3]1046; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[4,5,12,13,u,u,u,u,u,u,u,u,6,7,14,15,u,u,u,u,u,u,u,u,16,17,24,25,u,u,u,u]1047; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0],ymm3[1,2],ymm4[3],ymm3[4,5],ymm4[6],ymm3[7]1048; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [0,4,1,5,0,4,1,5]1049; AVX512-FCP-NEXT: # ymm4 = mem[0,1,0,1]1050; AVX512-FCP-NEXT: vpermd %ymm1, %ymm4, %ymm41051; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm5 = [0,1,8,9,0,1,4,5,4,5,12,13,2,3,10,11,18,19,22,23,24,25,30,31,20,21,28,29,24,25,28,29]1052; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm4, %ymm41053; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm6 = ymm0[0,2,0,2]1054; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm6, %ymm51055; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5,6],ymm4[7]1056; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [0,4,4,0,0,4,4,0]1057; AVX512-FCP-NEXT: # ymm5 = mem[0,1,0,1]1058; AVX512-FCP-NEXT: vpermd %ymm2, %ymm5, %ymm51059; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,u,u,u,u,u,12,13,8,9,u,u,u,u,u,u,u,u,18,19,22,23,u,u,u,u,u,u,u,u]1060; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]1061; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm31062; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [7,3,3,7,7,3,3,7]1063; AVX512-FCP-NEXT: # ymm4 = mem[0,1,0,1]1064; AVX512-FCP-NEXT: vpermd %ymm0, %ymm4, %ymm01065; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [2,3,10,11,8,9,10,11,8,9,12,13,4,5,12,13,18,19,26,27,22,23,18,19,22,23,30,31,20,21,28,29]1066; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm0, %ymm01067; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[1,3,1,3]1068; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm1, %ymm11069; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]1070; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm2[1,3,1,3]1071; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,2,3,10,11,u,u,u,u,u,u,u,u,20,21,28,29,u,u,u,u,u,u,u,u,22,23,30,31]1072; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]1073; AVX512-FCP-NEXT: vmovdqa %ymm0, 64(%rax)1074; AVX512-FCP-NEXT: vmovdqa64 %zmm3, (%rax)1075; AVX512-FCP-NEXT: vzeroupper1076; AVX512-FCP-NEXT: retq1077;1078; AVX512DQ-LABEL: store_i16_stride6_vf8:1079; AVX512DQ: # %bb.0:1080; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax1081; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm01082; AVX512DQ-NEXT: vmovdqa (%rdx), %xmm11083; AVX512DQ-NEXT: vmovdqa (%r8), %xmm21084; AVX512DQ-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm11085; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = ymm1[0,2,1,3]1086; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,8,9,6,7,14,15,6,7,14,15,2,3,10,11,16,17,24,25,16,17,24,25,24,25,26,27,18,19,26,27]1087; AVX512DQ-NEXT: vpshufb %ymm4, %ymm3, %ymm31088; AVX512DQ-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm01089; AVX512DQ-NEXT: vpermq {{.*#+}} ymm5 = ymm0[0,2,1,3]1090; AVX512DQ-NEXT: vpshufb %ymm4, %ymm5, %ymm41091; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7]1092; AVX512DQ-NEXT: vinserti128 $1, (%r9), %ymm2, %ymm21093; AVX512DQ-NEXT: vpermq {{.*#+}} ymm4 = ymm2[0,2,1,3]1094; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[4,5,12,13,u,u,u,u,u,u,u,u,6,7,14,15,u,u,u,u,u,u,u,u,16,17,24,25,u,u,u,u]1095; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0],ymm3[1,2],ymm4[3],ymm3[4,5],ymm4[6],ymm3[7]1096; AVX512DQ-NEXT: vpermq {{.*#+}} ymm4 = ymm1[0,2,0,2]1097; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm5 = [0,1,8,9,0,1,8,9,4,5,12,13,2,3,10,11,18,19,26,27,24,25,30,31,20,21,28,29,20,21,28,29]1098; AVX512DQ-NEXT: vpshufb %ymm5, %ymm4, %ymm41099; AVX512DQ-NEXT: vpermq {{.*#+}} ymm6 = ymm0[0,2,0,2]1100; AVX512DQ-NEXT: vpshufb %ymm5, %ymm6, %ymm51101; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5,6],ymm4[7]1102; AVX512DQ-NEXT: vpermq {{.*#+}} ymm5 = ymm2[0,2,0,2]1103; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,u,u,u,u,u,0,1,8,9,u,u,u,u,u,u,u,u,18,19,26,27,u,u,u,u,u,u,u,u]1104; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]1105; AVX512DQ-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm31106; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,3,1,3]1107; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm4 = [2,3,10,11,8,9,10,11,4,5,12,13,4,5,12,13,18,19,26,27,22,23,30,31,22,23,30,31,20,21,28,29]1108; AVX512DQ-NEXT: vpshufb %ymm4, %ymm0, %ymm01109; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[1,3,1,3]1110; AVX512DQ-NEXT: vpshufb %ymm4, %ymm1, %ymm11111; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]1112; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm2[1,3,1,3]1113; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,2,3,10,11,u,u,u,u,u,u,u,u,20,21,28,29,u,u,u,u,u,u,u,u,22,23,30,31]1114; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]1115; AVX512DQ-NEXT: vmovdqa %ymm0, 64(%rax)1116; AVX512DQ-NEXT: vmovdqa64 %zmm3, (%rax)1117; AVX512DQ-NEXT: vzeroupper1118; AVX512DQ-NEXT: retq1119;1120; AVX512DQ-FCP-LABEL: store_i16_stride6_vf8:1121; AVX512DQ-FCP: # %bb.0:1122; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax1123; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm01124; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %xmm11125; AVX512DQ-FCP-NEXT: vmovdqa (%r8), %xmm21126; AVX512DQ-FCP-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm11127; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [2,6,1,5,2,6,1,5]1128; AVX512DQ-FCP-NEXT: # ymm3 = mem[0,1,0,1]1129; AVX512DQ-FCP-NEXT: vpermd %ymm1, %ymm3, %ymm31130; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,8,9,6,7,14,15,10,11,14,15,2,3,10,11,16,17,24,25,16,17,20,21,24,25,26,27,18,19,26,27]1131; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm3, %ymm31132; AVX512DQ-FCP-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm01133; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm5 = ymm0[0,2,1,3]1134; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm5, %ymm41135; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7]1136; AVX512DQ-FCP-NEXT: vinserti128 $1, (%r9), %ymm2, %ymm21137; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm2[0,2,1,3]1138; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[4,5,12,13,u,u,u,u,u,u,u,u,6,7,14,15,u,u,u,u,u,u,u,u,16,17,24,25,u,u,u,u]1139; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0],ymm3[1,2],ymm4[3],ymm3[4,5],ymm4[6],ymm3[7]1140; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [0,4,1,5,0,4,1,5]1141; AVX512DQ-FCP-NEXT: # ymm4 = mem[0,1,0,1]1142; AVX512DQ-FCP-NEXT: vpermd %ymm1, %ymm4, %ymm41143; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm5 = [0,1,8,9,0,1,4,5,4,5,12,13,2,3,10,11,18,19,22,23,24,25,30,31,20,21,28,29,24,25,28,29]1144; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm4, %ymm41145; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm6 = ymm0[0,2,0,2]1146; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm6, %ymm51147; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5,6],ymm4[7]1148; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [0,4,4,0,0,4,4,0]1149; AVX512DQ-FCP-NEXT: # ymm5 = mem[0,1,0,1]1150; AVX512DQ-FCP-NEXT: vpermd %ymm2, %ymm5, %ymm51151; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,u,u,u,u,u,12,13,8,9,u,u,u,u,u,u,u,u,18,19,22,23,u,u,u,u,u,u,u,u]1152; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]1153; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm31154; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [7,3,3,7,7,3,3,7]1155; AVX512DQ-FCP-NEXT: # ymm4 = mem[0,1,0,1]1156; AVX512DQ-FCP-NEXT: vpermd %ymm0, %ymm4, %ymm01157; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [2,3,10,11,8,9,10,11,8,9,12,13,4,5,12,13,18,19,26,27,22,23,18,19,22,23,30,31,20,21,28,29]1158; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm0, %ymm01159; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[1,3,1,3]1160; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm1, %ymm11161; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]1162; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm2[1,3,1,3]1163; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,2,3,10,11,u,u,u,u,u,u,u,u,20,21,28,29,u,u,u,u,u,u,u,u,22,23,30,31]1164; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]1165; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, 64(%rax)1166; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm3, (%rax)1167; AVX512DQ-FCP-NEXT: vzeroupper1168; AVX512DQ-FCP-NEXT: retq1169;1170; AVX512BW-LABEL: store_i16_stride6_vf8:1171; AVX512BW: # %bb.0:1172; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax1173; AVX512BW-NEXT: vmovdqa (%rdi), %xmm01174; AVX512BW-NEXT: vmovdqa (%rdx), %xmm11175; AVX512BW-NEXT: vmovdqa (%r8), %xmm21176; AVX512BW-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm11177; AVX512BW-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm01178; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm01179; AVX512BW-NEXT: vinserti32x4 $1, (%r9), %zmm2, %zmm11180; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [21,29,37,45,6,14,22,30,38,46,7,15,23,31,39,47]1181; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm21182; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,8,16,24,32,40,1,9,17,25,33,41,2,10,18,26,34,42,3,11,19,27,35,43,4,12,20,28,36,44,5,13]1183; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm31184; AVX512BW-NEXT: vmovdqa64 %zmm3, (%rax)1185; AVX512BW-NEXT: vmovdqa %ymm2, 64(%rax)1186; AVX512BW-NEXT: vzeroupper1187; AVX512BW-NEXT: retq1188;1189; AVX512BW-FCP-LABEL: store_i16_stride6_vf8:1190; AVX512BW-FCP: # %bb.0:1191; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax1192; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %xmm01193; AVX512BW-FCP-NEXT: vmovdqa (%rdx), %xmm11194; AVX512BW-FCP-NEXT: vmovdqa (%r8), %xmm21195; AVX512BW-FCP-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm11196; AVX512BW-FCP-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm01197; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm01198; AVX512BW-FCP-NEXT: vinserti32x4 $1, (%r9), %zmm2, %zmm11199; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [21,29,37,45,6,14,22,30,38,46,7,15,23,31,39,47]1200; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm21201; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,8,16,24,32,40,1,9,17,25,33,41,2,10,18,26,34,42,3,11,19,27,35,43,4,12,20,28,36,44,5,13]1202; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm31203; AVX512BW-FCP-NEXT: vmovdqa64 %zmm3, (%rax)1204; AVX512BW-FCP-NEXT: vmovdqa %ymm2, 64(%rax)1205; AVX512BW-FCP-NEXT: vzeroupper1206; AVX512BW-FCP-NEXT: retq1207;1208; AVX512DQ-BW-LABEL: store_i16_stride6_vf8:1209; AVX512DQ-BW: # %bb.0:1210; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %rax1211; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %xmm01212; AVX512DQ-BW-NEXT: vmovdqa (%rdx), %xmm11213; AVX512DQ-BW-NEXT: vmovdqa (%r8), %xmm21214; AVX512DQ-BW-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm11215; AVX512DQ-BW-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm01216; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm01217; AVX512DQ-BW-NEXT: vinserti32x4 $1, (%r9), %zmm2, %zmm11218; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [21,29,37,45,6,14,22,30,38,46,7,15,23,31,39,47]1219; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm21220; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,8,16,24,32,40,1,9,17,25,33,41,2,10,18,26,34,42,3,11,19,27,35,43,4,12,20,28,36,44,5,13]1221; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm31222; AVX512DQ-BW-NEXT: vmovdqa64 %zmm3, (%rax)1223; AVX512DQ-BW-NEXT: vmovdqa %ymm2, 64(%rax)1224; AVX512DQ-BW-NEXT: vzeroupper1225; AVX512DQ-BW-NEXT: retq1226;1227; AVX512DQ-BW-FCP-LABEL: store_i16_stride6_vf8:1228; AVX512DQ-BW-FCP: # %bb.0:1229; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax1230; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %xmm01231; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdx), %xmm11232; AVX512DQ-BW-FCP-NEXT: vmovdqa (%r8), %xmm21233; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm11234; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm01235; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm01236; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, (%r9), %zmm2, %zmm11237; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [21,29,37,45,6,14,22,30,38,46,7,15,23,31,39,47]1238; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm21239; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,8,16,24,32,40,1,9,17,25,33,41,2,10,18,26,34,42,3,11,19,27,35,43,4,12,20,28,36,44,5,13]1240; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm31241; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm3, (%rax)1242; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm2, 64(%rax)1243; AVX512DQ-BW-FCP-NEXT: vzeroupper1244; AVX512DQ-BW-FCP-NEXT: retq1245 %in.vec0 = load <8 x i16>, ptr %in.vecptr0, align 641246 %in.vec1 = load <8 x i16>, ptr %in.vecptr1, align 641247 %in.vec2 = load <8 x i16>, ptr %in.vecptr2, align 641248 %in.vec3 = load <8 x i16>, ptr %in.vecptr3, align 641249 %in.vec4 = load <8 x i16>, ptr %in.vecptr4, align 641250 %in.vec5 = load <8 x i16>, ptr %in.vecptr5, align 641251 %1 = shufflevector <8 x i16> %in.vec0, <8 x i16> %in.vec1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>1252 %2 = shufflevector <8 x i16> %in.vec2, <8 x i16> %in.vec3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>1253 %3 = shufflevector <8 x i16> %in.vec4, <8 x i16> %in.vec5, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>1254 %4 = shufflevector <16 x i16> %1, <16 x i16> %2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>1255 %5 = shufflevector <16 x i16> %3, <16 x i16> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>1256 %6 = shufflevector <32 x i16> %4, <32 x i16> %5, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>1257 %interleaved.vec = shufflevector <48 x i16> %6, <48 x i16> poison, <48 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 40, i32 1, i32 9, i32 17, i32 25, i32 33, i32 41, i32 2, i32 10, i32 18, i32 26, i32 34, i32 42, i32 3, i32 11, i32 19, i32 27, i32 35, i32 43, i32 4, i32 12, i32 20, i32 28, i32 36, i32 44, i32 5, i32 13, i32 21, i32 29, i32 37, i32 45, i32 6, i32 14, i32 22, i32 30, i32 38, i32 46, i32 7, i32 15, i32 23, i32 31, i32 39, i32 47>1258 store <48 x i16> %interleaved.vec, ptr %out.vec, align 641259 ret void1260}1261 1262define void @store_i16_stride6_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %in.vecptr3, ptr %in.vecptr4, ptr %in.vecptr5, ptr %out.vec) nounwind {1263; SSE-LABEL: store_i16_stride6_vf16:1264; SSE: # %bb.0:1265; SSE-NEXT: subq $24, %rsp1266; SSE-NEXT: movdqa (%rdi), %xmm151267; SSE-NEXT: movdqa 16(%rdi), %xmm111268; SSE-NEXT: movdqa (%rsi), %xmm101269; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1270; SSE-NEXT: movdqa 16(%rsi), %xmm41271; SSE-NEXT: movdqa (%rdx), %xmm141272; SSE-NEXT: movdqa 16(%rdx), %xmm121273; SSE-NEXT: movdqa (%rcx), %xmm31274; SSE-NEXT: movdqa 16(%rcx), %xmm21275; SSE-NEXT: movdqa 16(%r8), %xmm01276; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1277; SSE-NEXT: movdqa 16(%r9), %xmm81278; SSE-NEXT: movdqa %xmm12, %xmm61279; SSE-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]1280; SSE-NEXT: movdqa %xmm11, %xmm51281; SSE-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]1282; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1283; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[2,3],xmm6[3,3]1284; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,3,3,4,5,6,7]1285; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[1,2],xmm0[0,1]1286; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[2,0,1,3]1287; SSE-NEXT: movaps {{.*#+}} xmm7 = [65535,0,65535,65535,65535,65535,65535,0]1288; SSE-NEXT: andps %xmm7, %xmm51289; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm8[0,2,2,3,4,5,6,7]1290; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,1,2,1]1291; SSE-NEXT: movaps %xmm7, %xmm01292; SSE-NEXT: andnps %xmm9, %xmm01293; SSE-NEXT: orps %xmm5, %xmm01294; SSE-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill1295; SSE-NEXT: movdqa %xmm14, %xmm51296; SSE-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]1297; SSE-NEXT: movdqa %xmm15, %xmm131298; SSE-NEXT: punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm10[4],xmm13[5],xmm10[5],xmm13[6],xmm10[6],xmm13[7],xmm10[7]1299; SSE-NEXT: movdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1300; SSE-NEXT: shufps {{.*#+}} xmm13 = xmm13[2,3],xmm5[3,3]1301; SSE-NEXT: movdqa (%r8), %xmm101302; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm10[0,1,2,3,6,5,7,7]1303; SSE-NEXT: shufps {{.*#+}} xmm13 = xmm13[1,2],xmm9[2,3]1304; SSE-NEXT: movdqa (%r9), %xmm91305; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm9[0,1,2,3,4,6,6,7]1306; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]1307; SSE-NEXT: movaps %xmm7, %xmm01308; SSE-NEXT: andnps %xmm1, %xmm01309; SSE-NEXT: shufps {{.*#+}} xmm13 = xmm13[2,0,1,3]1310; SSE-NEXT: andps %xmm7, %xmm131311; SSE-NEXT: orps %xmm13, %xmm01312; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1313; SSE-NEXT: punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm2[4],xmm12[5],xmm2[5],xmm12[6],xmm2[6],xmm12[7],xmm2[7]1314; SSE-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm4[4],xmm11[5],xmm4[5],xmm11[6],xmm4[6],xmm11[7],xmm4[7]1315; SSE-NEXT: movdqa %xmm11, %xmm11316; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,3],xmm12[3,3]1317; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload1318; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm13[0,1,2,3,6,5,7,7]1319; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,2],xmm4[2,3]1320; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1321; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm8[0,1,2,3,4,6,6,7]1322; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[2,1,2,3]1323; SSE-NEXT: movaps %xmm7, %xmm01324; SSE-NEXT: andnps %xmm4, %xmm01325; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0,1,3]1326; SSE-NEXT: andps %xmm7, %xmm11327; SSE-NEXT: orps %xmm1, %xmm01328; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1329; SSE-NEXT: punpcklwd {{.*#+}} xmm14 = xmm14[0],xmm3[0],xmm14[1],xmm3[1],xmm14[2],xmm3[2],xmm14[3],xmm3[3]1330; SSE-NEXT: punpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload1331; SSE-NEXT: # xmm15 = xmm15[0],mem[0],xmm15[1],mem[1],xmm15[2],mem[2],xmm15[3],mem[3]1332; SSE-NEXT: movdqa %xmm15, %xmm11333; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,3],xmm14[3,3]1334; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm10[2,1,3,3,4,5,6,7]1335; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,2],xmm3[0,1]1336; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0,1,3]1337; SSE-NEXT: andps %xmm7, %xmm11338; SSE-NEXT: movdqa %xmm9, %xmm21339; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1340; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm9[0,2,2,3,4,5,6,7]1341; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,2,1]1342; SSE-NEXT: andnps %xmm3, %xmm71343; SSE-NEXT: orps %xmm1, %xmm71344; SSE-NEXT: movdqa %xmm12, %xmm31345; SSE-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm11[0]1346; SSE-NEXT: movdqa %xmm13, %xmm11347; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,1],xmm11[1,3]1348; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm1[0,2]1349; SSE-NEXT: pslld $16, %xmm81350; SSE-NEXT: movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535,65535,0,65535,65535]1351; SSE-NEXT: movdqa %xmm1, %xmm131352; SSE-NEXT: pandn %xmm8, %xmm131353; SSE-NEXT: andps %xmm1, %xmm31354; SSE-NEXT: por %xmm3, %xmm131355; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload1356; SSE-NEXT: unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm5[1]1357; SSE-NEXT: movdqa %xmm10, %xmm41358; SSE-NEXT: psrldq {{.*#+}} xmm4 = xmm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero1359; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[1,1],xmm5[1,1]1360; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm3[0,2]1361; SSE-NEXT: movdqa {{.*#+}} xmm3 = [65535,65535,65535,0,65535,65535,65535,65535]1362; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm9[2,2,3,3]1363; SSE-NEXT: movdqa %xmm3, %xmm91364; SSE-NEXT: pandn %xmm0, %xmm91365; SSE-NEXT: andps %xmm3, %xmm41366; SSE-NEXT: por %xmm4, %xmm91367; SSE-NEXT: movdqa %xmm15, %xmm01368; SSE-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm14[1]1369; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm10[1,1,1,1,4,5,6,7]1370; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[1,1],xmm14[1,1]1371; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm0[0,2]1372; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,1,1]1373; SSE-NEXT: movdqa %xmm3, %xmm81374; SSE-NEXT: pandn %xmm0, %xmm81375; SSE-NEXT: andps %xmm3, %xmm41376; SSE-NEXT: por %xmm4, %xmm81377; SSE-NEXT: punpcklqdq {{.*#+}} xmm14 = xmm14[0],xmm15[0]1378; SSE-NEXT: movdqa %xmm10, %xmm01379; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm15[1,3]1380; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[2,0],xmm0[0,2]1381; SSE-NEXT: movdqa %xmm2, %xmm01382; SSE-NEXT: pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]1383; SSE-NEXT: movdqa %xmm1, %xmm151384; SSE-NEXT: pandn %xmm0, %xmm151385; SSE-NEXT: andps %xmm1, %xmm141386; SSE-NEXT: por %xmm14, %xmm151387; SSE-NEXT: unpckhpd {{.*#+}} xmm11 = xmm11[1],xmm12[1]1388; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload1389; SSE-NEXT: movdqa %xmm14, %xmm01390; SSE-NEXT: psrldq {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero1391; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm12[1,1]1392; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm11[0,2]1393; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload1394; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm12[2,2,3,3]1395; SSE-NEXT: movdqa %xmm3, %xmm111396; SSE-NEXT: pandn %xmm4, %xmm111397; SSE-NEXT: andps %xmm3, %xmm01398; SSE-NEXT: por %xmm0, %xmm111399; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload1400; SSE-NEXT: movaps %xmm2, %xmm01401; SSE-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm6[1]1402; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm14[1,1,1,1,4,5,6,7]1403; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[1,1],xmm6[1,1]1404; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm0[0,2]1405; SSE-NEXT: andps %xmm3, %xmm41406; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm12[0,0,1,1]1407; SSE-NEXT: pandn %xmm0, %xmm31408; SSE-NEXT: por %xmm4, %xmm31409; SSE-NEXT: movlhps {{.*#+}} xmm6 = xmm6[0],xmm2[0]1410; SSE-NEXT: movdqa %xmm14, %xmm01411; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[1,3]1412; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[2,0],xmm0[0,2]1413; SSE-NEXT: movdqa %xmm12, %xmm41414; SSE-NEXT: pslldq {{.*#+}} xmm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm4[0,1,2,3,4,5]1415; SSE-NEXT: movdqa %xmm1, %xmm01416; SSE-NEXT: pandn %xmm4, %xmm01417; SSE-NEXT: andps %xmm1, %xmm61418; SSE-NEXT: por %xmm6, %xmm01419; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload1420; SSE-NEXT: movlhps {{.*#+}} xmm5 = xmm5[0],xmm2[0]1421; SSE-NEXT: shufps {{.*#+}} xmm10 = xmm10[2,1],xmm2[1,3]1422; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[2,0],xmm10[0,2]1423; SSE-NEXT: andps %xmm1, %xmm51424; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload1425; SSE-NEXT: pslld $16, %xmm21426; SSE-NEXT: pandn %xmm2, %xmm11427; SSE-NEXT: por %xmm5, %xmm11428; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax1429; SSE-NEXT: movdqa %xmm1, 48(%rax)1430; SSE-NEXT: movdqa %xmm0, 96(%rax)1431; SSE-NEXT: movdqa %xmm3, 112(%rax)1432; SSE-NEXT: movdqa %xmm11, 160(%rax)1433; SSE-NEXT: movdqa %xmm15, (%rax)1434; SSE-NEXT: movdqa %xmm8, 16(%rax)1435; SSE-NEXT: movdqa %xmm9, 64(%rax)1436; SSE-NEXT: movdqa %xmm13, 144(%rax)1437; SSE-NEXT: movaps %xmm7, 32(%rax)1438; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1439; SSE-NEXT: movaps %xmm0, 176(%rax)1440; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1441; SSE-NEXT: movaps %xmm0, 80(%rax)1442; SSE-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload1443; SSE-NEXT: movaps %xmm0, 128(%rax)1444; SSE-NEXT: addq $24, %rsp1445; SSE-NEXT: retq1446;1447; AVX-LABEL: store_i16_stride6_vf16:1448; AVX: # %bb.0:1449; AVX-NEXT: vmovdqa (%rcx), %xmm31450; AVX-NEXT: vmovdqa 16(%rcx), %xmm01451; AVX-NEXT: vmovdqa (%rdx), %xmm41452; AVX-NEXT: vmovdqa 16(%rdx), %xmm11453; AVX-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]1454; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm8[2,2,3,3]1455; AVX-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]1456; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,0,1,1]1457; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm11458; AVX-NEXT: vmovdqa (%rsi), %xmm51459; AVX-NEXT: vmovdqa 16(%rsi), %xmm21460; AVX-NEXT: vmovdqa (%rdi), %xmm61461; AVX-NEXT: vmovdqa 16(%rdi), %xmm71462; AVX-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3]1463; AVX-NEXT: vpshufd {{.*#+}} xmm10 = xmm9[2,3,2,3]1464; AVX-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm7[4],xmm2[4],xmm7[5],xmm2[5],xmm7[6],xmm2[6],xmm7[7],xmm2[7]1465; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm2[0,1,0,1]1466; AVX-NEXT: vinsertf128 $1, %xmm7, %ymm10, %ymm71467; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm7[0,1],ymm1[2],ymm7[3,4],ymm1[5],ymm7[6,7]1468; AVX-NEXT: vextractf128 $1, %ymm1, %xmm71469; AVX-NEXT: vmovdqa 16(%r8), %xmm101470; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3],xmm10[4,5],xmm7[6,7]1471; AVX-NEXT: vmovdqa 16(%r9), %xmm111472; AVX-NEXT: vpslld $16, %xmm11, %xmm121473; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4],xmm12[5],xmm7[6,7]1474; AVX-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1475; AVX-NEXT: vpshuflw {{.*#+}} xmm7 = xmm10[2,1,3,3,4,5,6,7]1476; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[0,1,2,1]1477; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm7[0],xmm1[1,2],xmm7[3]1478; AVX-NEXT: vpshuflw {{.*#+}} xmm7 = xmm11[0,2,2,3,4,5,6,7]1479; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[0,1,2,1]1480; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm7[1],xmm1[2,3,4,5,6],xmm7[7]1481; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1482; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]1483; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm11484; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,2]1485; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]1486; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm01487; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]1488; AVX-NEXT: vpsrldq {{.*#+}} xmm1 = xmm10[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero1489; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2,3]1490; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm11[2,2,3,3]1491; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3],xmm1[4,5,6,7]1492; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1493; AVX-NEXT: vextractf128 $1, %ymm0, %xmm01494; AVX-NEXT: vpshufhw {{.*#+}} xmm1 = xmm10[0,1,2,3,6,5,7,7]1495; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]1496; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3,4,5],xmm1[6,7]1497; AVX-NEXT: vpshufhw {{.*#+}} xmm1 = xmm11[0,1,2,3,4,6,6,7]1498; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]1499; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6],xmm1[7]1500; AVX-NEXT: vpunpckhwd {{.*#+}} xmm13 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]1501; AVX-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]1502; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm1[1,1,2,2]1503; AVX-NEXT: vpshufd {{.*#+}} xmm12 = xmm1[2,2,3,3]1504; AVX-NEXT: vinsertf128 $1, %xmm12, %ymm0, %ymm01505; AVX-NEXT: vpshufd {{.*#+}} xmm12 = xmm13[2,3,2,3]1506; AVX-NEXT: vinsertf128 $1, %xmm12, %ymm13, %ymm121507; AVX-NEXT: vblendps {{.*#+}} ymm15 = ymm0[0,1],ymm12[2],ymm0[3,4],ymm12[5],ymm0[6,7]1508; AVX-NEXT: vmovdqa (%r8), %xmm141509; AVX-NEXT: vpshufhw {{.*#+}} xmm0 = xmm14[0,1,2,3,6,5,7,7]1510; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]1511; AVX-NEXT: vextractf128 $1, %ymm15, %xmm121512; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm0[0,1],xmm12[2,3,4,5],xmm0[6,7]1513; AVX-NEXT: vmovdqa (%r9), %xmm01514; AVX-NEXT: vpshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,6,6,7]1515; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,1,2,3]1516; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0],xmm2[1],xmm12[2,3,4,5,6],xmm2[7]1517; AVX-NEXT: vpsrldq {{.*#+}} xmm2 = xmm14[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero1518; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm15[0],xmm2[1],xmm15[2,3]1519; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm0[2,2,3,3]1520; AVX-NEXT: vpblendw {{.*#+}} xmm15 = xmm2[0,1,2],xmm15[3],xmm2[4,5,6,7]1521; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm8[0,0,1,1]1522; AVX-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[1,1,2,2]1523; AVX-NEXT: vinsertf128 $1, %xmm8, %ymm2, %ymm21524; AVX-NEXT: vpshufd {{.*#+}} xmm8 = xmm9[0,1,0,1]1525; AVX-NEXT: vinsertf128 $1, %xmm9, %ymm8, %ymm81526; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm8[0],ymm2[1],ymm8[2,3],ymm2[4],ymm8[5,6],ymm2[7]1527; AVX-NEXT: vinsertps {{.*#+}} xmm8 = xmm2[0,1],xmm10[0],xmm2[3]1528; AVX-NEXT: vpslldq {{.*#+}} xmm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm11[0,1,2,3,4,5]1529; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2,3,4],xmm9[5],xmm8[6,7]1530; AVX-NEXT: vextractf128 $1, %ymm2, %xmm21531; AVX-NEXT: vpmovzxwd {{.*#+}} xmm9 = xmm10[0],zero,xmm10[1],zero,xmm10[2],zero,xmm10[3],zero1532; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm9[2,3],xmm2[4,5,6,7]1533; AVX-NEXT: vpshufd {{.*#+}} xmm9 = xmm11[0,0,1,1]1534; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm2[0,1,2],xmm9[3],xmm2[4,5,6,7]1535; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]1536; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,0,1,1]1537; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[1,1,2,2]1538; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm31539; AVX-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]1540; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[0,1,0,1]1541; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm5, %ymm51542; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm5[0],ymm3[1],ymm5[2,3],ymm3[4],ymm5[5,6],ymm3[7]1543; AVX-NEXT: vextractf128 $1, %ymm3, %xmm51544; AVX-NEXT: vpmovzxwd {{.*#+}} xmm6 = xmm14[0],zero,xmm14[1],zero,xmm14[2],zero,xmm14[3],zero1545; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1],xmm6[2,3],xmm5[4,5,6,7]1546; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm0[0,0,1,1]1547; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2],xmm6[3],xmm5[4,5,6,7]1548; AVX-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm14[0],xmm3[3]1549; AVX-NEXT: vpslldq {{.*#+}} xmm6 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]1550; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm6[5],xmm3[6,7]1551; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]1552; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]1553; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm11554; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]1555; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm13[0,1,0,1]1556; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm2, %ymm21557; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]1558; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm14[2,1,3,3,4,5,6,7]1559; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,2,1]1560; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm2[0],xmm1[1,2],xmm2[3]1561; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm0[0,2,2,3,4,5,6,7]1562; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,1,2,1]1563; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm4[1],xmm2[2,3,4,5,6],xmm4[7]1564; AVX-NEXT: vextractf128 $1, %ymm1, %xmm11565; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm14[4,5],xmm1[6,7]1566; AVX-NEXT: vpslld $16, %xmm0, %xmm01567; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4],xmm0[5],xmm1[6,7]1568; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax1569; AVX-NEXT: vmovdqa %xmm0, 48(%rax)1570; AVX-NEXT: vmovdqa %xmm2, 32(%rax)1571; AVX-NEXT: vmovdqa %xmm3, (%rax)1572; AVX-NEXT: vmovdqa %xmm5, 16(%rax)1573; AVX-NEXT: vmovdqa %xmm9, 112(%rax)1574; AVX-NEXT: vmovdqa %xmm8, 96(%rax)1575; AVX-NEXT: vmovdqa %xmm15, 64(%rax)1576; AVX-NEXT: vmovdqa %xmm12, 80(%rax)1577; AVX-NEXT: vmovdqa %xmm7, 176(%rax)1578; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1579; AVX-NEXT: vmovaps %xmm0, 160(%rax)1580; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1581; AVX-NEXT: vmovaps %xmm0, 128(%rax)1582; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1583; AVX-NEXT: vmovaps %xmm0, 144(%rax)1584; AVX-NEXT: vzeroupper1585; AVX-NEXT: retq1586;1587; AVX2-LABEL: store_i16_stride6_vf16:1588; AVX2: # %bb.0:1589; AVX2-NEXT: vmovdqa (%rdi), %ymm11590; AVX2-NEXT: vmovdqa (%rsi), %ymm31591; AVX2-NEXT: vmovdqa (%rdx), %ymm21592; AVX2-NEXT: vmovdqa (%rcx), %ymm41593; AVX2-NEXT: vmovdqa (%r8), %ymm131594; AVX2-NEXT: vmovdqa (%rcx), %xmm61595; AVX2-NEXT: vpsrldq {{.*#+}} xmm5 = xmm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero1596; AVX2-NEXT: vmovdqa (%rdx), %xmm71597; AVX2-NEXT: vpsrldq {{.*#+}} xmm8 = xmm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero1598; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]1599; AVX2-NEXT: vpbroadcastq %xmm5, %ymm51600; AVX2-NEXT: vmovdqa (%rsi), %xmm81601; AVX2-NEXT: vpshufd {{.*#+}} xmm9 = xmm8[0,1,2,1]1602; AVX2-NEXT: vpshufhw {{.*#+}} xmm10 = xmm9[0,1,2,3,4,7,6,5]1603; AVX2-NEXT: vmovdqa (%rdi), %xmm91604; AVX2-NEXT: vpshufd {{.*#+}} xmm11 = xmm9[0,1,2,1]1605; AVX2-NEXT: vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,7,6,5]1606; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm10 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]1607; AVX2-NEXT: vpermq {{.*#+}} ymm10 = ymm10[0,1,0,1]1608; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm10[0,1],ymm5[2],ymm10[3,4],ymm5[5],ymm10[6,7]1609; AVX2-NEXT: vmovdqa (%r8), %xmm101610; AVX2-NEXT: vpshuflw {{.*#+}} xmm11 = xmm10[2,1,3,3,4,5,6,7]1611; AVX2-NEXT: vpermq {{.*#+}} ymm11 = ymm11[0,0,2,1]1612; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm11[0],ymm5[1,2],ymm11[3],ymm5[4,5],ymm11[6],ymm5[7]1613; AVX2-NEXT: vmovdqa (%r9), %xmm111614; AVX2-NEXT: vpshuflw {{.*#+}} xmm12 = xmm11[0,2,2,3,4,5,6,7]1615; AVX2-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,4,4,4]1616; AVX2-NEXT: vpermq {{.*#+}} ymm12 = ymm12[0,0,2,1]1617; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm0 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]1618; AVX2-NEXT: vpblendvb %ymm0, %ymm5, %ymm12, %ymm51619; AVX2-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill1620; AVX2-NEXT: vpsrldq {{.*#+}} ymm12 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero1621; AVX2-NEXT: vpsrldq {{.*#+}} ymm14 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero1622; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm14 = ymm14[0],ymm12[0],ymm14[1],ymm12[1],ymm14[2],ymm12[2],ymm14[3],ymm12[3],ymm14[8],ymm12[8],ymm14[9],ymm12[9],ymm14[10],ymm12[10],ymm14[11],ymm12[11]1623; AVX2-NEXT: vpshufd {{.*#+}} ymm12 = ymm3[2,1,2,3,6,5,6,7]1624; AVX2-NEXT: vpshuflw {{.*#+}} ymm12 = ymm12[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]1625; AVX2-NEXT: vpshufd {{.*#+}} ymm15 = ymm1[2,1,2,3,6,5,6,7]1626; AVX2-NEXT: vpshuflw {{.*#+}} ymm15 = ymm15[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]1627; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm15 = ymm15[0],ymm12[0],ymm15[1],ymm12[1],ymm15[2],ymm12[2],ymm15[3],ymm12[3],ymm15[8],ymm12[8],ymm15[9],ymm12[9],ymm15[10],ymm12[10],ymm15[11],ymm12[11]1628; AVX2-NEXT: vmovdqa (%r9), %ymm121629; AVX2-NEXT: vpermq {{.*#+}} ymm14 = ymm14[2,2,2,2]1630; AVX2-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,1,2,3]1631; AVX2-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1],ymm14[2],ymm15[3,4],ymm14[5],ymm15[6,7]1632; AVX2-NEXT: vpshuflw {{.*#+}} ymm15 = ymm13[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]1633; AVX2-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,2,2,3]1634; AVX2-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0],ymm14[1,2],ymm15[3],ymm14[4,5],ymm15[6],ymm14[7]1635; AVX2-NEXT: vpshuflw {{.*#+}} ymm15 = ymm12[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]1636; AVX2-NEXT: vpshufhw {{.*#+}} ymm15 = ymm15[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]1637; AVX2-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,2,2,3]1638; AVX2-NEXT: vpblendvb %ymm0, %ymm14, %ymm15, %ymm01639; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill1640; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm14 = xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]1641; AVX2-NEXT: vpermq {{.*#+}} ymm14 = ymm14[1,1,1,1]1642; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm15 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]1643; AVX2-NEXT: vpshufd {{.*#+}} xmm15 = xmm15[1,2,3,3]1644; AVX2-NEXT: vpermq {{.*#+}} ymm15 = ymm15[0,0,2,1]1645; AVX2-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0,1],ymm14[2],ymm15[3,4],ymm14[5],ymm15[6,7]1646; AVX2-NEXT: vpshufb {{.*#+}} xmm15 = xmm10[12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]1647; AVX2-NEXT: vpermq {{.*#+}} ymm15 = ymm15[0,1,0,1]1648; AVX2-NEXT: vpblendd {{.*#+}} ymm14 = ymm14[0],ymm15[1],ymm14[2,3],ymm15[4],ymm14[5,6],ymm15[7]1649; AVX2-NEXT: vpshufd {{.*#+}} xmm15 = xmm11[2,3,2,3]1650; AVX2-NEXT: vpshuflw {{.*#+}} xmm15 = xmm15[0,2,2,1,4,5,6,7]1651; AVX2-NEXT: vpermq {{.*#+}} ymm15 = ymm15[0,1,0,1]1652; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm0 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]1653; AVX2-NEXT: vpblendvb %ymm0, %ymm14, %ymm15, %ymm141654; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm15 = ymm1[4],ymm3[4],ymm1[5],ymm3[5],ymm1[6],ymm3[6],ymm1[7],ymm3[7],ymm1[12],ymm3[12],ymm1[13],ymm3[13],ymm1[14],ymm3[14],ymm1[15],ymm3[15]1655; AVX2-NEXT: vpermq {{.*#+}} ymm15 = ymm15[3,3,3,3]1656; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm2[4],ymm4[4],ymm2[5],ymm4[5],ymm2[6],ymm4[6],ymm2[7],ymm4[7],ymm2[12],ymm4[12],ymm2[13],ymm4[13],ymm2[14],ymm4[14],ymm2[15],ymm4[15]1657; AVX2-NEXT: vpshufd {{.*#+}} ymm5 = ymm5[1,2,3,3,5,6,7,7]1658; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm5[2,2,2,3]1659; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1],ymm15[2],ymm5[3,4],ymm15[5],ymm5[6,7]1660; AVX2-NEXT: vpshufb {{.*#+}} ymm15 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]1661; AVX2-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,1,2,3]1662; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0],ymm15[1],ymm5[2,3],ymm15[4],ymm5[5,6],ymm15[7]1663; AVX2-NEXT: vpshufd {{.*#+}} ymm15 = ymm12[2,3,2,3,6,7,6,7]1664; AVX2-NEXT: vpshuflw {{.*#+}} ymm15 = ymm15[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]1665; AVX2-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,1,2,3]1666; AVX2-NEXT: vpblendvb %ymm0, %ymm5, %ymm15, %ymm01667; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]1668; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]1669; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm5[0,0,2,1]1670; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[1,0,2,2]1671; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,1,0,1]1672; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0],ymm6[1],ymm5[2,3],ymm6[4],ymm5[5,6],ymm6[7]1673; AVX2-NEXT: vpmovzxwd {{.*#+}} xmm6 = xmm10[0],zero,xmm10[1],zero,xmm10[2],zero,xmm10[3],zero1674; AVX2-NEXT: vpbroadcastq %xmm6, %ymm61675; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1],ymm6[2],ymm5[3,4],ymm6[5],ymm5[6,7]1676; AVX2-NEXT: vpshuflw {{.*#+}} xmm6 = xmm11[0,0,2,1,4,5,6,7]1677; AVX2-NEXT: vpbroadcastq %xmm6, %ymm61678; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm7 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]1679; AVX2-NEXT: vpblendvb %ymm7, %ymm5, %ymm6, %ymm51680; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[8],ymm3[8],ymm1[9],ymm3[9],ymm1[10],ymm3[10],ymm1[11],ymm3[11]1681; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[8],ymm4[8],ymm2[9],ymm4[9],ymm2[10],ymm4[10],ymm2[11],ymm4[11]1682; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]1683; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[1,0,2,2,5,4,6,6]1684; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]1685; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7]1686; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm13[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]1687; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]1688; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]1689; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm12[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]1690; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]1691; AVX2-NEXT: vpblendvb %ymm7, %ymm1, %ymm2, %ymm11692; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax1693; AVX2-NEXT: vmovdqa %ymm1, 96(%rax)1694; AVX2-NEXT: vmovdqa %ymm0, 160(%rax)1695; AVX2-NEXT: vmovdqa %ymm14, 64(%rax)1696; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload1697; AVX2-NEXT: vmovaps %ymm0, 128(%rax)1698; AVX2-NEXT: vmovdqa %ymm5, (%rax)1699; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload1700; AVX2-NEXT: vmovaps %ymm0, 32(%rax)1701; AVX2-NEXT: vzeroupper1702; AVX2-NEXT: retq1703;1704; AVX2-FP-LABEL: store_i16_stride6_vf16:1705; AVX2-FP: # %bb.0:1706; AVX2-FP-NEXT: subq $24, %rsp1707; AVX2-FP-NEXT: vmovdqa (%rdi), %ymm01708; AVX2-FP-NEXT: vmovdqa (%rsi), %ymm101709; AVX2-FP-NEXT: vmovdqa (%rdx), %ymm11710; AVX2-FP-NEXT: vmovdqa (%rcx), %ymm21711; AVX2-FP-NEXT: vmovdqa (%r8), %ymm81712; AVX2-FP-NEXT: vmovaps (%r9), %ymm31713; AVX2-FP-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill1714; AVX2-FP-NEXT: vmovdqa (%rsi), %xmm71715; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm61716; AVX2-FP-NEXT: vmovdqa (%rcx), %xmm91717; AVX2-FP-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1718; AVX2-FP-NEXT: vmovdqa (%rdx), %xmm41719; AVX2-FP-NEXT: vmovdqa (%r8), %xmm51720; AVX2-FP-NEXT: vmovdqa (%r9), %xmm31721; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7]1722; AVX2-FP-NEXT: vpermq {{.*#+}} ymm12 = ymm12[1,1,1,1]1723; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm13 = xmm4[4],xmm9[4],xmm4[5],xmm9[5],xmm4[6],xmm9[6],xmm4[7],xmm9[7]1724; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[1,2,3,3]1725; AVX2-FP-NEXT: vpermq {{.*#+}} ymm13 = ymm13[0,0,2,1]1726; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0,1],ymm12[2],ymm13[3,4],ymm12[5],ymm13[6,7]1727; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm13 = xmm5[12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]1728; AVX2-FP-NEXT: vpermq {{.*#+}} ymm13 = ymm13[0,1,0,1]1729; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm12 = ymm12[0],ymm13[1],ymm12[2,3],ymm13[4],ymm12[5,6],ymm13[7]1730; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm13 = xmm3[8,9,12,13,12,13,10,11,8,9,10,11,12,13,14,15]1731; AVX2-FP-NEXT: vpermq {{.*#+}} ymm13 = ymm13[0,1,0,1]1732; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm14 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]1733; AVX2-FP-NEXT: vpblendvb %ymm14, %ymm12, %ymm13, %ymm91734; AVX2-FP-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill1735; AVX2-FP-NEXT: vmovdqa %ymm0, %ymm91736; AVX2-FP-NEXT: vmovdqa %ymm10, %ymm111737; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} ymm13 = ymm0[4],ymm10[4],ymm0[5],ymm10[5],ymm0[6],ymm10[6],ymm0[7],ymm10[7],ymm0[12],ymm10[12],ymm0[13],ymm10[13],ymm0[14],ymm10[14],ymm0[15],ymm10[15]1738; AVX2-FP-NEXT: vpermq {{.*#+}} ymm13 = ymm13[3,3,3,3]1739; AVX2-FP-NEXT: vmovdqa %ymm1, %ymm01740; AVX2-FP-NEXT: vmovdqa %ymm2, %ymm101741; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} ymm15 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]1742; AVX2-FP-NEXT: vpshufd {{.*#+}} ymm15 = ymm15[1,2,3,3,5,6,7,7]1743; AVX2-FP-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,2,2,3]1744; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm13 = ymm15[0,1],ymm13[2],ymm15[3,4],ymm13[5],ymm15[6,7]1745; AVX2-FP-NEXT: vmovdqa %ymm8, %ymm11746; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm15 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]1747; AVX2-FP-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,1,2,3]1748; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm13 = ymm13[0],ymm15[1],ymm13[2,3],ymm15[4],ymm13[5,6],ymm15[7]1749; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload1750; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm15 = ymm8[u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,24,25,28,29,28,29,26,27,24,25,26,27,28,29,30,31]1751; AVX2-FP-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,1,2,3]1752; AVX2-FP-NEXT: vpblendvb %ymm14, %ymm13, %ymm15, %ymm21753; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill1754; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]1755; AVX2-FP-NEXT: vmovdqa %xmm6, %xmm131756; AVX2-FP-NEXT: vpermq {{.*#+}} ymm14 = ymm14[0,0,2,1]1757; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload1758; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]1759; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm15 = xmm15[1,0,2,2]1760; AVX2-FP-NEXT: vpermq {{.*#+}} ymm15 = ymm15[0,1,0,1]1761; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm14 = ymm14[0],ymm15[1],ymm14[2,3],ymm15[4],ymm14[5,6],ymm15[7]1762; AVX2-FP-NEXT: vpmovzxwd {{.*#+}} xmm15 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero1763; AVX2-FP-NEXT: vpbroadcastq %xmm15, %ymm151764; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm14 = ymm14[0,1],ymm15[2],ymm14[3,4],ymm15[5],ymm14[6,7]1765; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm15 = xmm3[0,0,2,1,4,5,6,7]1766; AVX2-FP-NEXT: vpbroadcastq %xmm15, %ymm151767; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm6 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]1768; AVX2-FP-NEXT: vpblendvb %ymm6, %ymm14, %ymm15, %ymm121769; AVX2-FP-NEXT: vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill1770; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm15 = ymm9[0],ymm11[0],ymm9[1],ymm11[1],ymm9[2],ymm11[2],ymm9[3],ymm11[3],ymm9[8],ymm11[8],ymm9[9],ymm11[9],ymm9[10],ymm11[10],ymm9[11],ymm11[11]1771; AVX2-FP-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,2,2,3]1772; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm12 = ymm0[0],ymm10[0],ymm0[1],ymm10[1],ymm0[2],ymm10[2],ymm0[3],ymm10[3],ymm0[8],ymm10[8],ymm0[9],ymm10[9],ymm0[10],ymm10[10],ymm0[11],ymm10[11]1773; AVX2-FP-NEXT: vmovdqa %ymm0, %ymm141774; AVX2-FP-NEXT: vpshufd {{.*#+}} ymm12 = ymm12[1,0,2,2,5,4,6,6]1775; AVX2-FP-NEXT: vpermq {{.*#+}} ymm12 = ymm12[2,1,2,3]1776; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm12 = ymm15[0],ymm12[1],ymm15[2,3],ymm12[4],ymm15[5,6],ymm12[7]1777; AVX2-FP-NEXT: vpshuflw {{.*#+}} ymm15 = ymm1[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]1778; AVX2-FP-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,2,2,2]1779; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm12 = ymm12[0,1],ymm15[2],ymm12[3,4],ymm15[5],ymm12[6,7]1780; AVX2-FP-NEXT: vpshuflw {{.*#+}} ymm15 = ymm8[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]1781; AVX2-FP-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,2,2,2]1782; AVX2-FP-NEXT: vpblendvb %ymm6, %ymm12, %ymm15, %ymm151783; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7,8,9,6,7,4,5,10,11]1784; AVX2-FP-NEXT: vpshufb %xmm0, %xmm7, %xmm61785; AVX2-FP-NEXT: vpshufb %xmm0, %xmm13, %xmm01786; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm6[4],xmm0[5],xmm6[5],xmm0[6],xmm6[6],xmm0[7],xmm6[7]1787; AVX2-FP-NEXT: vpsrldq {{.*#+}} xmm6 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero1788; AVX2-FP-NEXT: vpsrldq {{.*#+}} xmm7 = xmm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero1789; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]1790; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]1791; AVX2-FP-NEXT: vpbroadcastq %xmm6, %ymm61792; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm6[2],ymm0[3,4],ymm6[5],ymm0[6,7]1793; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm6 = xmm5[2,1,3,3,4,5,6,7]1794; AVX2-FP-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,0,2,1]1795; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0],ymm0[1,2],ymm6[3],ymm0[4,5],ymm6[6],ymm0[7]1796; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm6 = xmm3[0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]1797; AVX2-FP-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,0,2,1]1798; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm7 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]1799; AVX2-FP-NEXT: vpblendvb %ymm7, %ymm0, %ymm6, %ymm01800; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm6 = [u,u,u,u,4,5,10,11,u,u,u,u,u,u,u,u,24,25,22,23,20,21,26,27,u,u,u,u,u,u,u,u]1801; AVX2-FP-NEXT: vpshufb %ymm6, %ymm11, %ymm51802; AVX2-FP-NEXT: vpshufb %ymm6, %ymm9, %ymm31803; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm3 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[2],ymm5[2],ymm3[3],ymm5[3],ymm3[8],ymm5[8],ymm3[9],ymm5[9],ymm3[10],ymm5[10],ymm3[11],ymm5[11]1804; AVX2-FP-NEXT: vpsrldq {{.*#+}} ymm4 = ymm10[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm10[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero1805; AVX2-FP-NEXT: vpsrldq {{.*#+}} ymm2 = ymm14[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm14[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero1806; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[8],ymm4[8],ymm2[9],ymm4[9],ymm2[10],ymm4[10],ymm2[11],ymm4[11]1807; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,1,2,3]1808; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]1809; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1],ymm2[2],ymm3[3,4],ymm2[5],ymm3[6,7]1810; AVX2-FP-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]1811; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]1812; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1,2],ymm1[3],ymm2[4,5],ymm1[6],ymm2[7]1813; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm2 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25]1814; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]1815; AVX2-FP-NEXT: vpblendvb %ymm7, %ymm1, %ymm2, %ymm11816; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %rax1817; AVX2-FP-NEXT: vmovdqa %ymm1, 128(%rax)1818; AVX2-FP-NEXT: vmovdqa %ymm15, 96(%rax)1819; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload1820; AVX2-FP-NEXT: vmovaps %ymm1, 160(%rax)1821; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload1822; AVX2-FP-NEXT: vmovaps %ymm1, 64(%rax)1823; AVX2-FP-NEXT: vmovdqa %ymm0, 32(%rax)1824; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload1825; AVX2-FP-NEXT: vmovaps %ymm0, (%rax)1826; AVX2-FP-NEXT: addq $24, %rsp1827; AVX2-FP-NEXT: vzeroupper1828; AVX2-FP-NEXT: retq1829;1830; AVX2-FCP-LABEL: store_i16_stride6_vf16:1831; AVX2-FCP: # %bb.0:1832; AVX2-FCP-NEXT: vmovdqa (%rdi), %ymm131833; AVX2-FCP-NEXT: vmovdqa (%rsi), %ymm11834; AVX2-FCP-NEXT: vmovdqa (%rdx), %ymm31835; AVX2-FCP-NEXT: vmovdqa (%rcx), %ymm41836; AVX2-FCP-NEXT: vmovdqa (%r8), %ymm21837; AVX2-FCP-NEXT: vmovdqa (%rsi), %xmm61838; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm5 = [0,1,2,3,4,5,6,7,8,9,6,7,4,5,10,11]1839; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm6, %xmm81840; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm71841; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm7, %xmm51842; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm5[4],xmm8[4],xmm5[5],xmm8[5],xmm5[6],xmm8[6],xmm5[7],xmm8[7]1843; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm5 = ymm5[0,1,0,1]1844; AVX2-FCP-NEXT: vmovdqa (%rcx), %xmm81845; AVX2-FCP-NEXT: vpsrldq {{.*#+}} xmm9 = xmm8[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero1846; AVX2-FCP-NEXT: vmovdqa (%rdx), %xmm101847; AVX2-FCP-NEXT: vpsrldq {{.*#+}} xmm11 = xmm10[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero1848; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]1849; AVX2-FCP-NEXT: vpbroadcastq %xmm9, %ymm91850; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1],ymm9[2],ymm5[3,4],ymm9[5],ymm5[6,7]1851; AVX2-FCP-NEXT: vmovdqa (%r8), %xmm91852; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm11 = xmm9[2,1,3,3,4,5,6,7]1853; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm11 = ymm11[0,0,2,1]1854; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm11[0],ymm5[1,2],ymm11[3],ymm5[4,5],ymm11[6],ymm5[7]1855; AVX2-FCP-NEXT: vmovdqa (%r9), %xmm111856; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm12 = xmm11[0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]1857; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm12 = ymm12[0,0,2,1]1858; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm0 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]1859; AVX2-FCP-NEXT: vpblendvb %ymm0, %ymm5, %ymm12, %ymm51860; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill1861; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm12 = [u,u,u,u,4,5,10,11,u,u,u,u,u,u,u,u,24,25,22,23,20,21,26,27,u,u,u,u,u,u,u,u]1862; AVX2-FCP-NEXT: vpshufb %ymm12, %ymm1, %ymm141863; AVX2-FCP-NEXT: vpshufb %ymm12, %ymm13, %ymm121864; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm14 = ymm12[0],ymm14[0],ymm12[1],ymm14[1],ymm12[2],ymm14[2],ymm12[3],ymm14[3],ymm12[8],ymm14[8],ymm12[9],ymm14[9],ymm12[10],ymm14[10],ymm12[11],ymm14[11]1865; AVX2-FCP-NEXT: vpsrldq {{.*#+}} ymm12 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero1866; AVX2-FCP-NEXT: vpsrldq {{.*#+}} ymm15 = ymm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm3[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero1867; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm15 = ymm15[0],ymm12[0],ymm15[1],ymm12[1],ymm15[2],ymm12[2],ymm15[3],ymm12[3],ymm15[8],ymm12[8],ymm15[9],ymm12[9],ymm15[10],ymm12[10],ymm15[11],ymm12[11]1868; AVX2-FCP-NEXT: vmovdqa (%r9), %ymm121869; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm14 = ymm14[2,1,2,3]1870; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,2,2,2]1871; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm14 = ymm14[0,1],ymm15[2],ymm14[3,4],ymm15[5],ymm14[6,7]1872; AVX2-FCP-NEXT: vpshuflw {{.*#+}} ymm15 = ymm2[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]1873; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,2,2,3]1874; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm14 = ymm15[0],ymm14[1,2],ymm15[3],ymm14[4,5],ymm15[6],ymm14[7]1875; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm15 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25]1876; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,2,2,3]1877; AVX2-FCP-NEXT: vpblendvb %ymm0, %ymm14, %ymm15, %ymm01878; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill1879; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} xmm14 = xmm10[4],xmm8[4],xmm10[5],xmm8[5],xmm10[6],xmm8[6],xmm10[7],xmm8[7]1880; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm15 = [1,0,0,2,0,0,3,0]1881; AVX2-FCP-NEXT: vpermd %ymm14, %ymm15, %ymm141882; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} xmm15 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]1883; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm15 = ymm15[1,1,1,1]1884; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm14 = ymm14[0,1],ymm15[2],ymm14[3,4],ymm15[5],ymm14[6,7]1885; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm15 = xmm9[12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]1886; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm15 = ymm15[0,1,0,1]1887; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm14 = ymm14[0],ymm15[1],ymm14[2,3],ymm15[4],ymm14[5,6],ymm15[7]1888; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm15 = xmm11[8,9,12,13,12,13,10,11,8,9,10,11,12,13,14,15]1889; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm15 = ymm15[0,1,0,1]1890; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm0 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]1891; AVX2-FCP-NEXT: vpblendvb %ymm0, %ymm14, %ymm15, %ymm141892; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} ymm15 = ymm3[4],ymm4[4],ymm3[5],ymm4[5],ymm3[6],ymm4[6],ymm3[7],ymm4[7],ymm3[12],ymm4[12],ymm3[13],ymm4[13],ymm3[14],ymm4[14],ymm3[15],ymm4[15]1893; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm5 = [5,0,0,6,0,0,7,0]1894; AVX2-FCP-NEXT: vpermd %ymm15, %ymm5, %ymm51895; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} ymm15 = ymm13[4],ymm1[4],ymm13[5],ymm1[5],ymm13[6],ymm1[6],ymm13[7],ymm1[7],ymm13[12],ymm1[12],ymm13[13],ymm1[13],ymm13[14],ymm1[14],ymm13[15],ymm1[15]1896; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm15 = ymm15[3,3,3,3]1897; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1],ymm15[2],ymm5[3,4],ymm15[5],ymm5[6,7]1898; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm15 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]1899; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,1,2,3]1900; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0],ymm15[1],ymm5[2,3],ymm15[4],ymm5[5,6],ymm15[7]1901; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm15 = ymm12[u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,24,25,28,29,28,29,26,27,24,25,26,27,28,29,30,31]1902; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,1,2,3]1903; AVX2-FCP-NEXT: vpblendvb %ymm0, %ymm5, %ymm15, %ymm01904; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]1905; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm8 = [1,0,2,2,1,0,2,2]1906; AVX2-FCP-NEXT: # ymm8 = mem[0,1,0,1]1907; AVX2-FCP-NEXT: vpermd %ymm5, %ymm8, %ymm51908; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]1909; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,0,2,1]1910; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2,3],ymm5[4],ymm6[5,6],ymm5[7]1911; AVX2-FCP-NEXT: vpmovzxwd {{.*#+}} xmm6 = xmm9[0],zero,xmm9[1],zero,xmm9[2],zero,xmm9[3],zero1912; AVX2-FCP-NEXT: vpbroadcastq %xmm6, %ymm61913; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1],ymm6[2],ymm5[3,4],ymm6[5],ymm5[6,7]1914; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm6 = xmm11[0,0,2,1,4,5,6,7]1915; AVX2-FCP-NEXT: vpbroadcastq %xmm6, %ymm61916; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm7 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]1917; AVX2-FCP-NEXT: vpblendvb %ymm7, %ymm5, %ymm6, %ymm51918; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[8],ymm4[8],ymm3[9],ymm4[9],ymm3[10],ymm4[10],ymm3[11],ymm4[11]1919; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [0,4,0,0,5,0,0,6]1920; AVX2-FCP-NEXT: vpermd %ymm3, %ymm4, %ymm31921; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm13[0],ymm1[0],ymm13[1],ymm1[1],ymm13[2],ymm1[2],ymm13[3],ymm1[3],ymm13[8],ymm1[8],ymm13[9],ymm1[9],ymm13[10],ymm1[10],ymm13[11],ymm1[11]1922; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]1923; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5,6],ymm3[7]1924; AVX2-FCP-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]1925; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]1926; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]1927; AVX2-FCP-NEXT: vpshuflw {{.*#+}} ymm2 = ymm12[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]1928; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]1929; AVX2-FCP-NEXT: vpblendvb %ymm7, %ymm1, %ymm2, %ymm11930; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax1931; AVX2-FCP-NEXT: vmovdqa %ymm1, 96(%rax)1932; AVX2-FCP-NEXT: vmovdqa %ymm0, 160(%rax)1933; AVX2-FCP-NEXT: vmovdqa %ymm14, 64(%rax)1934; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload1935; AVX2-FCP-NEXT: vmovaps %ymm0, 128(%rax)1936; AVX2-FCP-NEXT: vmovdqa %ymm5, (%rax)1937; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload1938; AVX2-FCP-NEXT: vmovaps %ymm0, 32(%rax)1939; AVX2-FCP-NEXT: vzeroupper1940; AVX2-FCP-NEXT: retq1941;1942; AVX512-LABEL: store_i16_stride6_vf16:1943; AVX512: # %bb.0:1944; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax1945; AVX512-NEXT: vmovdqa (%rdi), %ymm01946; AVX512-NEXT: vmovdqa (%rsi), %ymm21947; AVX512-NEXT: vmovdqa (%rdx), %ymm41948; AVX512-NEXT: vmovdqa (%rcx), %ymm51949; AVX512-NEXT: vmovdqa (%r8), %ymm11950; AVX512-NEXT: vmovdqa (%r9), %ymm31951; AVX512-NEXT: vpsrldq {{.*#+}} ymm6 = ymm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm5[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero1952; AVX512-NEXT: vpsrldq {{.*#+}} ymm7 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero1953; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm6 = ymm7[0],ymm6[0],ymm7[1],ymm6[1],ymm7[2],ymm6[2],ymm7[3],ymm6[3],ymm7[8],ymm6[8],ymm7[9],ymm6[9],ymm7[10],ymm6[10],ymm7[11],ymm6[11]1954; AVX512-NEXT: vpshufd {{.*#+}} ymm7 = ymm2[2,1,2,3,6,5,6,7]1955; AVX512-NEXT: vpshuflw {{.*#+}} ymm7 = ymm7[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]1956; AVX512-NEXT: vpshufd {{.*#+}} ymm8 = ymm0[2,1,2,3,6,5,6,7]1957; AVX512-NEXT: vpshuflw {{.*#+}} ymm8 = ymm8[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]1958; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm7 = ymm8[0],ymm7[0],ymm8[1],ymm7[1],ymm8[2],ymm7[2],ymm8[3],ymm7[3],ymm8[8],ymm7[8],ymm8[9],ymm7[9],ymm8[10],ymm7[10],ymm8[11],ymm7[11]1959; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm8 = [0,5,12,0,4,13,0,7]1960; AVX512-NEXT: vpermi2d %ymm6, %ymm7, %ymm81961; AVX512-NEXT: vpshufd {{.*#+}} ymm6 = ymm3[1,2,2,3,5,6,6,7]1962; AVX512-NEXT: vpshufd {{.*#+}} ymm7 = ymm1[1,2,2,3,5,6,6,7]1963; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm6 = ymm7[0],ymm6[0],ymm7[1],ymm6[1],ymm7[2],ymm6[2],ymm7[3],ymm6[3],ymm7[8],ymm6[8],ymm7[9],ymm6[9],ymm7[10],ymm6[10],ymm7[11],ymm6[11]1964; AVX512-NEXT: vpermq {{.*#+}} ymm6 = ymm6[2,2,2,3]1965; AVX512-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0],ymm8[1,2],ymm6[3],ymm8[4,5],ymm6[6],ymm8[7]1966; AVX512-NEXT: vpunpckhwd {{.*#+}} ymm7 = ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15]1967; AVX512-NEXT: vpunpckhwd {{.*#+}} ymm8 = ymm4[4],ymm5[4],ymm4[5],ymm5[5],ymm4[6],ymm5[6],ymm4[7],ymm5[7],ymm4[12],ymm5[12],ymm4[13],ymm5[13],ymm4[14],ymm5[14],ymm4[15],ymm5[15]1968; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm9 = [5,0,14,6,0,15,7,0]1969; AVX512-NEXT: vpermi2d %ymm7, %ymm8, %ymm91970; AVX512-NEXT: vinserti64x4 $1, %ymm9, %zmm0, %zmm71971; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm8 = [8,21,10,11,22,13,14,23]1972; AVX512-NEXT: vpunpckhwd {{.*#+}} ymm9 = ymm1[4],ymm3[4],ymm1[5],ymm3[5],ymm1[6],ymm3[6],ymm1[7],ymm3[7],ymm1[12],ymm3[12],ymm1[13],ymm3[13],ymm1[14],ymm3[14],ymm1[15],ymm3[15]1973; AVX512-NEXT: vpermi2d %zmm9, %zmm7, %zmm81974; AVX512-NEXT: vinserti64x4 $1, %ymm8, %zmm6, %zmm161975; AVX512-NEXT: vmovdqa (%rcx), %xmm61976; AVX512-NEXT: vmovdqa (%rdx), %xmm71977; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]1978; AVX512-NEXT: vmovdqa (%rsi), %xmm81979; AVX512-NEXT: vmovdqa (%rdi), %xmm101980; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]1981; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm13 = [0,8,0,1,9,0,2,10]1982; AVX512-NEXT: vpermi2d %ymm9, %ymm11, %ymm131983; AVX512-NEXT: vmovdqa (%r9), %xmm91984; AVX512-NEXT: vmovdqa (%r8), %xmm111985; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]1986; AVX512-NEXT: vpbroadcastq %xmm14, %ymm141987; AVX512-NEXT: vpblendd {{.*#+}} ymm13 = ymm13[0,1],ymm14[2],ymm13[3,4],ymm14[5],ymm13[6,7]1988; AVX512-NEXT: vpsrldq {{.*#+}} xmm14 = xmm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero1989; AVX512-NEXT: vpsrldq {{.*#+}} xmm15 = xmm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero1990; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]1991; AVX512-NEXT: vpshufd {{.*#+}} xmm15 = xmm8[0,1,2,1]1992; AVX512-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,7,6,5]1993; AVX512-NEXT: vpshufd {{.*#+}} xmm12 = xmm10[0,1,2,1]1994; AVX512-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,7,6,5]1995; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm12[4],xmm15[4],xmm12[5],xmm15[5],xmm12[6],xmm15[6],xmm12[7],xmm15[7]1996; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm15 = [0,1,8,0,0,9,0,3]1997; AVX512-NEXT: vpermi2d %ymm14, %ymm12, %ymm151998; AVX512-NEXT: vpshufd {{.*#+}} xmm12 = xmm9[1,2,2,3]1999; AVX512-NEXT: vpshufd {{.*#+}} xmm14 = xmm11[1,2,2,3]2000; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm14[0],xmm12[0],xmm14[1],xmm12[1],xmm14[2],xmm12[2],xmm14[3],xmm12[3]2001; AVX512-NEXT: vpermq {{.*#+}} ymm12 = ymm12[0,0,2,1]2002; AVX512-NEXT: vpblendd {{.*#+}} ymm12 = ymm12[0],ymm15[1,2],ymm12[3],ymm15[4,5],ymm12[6],ymm15[7]2003; AVX512-NEXT: vinserti64x4 $1, %ymm12, %zmm13, %zmm122004; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[8],ymm5[8],ymm4[9],ymm5[9],ymm4[10],ymm5[10],ymm4[11],ymm5[11]2005; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11]2006; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm2 = [4,12,0,5,13,0,6,14]2007; AVX512-NEXT: vpermi2d %ymm4, %ymm0, %ymm22008; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[8],ymm3[8],ymm1[9],ymm3[9],ymm1[10],ymm3[10],ymm1[11],ymm3[11]2009; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]2010; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm0[2],ymm2[3,4],ymm0[5],ymm2[6,7]2011; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm10[4],xmm8[4],xmm10[5],xmm8[5],xmm10[6],xmm8[6],xmm10[7],xmm8[7]2012; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]2013; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm3 = [1,0,10,2,0,11,3,0]2014; AVX512-NEXT: vpermi2d %ymm1, %ymm2, %ymm32015; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm11[4],xmm9[4],xmm11[5],xmm9[5],xmm11[6],xmm9[6],xmm11[7],xmm9[7]2016; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,9,2,3,10,5,6,11]2017; AVX512-NEXT: vpermi2d %ymm1, %ymm3, %ymm22018; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm02019; AVX512-NEXT: vmovdqa64 %zmm0, 64(%rax)2020; AVX512-NEXT: vmovdqa64 %zmm12, (%rax)2021; AVX512-NEXT: vmovdqa64 %zmm16, 128(%rax)2022; AVX512-NEXT: vzeroupper2023; AVX512-NEXT: retq2024;2025; AVX512-FCP-LABEL: store_i16_stride6_vf16:2026; AVX512-FCP: # %bb.0:2027; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax2028; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm22029; AVX512-FCP-NEXT: vmovdqa (%rsi), %ymm32030; AVX512-FCP-NEXT: vmovdqa (%rdx), %ymm42031; AVX512-FCP-NEXT: vmovdqa (%rcx), %ymm52032; AVX512-FCP-NEXT: vmovdqa64 (%r8), %ymm162033; AVX512-FCP-NEXT: vmovdqa (%r9), %ymm12034; AVX512-FCP-NEXT: vmovdqa (%rcx), %xmm62035; AVX512-FCP-NEXT: vmovdqa (%rdx), %xmm72036; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]2037; AVX512-FCP-NEXT: vmovdqa (%rsi), %xmm82038; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm102039; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]2040; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm12 = [0,8,0,1,9,0,2,10]2041; AVX512-FCP-NEXT: vpermi2d %ymm9, %ymm11, %ymm122042; AVX512-FCP-NEXT: vmovdqa (%r9), %xmm92043; AVX512-FCP-NEXT: vmovdqa (%r8), %xmm112044; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]2045; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm14 = [0,1,8,3,4,9,6,7]2046; AVX512-FCP-NEXT: vpermi2d %ymm13, %ymm12, %ymm142047; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm12 = [0,1,2,3,4,5,6,7,8,9,6,7,4,5,10,11]2048; AVX512-FCP-NEXT: vpshufb %xmm12, %xmm8, %xmm132049; AVX512-FCP-NEXT: vpshufb %xmm12, %xmm10, %xmm122050; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm12[4],xmm13[4],xmm12[5],xmm13[5],xmm12[6],xmm13[6],xmm12[7],xmm13[7]2051; AVX512-FCP-NEXT: vpsrldq {{.*#+}} xmm13 = xmm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero2052; AVX512-FCP-NEXT: vpsrldq {{.*#+}} xmm15 = xmm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero2053; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm15[0],xmm13[0],xmm15[1],xmm13[1],xmm15[2],xmm13[2],xmm15[3],xmm13[3]2054; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm15 = [0,1,8,0,0,9,0,3]2055; AVX512-FCP-NEXT: vpermi2d %ymm13, %ymm12, %ymm152056; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm15, %zmm0, %zmm122057; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm13 = [16,9,10,17,12,13,18,15]2058; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm15 = xmm9[1,2,2,3]2059; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm0 = xmm11[1,2,2,3]2060; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm15[0],xmm0[1],xmm15[1],xmm0[2],xmm15[2],xmm0[3],xmm15[3]2061; AVX512-FCP-NEXT: vpermi2d %zmm0, %zmm12, %zmm132062; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm13, %zmm14, %zmm02063; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm8 = xmm10[4],xmm8[4],xmm10[5],xmm8[5],xmm10[6],xmm8[6],xmm10[7],xmm8[7]2064; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm6 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]2065; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm7 = [1,0,10,2,0,11,3,0]2066; AVX512-FCP-NEXT: vpermi2d %ymm8, %ymm6, %ymm72067; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm6 = xmm11[4],xmm9[4],xmm11[5],xmm9[5],xmm11[6],xmm9[6],xmm11[7],xmm9[7]2068; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm8 = [0,9,2,3,10,5,6,11]2069; AVX512-FCP-NEXT: vpermi2d %ymm6, %ymm7, %ymm82070; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm6 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[8],ymm5[8],ymm4[9],ymm5[9],ymm4[10],ymm5[10],ymm4[11],ymm5[11]2071; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm7 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11]2072; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm9 = [4,12,0,5,13,0,6,14]2073; AVX512-FCP-NEXT: vpermi2d %ymm6, %ymm7, %ymm92074; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm9, %zmm0, %zmm62075; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm7 = [8,9,20,11,12,21,14,15]2076; AVX512-FCP-NEXT: vmovdqa64 %ymm16, %ymm102077; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm9 = ymm10[0],ymm1[0],ymm10[1],ymm1[1],ymm10[2],ymm1[2],ymm10[3],ymm1[3],ymm10[8],ymm1[8],ymm10[9],ymm1[9],ymm10[10],ymm1[10],ymm10[11],ymm1[11]2078; AVX512-FCP-NEXT: vpermi2d %zmm9, %zmm6, %zmm72079; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm8, %zmm62080; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm7 = [u,u,u,u,4,5,10,11,u,u,u,u,u,u,u,u,24,25,22,23,20,21,26,27,u,u,u,u,u,u,u,u]2081; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm3, %ymm82082; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm2, %ymm72083; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm7 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11]2084; AVX512-FCP-NEXT: vpsrldq {{.*#+}} ymm8 = ymm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm5[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero2085; AVX512-FCP-NEXT: vpsrldq {{.*#+}} ymm9 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero2086; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm8 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[8],ymm8[8],ymm9[9],ymm8[9],ymm9[10],ymm8[10],ymm9[11],ymm8[11]2087; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm9 = [0,5,12,0,4,13,0,7]2088; AVX512-FCP-NEXT: vpermi2d %ymm8, %ymm7, %ymm92089; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm7 = ymm1[1,2,2,3,5,6,6,7]2090; AVX512-FCP-NEXT: vpshufd {{.*#+}} ymm8 = ymm16[1,2,2,3,5,6,6,7]2091; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm7 = ymm8[0],ymm7[0],ymm8[1],ymm7[1],ymm8[2],ymm7[2],ymm8[3],ymm7[3],ymm8[8],ymm7[8],ymm8[9],ymm7[9],ymm8[10],ymm7[10],ymm8[11],ymm7[11]2092; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm8 = [12,1,2,13,4,5,14,7]2093; AVX512-FCP-NEXT: vpermi2d %ymm7, %ymm9, %ymm82094; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15]2095; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm5[4],ymm4[5],ymm5[5],ymm4[6],ymm5[6],ymm4[7],ymm5[7],ymm4[12],ymm5[12],ymm4[13],ymm5[13],ymm4[14],ymm5[14],ymm4[15],ymm5[15]2096; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [5,0,14,6,0,15,7,0]2097; AVX512-FCP-NEXT: vpermi2d %ymm2, %ymm3, %ymm42098; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm22099; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [8,21,10,11,22,13,14,23]2100; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm10[4],ymm1[4],ymm10[5],ymm1[5],ymm10[6],ymm1[6],ymm10[7],ymm1[7],ymm10[12],ymm1[12],ymm10[13],ymm1[13],ymm10[14],ymm1[14],ymm10[15],ymm1[15]2101; AVX512-FCP-NEXT: vpermi2d %zmm1, %zmm2, %zmm32102; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm8, %zmm12103; AVX512-FCP-NEXT: vmovdqa64 %zmm1, 128(%rax)2104; AVX512-FCP-NEXT: vmovdqa64 %zmm6, 64(%rax)2105; AVX512-FCP-NEXT: vmovdqa64 %zmm0, (%rax)2106; AVX512-FCP-NEXT: vzeroupper2107; AVX512-FCP-NEXT: retq2108;2109; AVX512DQ-LABEL: store_i16_stride6_vf16:2110; AVX512DQ: # %bb.0:2111; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax2112; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm02113; AVX512DQ-NEXT: vmovdqa (%rsi), %ymm22114; AVX512DQ-NEXT: vmovdqa (%rdx), %ymm42115; AVX512DQ-NEXT: vmovdqa (%rcx), %ymm52116; AVX512DQ-NEXT: vmovdqa (%r8), %ymm12117; AVX512DQ-NEXT: vmovdqa (%r9), %ymm32118; AVX512DQ-NEXT: vpsrldq {{.*#+}} ymm6 = ymm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm5[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero2119; AVX512DQ-NEXT: vpsrldq {{.*#+}} ymm7 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero2120; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm6 = ymm7[0],ymm6[0],ymm7[1],ymm6[1],ymm7[2],ymm6[2],ymm7[3],ymm6[3],ymm7[8],ymm6[8],ymm7[9],ymm6[9],ymm7[10],ymm6[10],ymm7[11],ymm6[11]2121; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm7 = ymm2[2,1,2,3,6,5,6,7]2122; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm7 = ymm7[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]2123; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm8 = ymm0[2,1,2,3,6,5,6,7]2124; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm8 = ymm8[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]2125; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm7 = ymm8[0],ymm7[0],ymm8[1],ymm7[1],ymm8[2],ymm7[2],ymm8[3],ymm7[3],ymm8[8],ymm7[8],ymm8[9],ymm7[9],ymm8[10],ymm7[10],ymm8[11],ymm7[11]2126; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm8 = [0,5,12,0,4,13,0,7]2127; AVX512DQ-NEXT: vpermi2d %ymm6, %ymm7, %ymm82128; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm6 = ymm3[1,2,2,3,5,6,6,7]2129; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm7 = ymm1[1,2,2,3,5,6,6,7]2130; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm6 = ymm7[0],ymm6[0],ymm7[1],ymm6[1],ymm7[2],ymm6[2],ymm7[3],ymm6[3],ymm7[8],ymm6[8],ymm7[9],ymm6[9],ymm7[10],ymm6[10],ymm7[11],ymm6[11]2131; AVX512DQ-NEXT: vpermq {{.*#+}} ymm6 = ymm6[2,2,2,3]2132; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0],ymm8[1,2],ymm6[3],ymm8[4,5],ymm6[6],ymm8[7]2133; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm7 = ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15]2134; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm8 = ymm4[4],ymm5[4],ymm4[5],ymm5[5],ymm4[6],ymm5[6],ymm4[7],ymm5[7],ymm4[12],ymm5[12],ymm4[13],ymm5[13],ymm4[14],ymm5[14],ymm4[15],ymm5[15]2135; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm9 = [5,0,14,6,0,15,7,0]2136; AVX512DQ-NEXT: vpermi2d %ymm7, %ymm8, %ymm92137; AVX512DQ-NEXT: vinserti64x4 $1, %ymm9, %zmm0, %zmm72138; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm8 = [8,21,10,11,22,13,14,23]2139; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm9 = ymm1[4],ymm3[4],ymm1[5],ymm3[5],ymm1[6],ymm3[6],ymm1[7],ymm3[7],ymm1[12],ymm3[12],ymm1[13],ymm3[13],ymm1[14],ymm3[14],ymm1[15],ymm3[15]2140; AVX512DQ-NEXT: vpermi2d %zmm9, %zmm7, %zmm82141; AVX512DQ-NEXT: vinserti64x4 $1, %ymm8, %zmm6, %zmm162142; AVX512DQ-NEXT: vmovdqa (%rcx), %xmm62143; AVX512DQ-NEXT: vmovdqa (%rdx), %xmm72144; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]2145; AVX512DQ-NEXT: vmovdqa (%rsi), %xmm82146; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm102147; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]2148; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm13 = [0,8,0,1,9,0,2,10]2149; AVX512DQ-NEXT: vpermi2d %ymm9, %ymm11, %ymm132150; AVX512DQ-NEXT: vmovdqa (%r9), %xmm92151; AVX512DQ-NEXT: vmovdqa (%r8), %xmm112152; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]2153; AVX512DQ-NEXT: vpbroadcastq %xmm14, %ymm142154; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm13 = ymm13[0,1],ymm14[2],ymm13[3,4],ymm14[5],ymm13[6,7]2155; AVX512DQ-NEXT: vpsrldq {{.*#+}} xmm14 = xmm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero2156; AVX512DQ-NEXT: vpsrldq {{.*#+}} xmm15 = xmm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero2157; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]2158; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm15 = xmm8[0,1,2,1]2159; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,7,6,5]2160; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm12 = xmm10[0,1,2,1]2161; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,7,6,5]2162; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm12[4],xmm15[4],xmm12[5],xmm15[5],xmm12[6],xmm15[6],xmm12[7],xmm15[7]2163; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm15 = [0,1,8,0,0,9,0,3]2164; AVX512DQ-NEXT: vpermi2d %ymm14, %ymm12, %ymm152165; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm12 = xmm9[1,2,2,3]2166; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm14 = xmm11[1,2,2,3]2167; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm14[0],xmm12[0],xmm14[1],xmm12[1],xmm14[2],xmm12[2],xmm14[3],xmm12[3]2168; AVX512DQ-NEXT: vpermq {{.*#+}} ymm12 = ymm12[0,0,2,1]2169; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm12 = ymm12[0],ymm15[1,2],ymm12[3],ymm15[4,5],ymm12[6],ymm15[7]2170; AVX512DQ-NEXT: vinserti64x4 $1, %ymm12, %zmm13, %zmm122171; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[8],ymm5[8],ymm4[9],ymm5[9],ymm4[10],ymm5[10],ymm4[11],ymm5[11]2172; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11]2173; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm2 = [4,12,0,5,13,0,6,14]2174; AVX512DQ-NEXT: vpermi2d %ymm4, %ymm0, %ymm22175; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[8],ymm3[8],ymm1[9],ymm3[9],ymm1[10],ymm3[10],ymm1[11],ymm3[11]2176; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]2177; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm0[2],ymm2[3,4],ymm0[5],ymm2[6,7]2178; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm10[4],xmm8[4],xmm10[5],xmm8[5],xmm10[6],xmm8[6],xmm10[7],xmm8[7]2179; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]2180; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm3 = [1,0,10,2,0,11,3,0]2181; AVX512DQ-NEXT: vpermi2d %ymm1, %ymm2, %ymm32182; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm11[4],xmm9[4],xmm11[5],xmm9[5],xmm11[6],xmm9[6],xmm11[7],xmm9[7]2183; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,9,2,3,10,5,6,11]2184; AVX512DQ-NEXT: vpermi2d %ymm1, %ymm3, %ymm22185; AVX512DQ-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm02186; AVX512DQ-NEXT: vmovdqa64 %zmm0, 64(%rax)2187; AVX512DQ-NEXT: vmovdqa64 %zmm12, (%rax)2188; AVX512DQ-NEXT: vmovdqa64 %zmm16, 128(%rax)2189; AVX512DQ-NEXT: vzeroupper2190; AVX512DQ-NEXT: retq2191;2192; AVX512DQ-FCP-LABEL: store_i16_stride6_vf16:2193; AVX512DQ-FCP: # %bb.0:2194; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax2195; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm22196; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %ymm32197; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %ymm42198; AVX512DQ-FCP-NEXT: vmovdqa (%rcx), %ymm52199; AVX512DQ-FCP-NEXT: vmovdqa64 (%r8), %ymm162200; AVX512DQ-FCP-NEXT: vmovdqa (%r9), %ymm12201; AVX512DQ-FCP-NEXT: vmovdqa (%rcx), %xmm62202; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %xmm72203; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]2204; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %xmm82205; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm102206; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]2207; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm12 = [0,8,0,1,9,0,2,10]2208; AVX512DQ-FCP-NEXT: vpermi2d %ymm9, %ymm11, %ymm122209; AVX512DQ-FCP-NEXT: vmovdqa (%r9), %xmm92210; AVX512DQ-FCP-NEXT: vmovdqa (%r8), %xmm112211; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]2212; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm14 = [0,1,8,3,4,9,6,7]2213; AVX512DQ-FCP-NEXT: vpermi2d %ymm13, %ymm12, %ymm142214; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm12 = [0,1,2,3,4,5,6,7,8,9,6,7,4,5,10,11]2215; AVX512DQ-FCP-NEXT: vpshufb %xmm12, %xmm8, %xmm132216; AVX512DQ-FCP-NEXT: vpshufb %xmm12, %xmm10, %xmm122217; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm12[4],xmm13[4],xmm12[5],xmm13[5],xmm12[6],xmm13[6],xmm12[7],xmm13[7]2218; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} xmm13 = xmm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero2219; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} xmm15 = xmm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero2220; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm13 = xmm15[0],xmm13[0],xmm15[1],xmm13[1],xmm15[2],xmm13[2],xmm15[3],xmm13[3]2221; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm15 = [0,1,8,0,0,9,0,3]2222; AVX512DQ-FCP-NEXT: vpermi2d %ymm13, %ymm12, %ymm152223; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm15, %zmm0, %zmm122224; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm13 = [16,9,10,17,12,13,18,15]2225; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm15 = xmm9[1,2,2,3]2226; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm0 = xmm11[1,2,2,3]2227; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm15[0],xmm0[1],xmm15[1],xmm0[2],xmm15[2],xmm0[3],xmm15[3]2228; AVX512DQ-FCP-NEXT: vpermi2d %zmm0, %zmm12, %zmm132229; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm13, %zmm14, %zmm02230; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm8 = xmm10[4],xmm8[4],xmm10[5],xmm8[5],xmm10[6],xmm8[6],xmm10[7],xmm8[7]2231; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm6 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]2232; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm7 = [1,0,10,2,0,11,3,0]2233; AVX512DQ-FCP-NEXT: vpermi2d %ymm8, %ymm6, %ymm72234; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm6 = xmm11[4],xmm9[4],xmm11[5],xmm9[5],xmm11[6],xmm9[6],xmm11[7],xmm9[7]2235; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm8 = [0,9,2,3,10,5,6,11]2236; AVX512DQ-FCP-NEXT: vpermi2d %ymm6, %ymm7, %ymm82237; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm6 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[8],ymm5[8],ymm4[9],ymm5[9],ymm4[10],ymm5[10],ymm4[11],ymm5[11]2238; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm7 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11]2239; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm9 = [4,12,0,5,13,0,6,14]2240; AVX512DQ-FCP-NEXT: vpermi2d %ymm6, %ymm7, %ymm92241; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm9, %zmm0, %zmm62242; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm7 = [8,9,20,11,12,21,14,15]2243; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm16, %ymm102244; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm9 = ymm10[0],ymm1[0],ymm10[1],ymm1[1],ymm10[2],ymm1[2],ymm10[3],ymm1[3],ymm10[8],ymm1[8],ymm10[9],ymm1[9],ymm10[10],ymm1[10],ymm10[11],ymm1[11]2245; AVX512DQ-FCP-NEXT: vpermi2d %zmm9, %zmm6, %zmm72246; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm8, %zmm62247; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm7 = [u,u,u,u,4,5,10,11,u,u,u,u,u,u,u,u,24,25,22,23,20,21,26,27,u,u,u,u,u,u,u,u]2248; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm3, %ymm82249; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm2, %ymm72250; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm7 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11]2251; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} ymm8 = ymm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm5[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero2252; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} ymm9 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero2253; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm8 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[8],ymm8[8],ymm9[9],ymm8[9],ymm9[10],ymm8[10],ymm9[11],ymm8[11]2254; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm9 = [0,5,12,0,4,13,0,7]2255; AVX512DQ-FCP-NEXT: vpermi2d %ymm8, %ymm7, %ymm92256; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm7 = ymm1[1,2,2,3,5,6,6,7]2257; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} ymm8 = ymm16[1,2,2,3,5,6,6,7]2258; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm7 = ymm8[0],ymm7[0],ymm8[1],ymm7[1],ymm8[2],ymm7[2],ymm8[3],ymm7[3],ymm8[8],ymm7[8],ymm8[9],ymm7[9],ymm8[10],ymm7[10],ymm8[11],ymm7[11]2259; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm8 = [12,1,2,13,4,5,14,7]2260; AVX512DQ-FCP-NEXT: vpermi2d %ymm7, %ymm9, %ymm82261; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15]2262; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm5[4],ymm4[5],ymm5[5],ymm4[6],ymm5[6],ymm4[7],ymm5[7],ymm4[12],ymm5[12],ymm4[13],ymm5[13],ymm4[14],ymm5[14],ymm4[15],ymm5[15]2263; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [5,0,14,6,0,15,7,0]2264; AVX512DQ-FCP-NEXT: vpermi2d %ymm2, %ymm3, %ymm42265; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm22266; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [8,21,10,11,22,13,14,23]2267; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm10[4],ymm1[4],ymm10[5],ymm1[5],ymm10[6],ymm1[6],ymm10[7],ymm1[7],ymm10[12],ymm1[12],ymm10[13],ymm1[13],ymm10[14],ymm1[14],ymm10[15],ymm1[15]2268; AVX512DQ-FCP-NEXT: vpermi2d %zmm1, %zmm2, %zmm32269; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm8, %zmm12270; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, 128(%rax)2271; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm6, 64(%rax)2272; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, (%rax)2273; AVX512DQ-FCP-NEXT: vzeroupper2274; AVX512DQ-FCP-NEXT: retq2275;2276; AVX512BW-LABEL: store_i16_stride6_vf16:2277; AVX512BW: # %bb.0:2278; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax2279; AVX512BW-NEXT: vmovdqa (%rdi), %ymm02280; AVX512BW-NEXT: vmovdqa (%rdx), %ymm12281; AVX512BW-NEXT: vmovdqa (%r8), %ymm22282; AVX512BW-NEXT: vinserti64x4 $1, (%rsi), %zmm0, %zmm02283; AVX512BW-NEXT: vinserti64x4 $1, (%rcx), %zmm1, %zmm12284; AVX512BW-NEXT: vinserti64x4 $1, (%r9), %zmm2, %zmm22285; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,16,32,48,0,0,1,17,33,49,0,0,2,18,34,50,0,0,3,19,35,51,0,0,4,20,36,52,0,0,5,21]2286; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm32287; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm4 = [0,1,2,3,32,48,6,7,8,9,33,49,12,13,14,15,34,50,18,19,20,21,35,51,24,25,26,27,36,52,30,31]2288; AVX512BW-NEXT: vpermi2w %zmm2, %zmm3, %zmm42289; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [5,21,0,0,38,54,6,22,0,0,39,55,7,23,0,0,40,56,8,24,0,0,41,57,9,25,0,0,42,58,10,26]2290; AVX512BW-NEXT: vpermi2w %zmm0, %zmm1, %zmm32291; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm5 = [0,1,37,53,4,5,6,7,38,54,10,11,12,13,39,55,16,17,18,19,40,56,22,23,24,25,41,57,28,29,30,31]2292; AVX512BW-NEXT: vpermi2w %zmm2, %zmm3, %zmm52293; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,0,11,27,43,59,0,0,12,28,44,60,0,0,13,29,45,61,0,0,14,30,46,62,0,0,15,31,47,63,0,0]2294; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm32295; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm0 = [42,58,2,3,4,5,43,59,8,9,10,11,44,60,14,15,16,17,45,61,20,21,22,23,46,62,26,27,28,29,47,63]2296; AVX512BW-NEXT: vpermi2w %zmm2, %zmm3, %zmm02297; AVX512BW-NEXT: vmovdqa64 %zmm0, 128(%rax)2298; AVX512BW-NEXT: vmovdqa64 %zmm5, 64(%rax)2299; AVX512BW-NEXT: vmovdqa64 %zmm4, (%rax)2300; AVX512BW-NEXT: vzeroupper2301; AVX512BW-NEXT: retq2302;2303; AVX512BW-FCP-LABEL: store_i16_stride6_vf16:2304; AVX512BW-FCP: # %bb.0:2305; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax2306; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %ymm02307; AVX512BW-FCP-NEXT: vmovdqa (%rdx), %ymm12308; AVX512BW-FCP-NEXT: vmovdqa (%r8), %ymm22309; AVX512BW-FCP-NEXT: vinserti64x4 $1, (%rsi), %zmm0, %zmm02310; AVX512BW-FCP-NEXT: vinserti64x4 $1, (%rcx), %zmm1, %zmm12311; AVX512BW-FCP-NEXT: vinserti64x4 $1, (%r9), %zmm2, %zmm22312; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,16,32,48,0,0,1,17,33,49,0,0,2,18,34,50,0,0,3,19,35,51,0,0,4,20,36,52,0,0,5,21]2313; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm32314; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm4 = [0,1,2,3,32,48,6,7,8,9,33,49,12,13,14,15,34,50,18,19,20,21,35,51,24,25,26,27,36,52,30,31]2315; AVX512BW-FCP-NEXT: vpermi2w %zmm2, %zmm3, %zmm42316; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [5,21,0,0,38,54,6,22,0,0,39,55,7,23,0,0,40,56,8,24,0,0,41,57,9,25,0,0,42,58,10,26]2317; AVX512BW-FCP-NEXT: vpermi2w %zmm0, %zmm1, %zmm32318; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm5 = [0,1,37,53,4,5,6,7,38,54,10,11,12,13,39,55,16,17,18,19,40,56,22,23,24,25,41,57,28,29,30,31]2319; AVX512BW-FCP-NEXT: vpermi2w %zmm2, %zmm3, %zmm52320; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,0,11,27,43,59,0,0,12,28,44,60,0,0,13,29,45,61,0,0,14,30,46,62,0,0,15,31,47,63,0,0]2321; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm32322; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm0 = [42,58,2,3,4,5,43,59,8,9,10,11,44,60,14,15,16,17,45,61,20,21,22,23,46,62,26,27,28,29,47,63]2323; AVX512BW-FCP-NEXT: vpermi2w %zmm2, %zmm3, %zmm02324; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, 128(%rax)2325; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, 64(%rax)2326; AVX512BW-FCP-NEXT: vmovdqa64 %zmm4, (%rax)2327; AVX512BW-FCP-NEXT: vzeroupper2328; AVX512BW-FCP-NEXT: retq2329;2330; AVX512DQ-BW-LABEL: store_i16_stride6_vf16:2331; AVX512DQ-BW: # %bb.0:2332; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %rax2333; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %ymm02334; AVX512DQ-BW-NEXT: vmovdqa (%rdx), %ymm12335; AVX512DQ-BW-NEXT: vmovdqa (%r8), %ymm22336; AVX512DQ-BW-NEXT: vinserti64x4 $1, (%rsi), %zmm0, %zmm02337; AVX512DQ-BW-NEXT: vinserti64x4 $1, (%rcx), %zmm1, %zmm12338; AVX512DQ-BW-NEXT: vinserti64x4 $1, (%r9), %zmm2, %zmm22339; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,16,32,48,0,0,1,17,33,49,0,0,2,18,34,50,0,0,3,19,35,51,0,0,4,20,36,52,0,0,5,21]2340; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm32341; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm4 = [0,1,2,3,32,48,6,7,8,9,33,49,12,13,14,15,34,50,18,19,20,21,35,51,24,25,26,27,36,52,30,31]2342; AVX512DQ-BW-NEXT: vpermi2w %zmm2, %zmm3, %zmm42343; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [5,21,0,0,38,54,6,22,0,0,39,55,7,23,0,0,40,56,8,24,0,0,41,57,9,25,0,0,42,58,10,26]2344; AVX512DQ-BW-NEXT: vpermi2w %zmm0, %zmm1, %zmm32345; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm5 = [0,1,37,53,4,5,6,7,38,54,10,11,12,13,39,55,16,17,18,19,40,56,22,23,24,25,41,57,28,29,30,31]2346; AVX512DQ-BW-NEXT: vpermi2w %zmm2, %zmm3, %zmm52347; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,0,11,27,43,59,0,0,12,28,44,60,0,0,13,29,45,61,0,0,14,30,46,62,0,0,15,31,47,63,0,0]2348; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm32349; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm0 = [42,58,2,3,4,5,43,59,8,9,10,11,44,60,14,15,16,17,45,61,20,21,22,23,46,62,26,27,28,29,47,63]2350; AVX512DQ-BW-NEXT: vpermi2w %zmm2, %zmm3, %zmm02351; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, 128(%rax)2352; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, 64(%rax)2353; AVX512DQ-BW-NEXT: vmovdqa64 %zmm4, (%rax)2354; AVX512DQ-BW-NEXT: vzeroupper2355; AVX512DQ-BW-NEXT: retq2356;2357; AVX512DQ-BW-FCP-LABEL: store_i16_stride6_vf16:2358; AVX512DQ-BW-FCP: # %bb.0:2359; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax2360; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %ymm02361; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdx), %ymm12362; AVX512DQ-BW-FCP-NEXT: vmovdqa (%r8), %ymm22363; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, (%rsi), %zmm0, %zmm02364; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, (%rcx), %zmm1, %zmm12365; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, (%r9), %zmm2, %zmm22366; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,16,32,48,0,0,1,17,33,49,0,0,2,18,34,50,0,0,3,19,35,51,0,0,4,20,36,52,0,0,5,21]2367; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm32368; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm4 = [0,1,2,3,32,48,6,7,8,9,33,49,12,13,14,15,34,50,18,19,20,21,35,51,24,25,26,27,36,52,30,31]2369; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm2, %zmm3, %zmm42370; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [5,21,0,0,38,54,6,22,0,0,39,55,7,23,0,0,40,56,8,24,0,0,41,57,9,25,0,0,42,58,10,26]2371; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm0, %zmm1, %zmm32372; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm5 = [0,1,37,53,4,5,6,7,38,54,10,11,12,13,39,55,16,17,18,19,40,56,22,23,24,25,41,57,28,29,30,31]2373; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm2, %zmm3, %zmm52374; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,0,11,27,43,59,0,0,12,28,44,60,0,0,13,29,45,61,0,0,14,30,46,62,0,0,15,31,47,63,0,0]2375; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm32376; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm0 = [42,58,2,3,4,5,43,59,8,9,10,11,44,60,14,15,16,17,45,61,20,21,22,23,46,62,26,27,28,29,47,63]2377; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm2, %zmm3, %zmm02378; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, 128(%rax)2379; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, 64(%rax)2380; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm4, (%rax)2381; AVX512DQ-BW-FCP-NEXT: vzeroupper2382; AVX512DQ-BW-FCP-NEXT: retq2383 %in.vec0 = load <16 x i16>, ptr %in.vecptr0, align 642384 %in.vec1 = load <16 x i16>, ptr %in.vecptr1, align 642385 %in.vec2 = load <16 x i16>, ptr %in.vecptr2, align 642386 %in.vec3 = load <16 x i16>, ptr %in.vecptr3, align 642387 %in.vec4 = load <16 x i16>, ptr %in.vecptr4, align 642388 %in.vec5 = load <16 x i16>, ptr %in.vecptr5, align 642389 %1 = shufflevector <16 x i16> %in.vec0, <16 x i16> %in.vec1, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>2390 %2 = shufflevector <16 x i16> %in.vec2, <16 x i16> %in.vec3, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>2391 %3 = shufflevector <16 x i16> %in.vec4, <16 x i16> %in.vec5, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>2392 %4 = shufflevector <32 x i16> %1, <32 x i16> %2, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>2393 %5 = shufflevector <32 x i16> %3, <32 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>2394 %6 = shufflevector <64 x i16> %4, <64 x i16> %5, <96 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95>2395 %interleaved.vec = shufflevector <96 x i16> %6, <96 x i16> poison, <96 x i32> <i32 0, i32 16, i32 32, i32 48, i32 64, i32 80, i32 1, i32 17, i32 33, i32 49, i32 65, i32 81, i32 2, i32 18, i32 34, i32 50, i32 66, i32 82, i32 3, i32 19, i32 35, i32 51, i32 67, i32 83, i32 4, i32 20, i32 36, i32 52, i32 68, i32 84, i32 5, i32 21, i32 37, i32 53, i32 69, i32 85, i32 6, i32 22, i32 38, i32 54, i32 70, i32 86, i32 7, i32 23, i32 39, i32 55, i32 71, i32 87, i32 8, i32 24, i32 40, i32 56, i32 72, i32 88, i32 9, i32 25, i32 41, i32 57, i32 73, i32 89, i32 10, i32 26, i32 42, i32 58, i32 74, i32 90, i32 11, i32 27, i32 43, i32 59, i32 75, i32 91, i32 12, i32 28, i32 44, i32 60, i32 76, i32 92, i32 13, i32 29, i32 45, i32 61, i32 77, i32 93, i32 14, i32 30, i32 46, i32 62, i32 78, i32 94, i32 15, i32 31, i32 47, i32 63, i32 79, i32 95>2396 store <96 x i16> %interleaved.vec, ptr %out.vec, align 642397 ret void2398}2399 2400define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %in.vecptr3, ptr %in.vecptr4, ptr %in.vecptr5, ptr %out.vec) nounwind {2401; SSE-LABEL: store_i16_stride6_vf32:2402; SSE: # %bb.0:2403; SSE-NEXT: subq $312, %rsp # imm = 0x1382404; SSE-NEXT: movdqa (%rdi), %xmm22405; SSE-NEXT: movdqa 16(%rdi), %xmm132406; SSE-NEXT: movdqa (%rsi), %xmm32407; SSE-NEXT: movdqa 16(%rsi), %xmm12408; SSE-NEXT: movdqa (%rdx), %xmm52409; SSE-NEXT: movdqa 16(%rdx), %xmm142410; SSE-NEXT: movdqa (%rcx), %xmm42411; SSE-NEXT: movdqa 16(%rcx), %xmm102412; SSE-NEXT: movdqa (%r8), %xmm82413; SSE-NEXT: movdqa (%r9), %xmm112414; SSE-NEXT: movdqa %xmm5, %xmm02415; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]2416; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2417; SSE-NEXT: movdqa %xmm2, %xmm92418; SSE-NEXT: punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm3[0],xmm9[1],xmm3[1],xmm9[2],xmm3[2],xmm9[3],xmm3[3]2419; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2420; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[2,3],xmm0[3,3]2421; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm8[2,1,3,3,4,5,6,7]2422; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[1,2],xmm7[0,1]2423; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[2,0,1,3]2424; SSE-NEXT: movaps {{.*#+}} xmm6 = [65535,0,65535,65535,65535,65535,65535,0]2425; SSE-NEXT: andps %xmm6, %xmm92426; SSE-NEXT: movdqa %xmm11, %xmm72427; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2428; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm11[0,2,2,3,4,5,6,7]2429; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm11[0,1,2,1]2430; SSE-NEXT: movaps %xmm6, %xmm02431; SSE-NEXT: andnps %xmm11, %xmm02432; SSE-NEXT: orps %xmm9, %xmm02433; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2434; SSE-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]2435; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2436; SSE-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]2437; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2438; SSE-NEXT: movdqa %xmm2, %xmm32439; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,3],xmm5[3,3]2440; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm8[0,1,2,3,6,5,7,7]2441; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,2],xmm4[2,3]2442; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0,1,3]2443; SSE-NEXT: andps %xmm6, %xmm32444; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm7[0,1,2,3,4,6,6,7]2445; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[2,1,2,3]2446; SSE-NEXT: movaps %xmm6, %xmm02447; SSE-NEXT: andnps %xmm4, %xmm02448; SSE-NEXT: orps %xmm3, %xmm02449; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2450; SSE-NEXT: movdqa %xmm14, %xmm02451; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]2452; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2453; SSE-NEXT: movdqa %xmm13, %xmm112454; SSE-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm1[0],xmm11[1],xmm1[1],xmm11[2],xmm1[2],xmm11[3],xmm1[3]2455; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2456; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[2,3],xmm0[3,3]2457; SSE-NEXT: movdqa 16(%r8), %xmm152458; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm15[2,1,3,3,4,5,6,7]2459; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[1,2],xmm9[0,1]2460; SSE-NEXT: movdqa 16(%r9), %xmm92461; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm9[0,2,2,3,4,5,6,7]2462; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm12[0,1,2,1]2463; SSE-NEXT: movaps %xmm6, %xmm02464; SSE-NEXT: andnps %xmm12, %xmm02465; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[2,0,1,3]2466; SSE-NEXT: andps %xmm6, %xmm112467; SSE-NEXT: orps %xmm11, %xmm02468; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2469; SSE-NEXT: punpckhwd {{.*#+}} xmm14 = xmm14[4],xmm10[4],xmm14[5],xmm10[5],xmm14[6],xmm10[6],xmm14[7],xmm10[7]2470; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2471; SSE-NEXT: punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm1[4],xmm13[5],xmm1[5],xmm13[6],xmm1[6],xmm13[7],xmm1[7]2472; SSE-NEXT: movdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2473; SSE-NEXT: movdqa %xmm13, %xmm12474; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,3],xmm14[3,3]2475; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm15[0,1,2,3,6,5,7,7]2476; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,2],xmm10[2,3]2477; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm9[0,1,2,3,4,6,6,7]2478; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[2,1,2,3]2479; SSE-NEXT: movaps %xmm6, %xmm02480; SSE-NEXT: andnps %xmm10, %xmm02481; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0,1,3]2482; SSE-NEXT: andps %xmm6, %xmm12483; SSE-NEXT: orps %xmm1, %xmm02484; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2485; SSE-NEXT: movdqa 32(%rdx), %xmm22486; SSE-NEXT: movdqa 32(%rcx), %xmm12487; SSE-NEXT: movdqa %xmm2, %xmm02488; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]2489; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2490; SSE-NEXT: movdqa 32(%rdi), %xmm32491; SSE-NEXT: movdqa 32(%rsi), %xmm112492; SSE-NEXT: movdqa %xmm3, %xmm102493; SSE-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1],xmm10[2],xmm11[2],xmm10[3],xmm11[3]2494; SSE-NEXT: movdqa %xmm10, %xmm122495; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[2,3],xmm0[3,3]2496; SSE-NEXT: movdqa 32(%r8), %xmm142497; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm14[2,1,3,3,4,5,6,7]2498; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[1,2],xmm13[0,1]2499; SSE-NEXT: movdqa 32(%r9), %xmm42500; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm4[0,2,2,3,4,5,6,7]2501; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2502; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1]2503; SSE-NEXT: movaps %xmm6, %xmm132504; SSE-NEXT: andnps %xmm0, %xmm132505; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[2,0,1,3]2506; SSE-NEXT: andps %xmm6, %xmm122507; SSE-NEXT: orps %xmm12, %xmm132508; SSE-NEXT: movaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2509; SSE-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]2510; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2511; SSE-NEXT: movdqa %xmm3, %xmm02512; SSE-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm11[4],xmm0[5],xmm11[5],xmm0[6],xmm11[6],xmm0[7],xmm11[7]2513; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2514; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,3],xmm2[3,3]2515; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm14[0,1,2,3,6,5,7,7]2516; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,2],xmm1[2,3]2517; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm4[0,1,2,3,4,6,6,7]2518; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]2519; SSE-NEXT: movaps %xmm6, %xmm112520; SSE-NEXT: andnps %xmm1, %xmm112521; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0,1,3]2522; SSE-NEXT: andps %xmm6, %xmm02523; SSE-NEXT: orps %xmm0, %xmm112524; SSE-NEXT: movaps %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2525; SSE-NEXT: movdqa 48(%rdx), %xmm32526; SSE-NEXT: movdqa 48(%rcx), %xmm42527; SSE-NEXT: movdqa %xmm3, %xmm52528; SSE-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]2529; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2530; SSE-NEXT: movdqa 48(%rdi), %xmm02531; SSE-NEXT: movdqa 48(%rsi), %xmm12532; SSE-NEXT: movdqa %xmm0, %xmm112533; SSE-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm1[0],xmm11[1],xmm1[1],xmm11[2],xmm1[2],xmm11[3],xmm1[3]2534; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2535; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[2,3],xmm5[3,3]2536; SSE-NEXT: movdqa 48(%r8), %xmm122537; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm12[2,1,3,3,4,5,6,7]2538; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[1,2],xmm13[0,1]2539; SSE-NEXT: movdqa 48(%r9), %xmm22540; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm2[0,2,2,3,4,5,6,7]2541; SSE-NEXT: movdqa %xmm2, (%rsp) # 16-byte Spill2542; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm13[0,1,2,1]2543; SSE-NEXT: movaps %xmm6, %xmm72544; SSE-NEXT: andnps %xmm13, %xmm72545; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[2,0,1,3]2546; SSE-NEXT: andps %xmm6, %xmm112547; SSE-NEXT: orps %xmm11, %xmm72548; SSE-NEXT: movaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2549; SSE-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]2550; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2551; SSE-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]2552; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2553; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,3],xmm3[3,3]2554; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm12[0,1,2,3,6,5,7,7]2555; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,2],xmm1[2,3]2556; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0,1,3]2557; SSE-NEXT: andps %xmm6, %xmm02558; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm2[0,1,2,3,4,6,6,7]2559; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]2560; SSE-NEXT: andnps %xmm1, %xmm62561; SSE-NEXT: orps %xmm0, %xmm62562; SSE-NEXT: movaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2563; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload2564; SSE-NEXT: movaps %xmm5, %xmm02565; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload2566; SSE-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm4[0]2567; SSE-NEXT: movdqa %xmm8, %xmm12568; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm4[1,3]2569; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[0,2]2570; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload2571; SSE-NEXT: movdqa %xmm3, %xmm112572; SSE-NEXT: pslldq {{.*#+}} xmm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm11[0,1,2,3,4,5]2573; SSE-NEXT: movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535,65535,0,65535,65535]2574; SSE-NEXT: movdqa %xmm1, %xmm22575; SSE-NEXT: pandn %xmm11, %xmm22576; SSE-NEXT: andps %xmm1, %xmm02577; SSE-NEXT: por %xmm0, %xmm22578; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2579; SSE-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm5[1]2580; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm8[1,1,1,1,4,5,6,7]2581; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[1,1],xmm5[1,1]2582; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[2,0],xmm4[0,2]2583; SSE-NEXT: movdqa {{.*#+}} xmm5 = [65535,65535,65535,0,65535,65535,65535,65535]2584; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm3[0,0,1,1]2585; SSE-NEXT: movdqa %xmm5, %xmm02586; SSE-NEXT: pandn %xmm6, %xmm02587; SSE-NEXT: andps %xmm5, %xmm112588; SSE-NEXT: por %xmm11, %xmm02589; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2590; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload2591; SSE-NEXT: movaps %xmm7, %xmm62592; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload2593; SSE-NEXT: movlhps {{.*#+}} xmm6 = xmm6[0],xmm4[0]2594; SSE-NEXT: movdqa %xmm8, %xmm112595; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[2,1],xmm4[1,3]2596; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[2,0],xmm11[0,2]2597; SSE-NEXT: movdqa %xmm3, %xmm02598; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm3[2,2,3,3]2599; SSE-NEXT: pslld $16, %xmm02600; SSE-NEXT: movdqa %xmm1, %xmm22601; SSE-NEXT: pandn %xmm0, %xmm22602; SSE-NEXT: andps %xmm1, %xmm62603; SSE-NEXT: por %xmm6, %xmm22604; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2605; SSE-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm7[1]2606; SSE-NEXT: psrldq {{.*#+}} xmm8 = xmm8[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero2607; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[1,1],xmm7[1,1]2608; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[2,0],xmm4[0,2]2609; SSE-NEXT: movdqa %xmm5, %xmm02610; SSE-NEXT: pandn %xmm11, %xmm02611; SSE-NEXT: andps %xmm5, %xmm82612; SSE-NEXT: por %xmm8, %xmm02613; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2614; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload2615; SSE-NEXT: movaps %xmm3, %xmm62616; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2617; SSE-NEXT: movlhps {{.*#+}} xmm6 = xmm6[0],xmm0[0]2618; SSE-NEXT: movdqa %xmm15, %xmm22619; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm0[1,3]2620; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[2,0],xmm2[0,2]2621; SSE-NEXT: movdqa %xmm9, %xmm82622; SSE-NEXT: pslldq {{.*#+}} xmm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm8[0,1,2,3,4,5]2623; SSE-NEXT: movdqa %xmm1, %xmm22624; SSE-NEXT: pandn %xmm8, %xmm22625; SSE-NEXT: andps %xmm1, %xmm62626; SSE-NEXT: por %xmm6, %xmm22627; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2628; SSE-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm3[1]2629; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm15[1,1,1,1,4,5,6,7]2630; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[1,1],xmm3[1,1]2631; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[2,0],xmm0[0,2]2632; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm9[0,0,1,1]2633; SSE-NEXT: movdqa %xmm5, %xmm02634; SSE-NEXT: pandn %xmm8, %xmm02635; SSE-NEXT: andps %xmm5, %xmm62636; SSE-NEXT: por %xmm6, %xmm02637; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2638; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload2639; SSE-NEXT: movaps %xmm2, %xmm82640; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2641; SSE-NEXT: movlhps {{.*#+}} xmm8 = xmm8[0],xmm0[0]2642; SSE-NEXT: movdqa %xmm15, %xmm62643; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[2,1],xmm0[1,3]2644; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[2,0],xmm6[0,2]2645; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm9[2,2,3,3]2646; SSE-NEXT: pslld $16, %xmm92647; SSE-NEXT: movdqa %xmm1, %xmm72648; SSE-NEXT: pandn %xmm9, %xmm72649; SSE-NEXT: andps %xmm1, %xmm82650; SSE-NEXT: por %xmm8, %xmm72651; SSE-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm2[1]2652; SSE-NEXT: psrldq {{.*#+}} xmm15 = xmm15[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero2653; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[1,1],xmm2[1,1]2654; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[2,0],xmm0[0,2]2655; SSE-NEXT: movdqa %xmm5, %xmm82656; SSE-NEXT: pandn %xmm11, %xmm82657; SSE-NEXT: andps %xmm5, %xmm152658; SSE-NEXT: por %xmm15, %xmm82659; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload2660; SSE-NEXT: movdqa %xmm2, %xmm92661; SSE-NEXT: punpcklqdq {{.*#+}} xmm9 = xmm9[0],xmm10[0]2662; SSE-NEXT: movdqa %xmm14, %xmm32663; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm10[1,3]2664; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[2,0],xmm3[0,2]2665; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2666; SSE-NEXT: movdqa %xmm0, %xmm112667; SSE-NEXT: pslldq {{.*#+}} xmm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm11[0,1,2,3,4,5]2668; SSE-NEXT: movdqa %xmm1, %xmm62669; SSE-NEXT: pandn %xmm11, %xmm62670; SSE-NEXT: andps %xmm1, %xmm92671; SSE-NEXT: por %xmm9, %xmm62672; SSE-NEXT: punpckhqdq {{.*#+}} xmm10 = xmm10[1],xmm2[1]2673; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm14[1,1,1,1,4,5,6,7]2674; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[1,1],xmm2[1,1]2675; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[2,0],xmm10[0,2]2676; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm0[0,0,1,1]2677; SSE-NEXT: movdqa %xmm5, %xmm92678; SSE-NEXT: pandn %xmm13, %xmm92679; SSE-NEXT: andps %xmm5, %xmm112680; SSE-NEXT: por %xmm11, %xmm92681; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload2682; SSE-NEXT: movaps %xmm3, %xmm112683; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload2684; SSE-NEXT: movlhps {{.*#+}} xmm11 = xmm11[0],xmm2[0]2685; SSE-NEXT: movdqa %xmm14, %xmm132686; SSE-NEXT: shufps {{.*#+}} xmm13 = xmm13[2,1],xmm2[1,3]2687; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[2,0],xmm13[0,2]2688; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm0[2,2,3,3]2689; SSE-NEXT: pslld $16, %xmm02690; SSE-NEXT: movdqa %xmm1, %xmm152691; SSE-NEXT: pandn %xmm0, %xmm152692; SSE-NEXT: andps %xmm1, %xmm112693; SSE-NEXT: por %xmm11, %xmm152694; SSE-NEXT: unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm3[1]2695; SSE-NEXT: psrldq {{.*#+}} xmm14 = xmm14[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero2696; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[1,1],xmm3[1,1]2697; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[2,0],xmm2[0,2]2698; SSE-NEXT: movdqa %xmm5, %xmm102699; SSE-NEXT: pandn %xmm13, %xmm102700; SSE-NEXT: andps %xmm5, %xmm142701; SSE-NEXT: por %xmm14, %xmm102702; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload2703; SSE-NEXT: movaps %xmm2, %xmm112704; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2705; SSE-NEXT: movlhps {{.*#+}} xmm11 = xmm11[0],xmm0[0]2706; SSE-NEXT: movdqa %xmm12, %xmm132707; SSE-NEXT: shufps {{.*#+}} xmm13 = xmm13[0,1],xmm0[1,3]2708; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[2,0],xmm13[0,2]2709; SSE-NEXT: movdqa (%rsp), %xmm4 # 16-byte Reload2710; SSE-NEXT: movdqa %xmm4, %xmm142711; SSE-NEXT: pslldq {{.*#+}} xmm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm14[0,1,2,3,4,5]2712; SSE-NEXT: movdqa %xmm1, %xmm132713; SSE-NEXT: pandn %xmm14, %xmm132714; SSE-NEXT: andps %xmm1, %xmm112715; SSE-NEXT: por %xmm11, %xmm132716; SSE-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm2[1]2717; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm12[1,1,1,1,4,5,6,7]2718; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[1,1],xmm2[1,1]2719; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[2,0],xmm0[0,2]2720; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,0,1,1]2721; SSE-NEXT: movdqa %xmm5, %xmm112722; SSE-NEXT: pandn %xmm2, %xmm112723; SSE-NEXT: andps %xmm5, %xmm142724; SSE-NEXT: por %xmm14, %xmm112725; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload2726; SSE-NEXT: movaps %xmm3, %xmm22727; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2728; SSE-NEXT: movlhps {{.*#+}} xmm2 = xmm2[0],xmm0[0]2729; SSE-NEXT: movdqa %xmm12, %xmm142730; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[2,1],xmm0[1,3]2731; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm14[0,2]2732; SSE-NEXT: andps %xmm1, %xmm22733; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm4[2,2,3,3]2734; SSE-NEXT: pslld $16, %xmm42735; SSE-NEXT: pandn %xmm4, %xmm12736; SSE-NEXT: por %xmm2, %xmm12737; SSE-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm3[1]2738; SSE-NEXT: psrldq {{.*#+}} xmm12 = xmm12[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero2739; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[1,1],xmm3[1,1]2740; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[2,0],xmm0[0,2]2741; SSE-NEXT: andps %xmm5, %xmm122742; SSE-NEXT: pandn %xmm14, %xmm52743; SSE-NEXT: por %xmm12, %xmm52744; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax2745; SSE-NEXT: movdqa %xmm5, 352(%rax)2746; SSE-NEXT: movdqa %xmm1, 336(%rax)2747; SSE-NEXT: movdqa %xmm11, 304(%rax)2748; SSE-NEXT: movdqa %xmm13, 288(%rax)2749; SSE-NEXT: movdqa %xmm10, 256(%rax)2750; SSE-NEXT: movdqa %xmm15, 240(%rax)2751; SSE-NEXT: movdqa %xmm9, 208(%rax)2752; SSE-NEXT: movdqa %xmm6, 192(%rax)2753; SSE-NEXT: movdqa %xmm8, 160(%rax)2754; SSE-NEXT: movdqa %xmm7, 144(%rax)2755; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2756; SSE-NEXT: movaps %xmm0, 112(%rax)2757; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2758; SSE-NEXT: movaps %xmm0, 96(%rax)2759; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2760; SSE-NEXT: movaps %xmm0, 64(%rax)2761; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2762; SSE-NEXT: movaps %xmm0, 48(%rax)2763; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2764; SSE-NEXT: movaps %xmm0, 16(%rax)2765; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2766; SSE-NEXT: movaps %xmm0, (%rax)2767; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2768; SSE-NEXT: movaps %xmm0, 368(%rax)2769; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2770; SSE-NEXT: movaps %xmm0, 320(%rax)2771; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2772; SSE-NEXT: movaps %xmm0, 272(%rax)2773; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2774; SSE-NEXT: movaps %xmm0, 224(%rax)2775; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2776; SSE-NEXT: movaps %xmm0, 176(%rax)2777; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2778; SSE-NEXT: movaps %xmm0, 128(%rax)2779; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2780; SSE-NEXT: movaps %xmm0, 80(%rax)2781; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2782; SSE-NEXT: movaps %xmm0, 32(%rax)2783; SSE-NEXT: addq $312, %rsp # imm = 0x1382784; SSE-NEXT: retq2785;2786; AVX-LABEL: store_i16_stride6_vf32:2787; AVX: # %bb.0:2788; AVX-NEXT: subq $120, %rsp2789; AVX-NEXT: vmovdqa 32(%rcx), %xmm82790; AVX-NEXT: vmovdqa 48(%rcx), %xmm02791; AVX-NEXT: vmovdqa 32(%rdx), %xmm92792; AVX-NEXT: vmovdqa 48(%rdx), %xmm12793; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]2794; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm6[2,2,3,3]2795; AVX-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]2796; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm3[0,0,1,1]2797; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm12798; AVX-NEXT: vmovdqa 32(%rsi), %xmm102799; AVX-NEXT: vmovdqa 48(%rsi), %xmm22800; AVX-NEXT: vmovdqa 32(%rdi), %xmm112801; AVX-NEXT: vmovdqa 48(%rdi), %xmm42802; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]2803; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[2,3,2,3]2804; AVX-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]2805; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm4[0,1,0,1]2806; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm5, %ymm22807; AVX-NEXT: vblendps {{.*#+}} ymm5 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]2808; AVX-NEXT: vextractf128 $1, %ymm5, %xmm22809; AVX-NEXT: vmovdqa 48(%r8), %xmm12810; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm2[0,1,2,3],xmm1[4,5],xmm2[6,7]2811; AVX-NEXT: vmovdqa 48(%r9), %xmm22812; AVX-NEXT: vpslld $16, %xmm2, %xmm122813; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4],xmm12[5],xmm7[6,7]2814; AVX-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2815; AVX-NEXT: vpshuflw {{.*#+}} xmm7 = xmm1[2,1,3,3,4,5,6,7]2816; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[0,1,2,1]2817; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm7[0],xmm5[1,2],xmm7[3]2818; AVX-NEXT: vpshuflw {{.*#+}} xmm7 = xmm2[0,2,2,3,4,5,6,7]2819; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[0,1,2,1]2820; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0],xmm7[1],xmm5[2,3,4,5,6],xmm7[7]2821; AVX-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2822; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]2823; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm4, %ymm42824; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[1,1,2,2]2825; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,2,3,3]2826; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm5, %ymm32827; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]2828; AVX-NEXT: vextractf128 $1, %ymm3, %xmm42829; AVX-NEXT: vpshufhw {{.*#+}} xmm5 = xmm1[0,1,2,3,6,5,7,7]2830; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[2,1,2,3]2831; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3,4,5],xmm5[6,7]2832; AVX-NEXT: vpshufhw {{.*#+}} xmm5 = xmm2[0,1,2,3,4,6,6,7]2833; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[2,1,2,3]2834; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm5[1],xmm4[2,3,4,5,6],xmm5[7]2835; AVX-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2836; AVX-NEXT: vpsrldq {{.*#+}} xmm4 = xmm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero2837; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm3[0],xmm4[1],xmm3[2,3]2838; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,2,3,3]2839; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[3],xmm3[4,5,6,7]2840; AVX-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2841; AVX-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]2842; AVX-NEXT: vpunpckhwd {{.*#+}} xmm7 = xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]2843; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm7[1,1,2,2]2844; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm7[2,2,3,3]2845; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm4, %ymm42846; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[2,3,2,3]2847; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm3, %ymm52848; AVX-NEXT: vblendps {{.*#+}} ymm12 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]2849; AVX-NEXT: vmovdqa 32(%r8), %xmm42850; AVX-NEXT: vpshufhw {{.*#+}} xmm5 = xmm4[0,1,2,3,6,5,7,7]2851; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[2,1,2,3]2852; AVX-NEXT: vextractf128 $1, %ymm12, %xmm132853; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm5[0,1],xmm13[2,3,4,5],xmm5[6,7]2854; AVX-NEXT: vmovdqa 32(%r9), %xmm52855; AVX-NEXT: vpshufhw {{.*#+}} xmm14 = xmm5[0,1,2,3,4,6,6,7]2856; AVX-NEXT: vpshufd {{.*#+}} xmm14 = xmm14[2,1,2,3]2857; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0],xmm14[1],xmm13[2,3,4,5,6],xmm14[7]2858; AVX-NEXT: vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2859; AVX-NEXT: vpsrldq {{.*#+}} xmm13 = xmm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero2860; AVX-NEXT: vblendps {{.*#+}} xmm12 = xmm12[0],xmm13[1],xmm12[2,3]2861; AVX-NEXT: vpshufd {{.*#+}} xmm13 = xmm5[2,2,3,3]2862; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1,2],xmm13[3],xmm12[4,5,6,7]2863; AVX-NEXT: vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2864; AVX-NEXT: vpshufd {{.*#+}} xmm12 = xmm6[0,0,1,1]2865; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[1,1,2,2]2866; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm12, %ymm62867; AVX-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[0,1,0,1]2868; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm12, %ymm02869; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0],ymm6[1],ymm0[2,3],ymm6[4],ymm0[5,6],ymm6[7]2870; AVX-NEXT: vextractf128 $1, %ymm0, %xmm62871; AVX-NEXT: vpmovzxwd {{.*#+}} xmm12 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero2872; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm12[2,3],xmm6[4,5,6,7]2873; AVX-NEXT: vpshufd {{.*#+}} xmm12 = xmm2[0,0,1,1]2874; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2],xmm12[3],xmm6[4,5,6,7]2875; AVX-NEXT: vmovdqa %xmm6, (%rsp) # 16-byte Spill2876; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]2877; AVX-NEXT: vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2,3,4,5]2878; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm1[5],xmm0[6,7]2879; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2880; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]2881; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,0,1,1]2882; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,2]2883; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm12884; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]2885; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm2[0,1,0,1]2886; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm6, %ymm62887; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm6[0],ymm1[1],ymm6[2,3],ymm1[4],ymm6[5,6],ymm1[7]2888; AVX-NEXT: vextractf128 $1, %ymm1, %xmm62889; AVX-NEXT: vpmovzxwd {{.*#+}} xmm8 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero2890; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm8[2,3],xmm6[4,5,6,7]2891; AVX-NEXT: vpshufd {{.*#+}} xmm8 = xmm5[0,0,1,1]2892; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2],xmm8[3],xmm6[4,5,6,7]2893; AVX-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2894; AVX-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm4[0],xmm1[3]2895; AVX-NEXT: vpslldq {{.*#+}} xmm6 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm5[0,1,2,3,4,5]2896; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm6[5],xmm1[6,7]2897; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2898; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]2899; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm7[0,0,1,1]2900; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm02901; AVX-NEXT: vmovdqa 16(%rcx), %xmm12902; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]2903; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,1,0,1]2904; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm22905; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2],ymm2[3,4],ymm0[5],ymm2[6,7]2906; AVX-NEXT: vextractf128 $1, %ymm0, %xmm22907; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm4[4,5],xmm2[6,7]2908; AVX-NEXT: vpslld $16, %xmm5, %xmm32909; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4],xmm3[5],xmm2[6,7]2910; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2911; AVX-NEXT: vmovdqa 16(%rdx), %xmm22912; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm4[2,1,3,3,4,5,6,7]2913; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,1,2,1]2914; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm3[0],xmm0[1,2],xmm3[3]2915; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm5[0,2,2,3,4,5,6,7]2916; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,1,2,1]2917; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2,3,4,5,6],xmm3[7]2918; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2919; AVX-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]2920; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm11[2,2,3,3]2921; AVX-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]2922; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[0,0,1,1]2923; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm02924; AVX-NEXT: vmovdqa 16(%rsi), %xmm22925; AVX-NEXT: vmovdqa 16(%rdi), %xmm32926; AVX-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]2927; AVX-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]2928; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm10[2,3,2,3]2929; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[0,1,0,1]2930; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm22931; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2],ymm2[3,4],ymm0[5],ymm2[6,7]2932; AVX-NEXT: vmovdqa 16(%r8), %xmm32933; AVX-NEXT: vextractf128 $1, %ymm0, %xmm22934; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm2[0,1,2,3],xmm3[4,5],xmm2[6,7]2935; AVX-NEXT: vmovdqa 16(%r9), %xmm22936; AVX-NEXT: vpslld $16, %xmm2, %xmm62937; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4],xmm6[5],xmm5[6,7]2938; AVX-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2939; AVX-NEXT: vpshuflw {{.*#+}} xmm5 = xmm3[2,1,3,3,4,5,6,7]2940; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[0,1,2,1]2941; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm5[0],xmm0[1,2],xmm5[3]2942; AVX-NEXT: vpshuflw {{.*#+}} xmm5 = xmm2[0,2,2,3,4,5,6,7]2943; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[0,1,2,1]2944; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm5[1],xmm0[2,3,4,5,6],xmm5[7]2945; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2946; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]2947; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm4, %ymm02948; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[1,1,2,2]2949; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]2950; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm4, %ymm12951; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]2952; AVX-NEXT: vpshufhw {{.*#+}} xmm1 = xmm3[0,1,2,3,6,5,7,7]2953; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]2954; AVX-NEXT: vextractf128 $1, %ymm0, %xmm42955; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3,4,5],xmm1[6,7]2956; AVX-NEXT: vpshufhw {{.*#+}} xmm4 = xmm2[0,1,2,3,4,6,6,7]2957; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[2,1,2,3]2958; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm4[1],xmm1[2,3,4,5,6],xmm4[7]2959; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2960; AVX-NEXT: vpsrldq {{.*#+}} xmm1 = xmm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero2961; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]2962; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]2963; AVX-NEXT: vpblendw {{.*#+}} xmm15 = xmm0[0,1,2],xmm1[3],xmm0[4,5,6,7]2964; AVX-NEXT: vmovdqa (%rcx), %xmm92965; AVX-NEXT: vmovdqa (%rdx), %xmm82966; AVX-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm8[4],xmm9[4],xmm8[5],xmm9[5],xmm8[6],xmm9[6],xmm8[7],xmm9[7]2967; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm5[1,1,2,2]2968; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm5[2,2,3,3]2969; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm02970; AVX-NEXT: vmovdqa (%rsi), %xmm72971; AVX-NEXT: vmovdqa (%rdi), %xmm62972; AVX-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7]2973; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm4[2,3,2,3]2974; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm4, %ymm12975; AVX-NEXT: vblendps {{.*#+}} ymm12 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]2976; AVX-NEXT: vmovdqa (%r8), %xmm12977; AVX-NEXT: vpshufhw {{.*#+}} xmm0 = xmm1[0,1,2,3,6,5,7,7]2978; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]2979; AVX-NEXT: vextractf128 $1, %ymm12, %xmm132980; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm0[0,1],xmm13[2,3,4,5],xmm0[6,7]2981; AVX-NEXT: vmovdqa (%r9), %xmm02982; AVX-NEXT: vpshufhw {{.*#+}} xmm14 = xmm0[0,1,2,3,4,6,6,7]2983; AVX-NEXT: vpshufd {{.*#+}} xmm14 = xmm14[2,1,2,3]2984; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0],xmm14[1],xmm13[2,3,4,5,6],xmm14[7]2985; AVX-NEXT: vpsrldq {{.*#+}} xmm14 = xmm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero2986; AVX-NEXT: vblendps {{.*#+}} xmm12 = xmm12[0],xmm14[1],xmm12[2,3]2987; AVX-NEXT: vpshufd {{.*#+}} xmm14 = xmm0[2,2,3,3]2988; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1,2],xmm14[3],xmm12[4,5,6,7]2989; AVX-NEXT: vpshufd {{.*#+}} xmm14 = xmm11[0,0,1,1]2990; AVX-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[1,1,2,2]2991; AVX-NEXT: vinsertf128 $1, %xmm11, %ymm14, %ymm112992; AVX-NEXT: vpshufd {{.*#+}} xmm14 = xmm10[0,1,0,1]2993; AVX-NEXT: vinsertf128 $1, %xmm10, %ymm14, %ymm102994; AVX-NEXT: vblendps {{.*#+}} ymm11 = ymm10[0],ymm11[1],ymm10[2,3],ymm11[4],ymm10[5,6],ymm11[7]2995; AVX-NEXT: vextractf128 $1, %ymm11, %xmm102996; AVX-NEXT: vpmovzxwd {{.*#+}} xmm14 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero2997; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1],xmm14[2,3],xmm10[4,5,6,7]2998; AVX-NEXT: vpshufd {{.*#+}} xmm14 = xmm2[0,0,1,1]2999; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1,2],xmm14[3],xmm10[4,5,6,7]3000; AVX-NEXT: vinsertps {{.*#+}} xmm3 = xmm11[0,1],xmm3[0],xmm11[3]3001; AVX-NEXT: vpslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2,3,4,5]3002; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3,4],xmm2[5],xmm3[6,7]3003; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]3004; AVX-NEXT: vpshufd {{.*#+}} xmm8 = xmm3[0,0,1,1]3005; AVX-NEXT: vpshufd {{.*#+}} xmm9 = xmm3[1,1,2,2]3006; AVX-NEXT: vinsertf128 $1, %xmm9, %ymm8, %ymm83007; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]3008; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[0,1,0,1]3009; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm7, %ymm73010; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm7[0],ymm8[1],ymm7[2,3],ymm8[4],ymm7[5,6],ymm8[7]3011; AVX-NEXT: vextractf128 $1, %ymm7, %xmm83012; AVX-NEXT: vpmovzxwd {{.*#+}} xmm9 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero3013; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1],xmm9[2,3],xmm8[4,5,6,7]3014; AVX-NEXT: vpshufd {{.*#+}} xmm9 = xmm0[0,0,1,1]3015; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2],xmm9[3],xmm8[4,5,6,7]3016; AVX-NEXT: vinsertps {{.*#+}} xmm7 = xmm7[0,1],xmm1[0],xmm7[3]3017; AVX-NEXT: vpslldq {{.*#+}} xmm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]3018; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4],xmm9[5],xmm7[6,7]3019; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,2,3,3]3020; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[0,0,1,1]3021; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm3, %ymm33022; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm6[2,3,2,3]3023; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]3024; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm5, %ymm43025; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7]3026; AVX-NEXT: vextractf128 $1, %ymm3, %xmm43027; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm1[4,5],xmm4[6,7]3028; AVX-NEXT: vpslld $16, %xmm0, %xmm53029; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm5[5],xmm4[6,7]3030; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,1,3,3,4,5,6,7]3031; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]3032; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0],xmm3[1,2],xmm1[3]3033; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]3034; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,2,1]3035; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3,4,5,6],xmm0[7]3036; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax3037; AVX-NEXT: vmovdqa %xmm0, 32(%rax)3038; AVX-NEXT: vmovdqa %xmm4, 48(%rax)3039; AVX-NEXT: vmovdqa %xmm7, (%rax)3040; AVX-NEXT: vmovdqa %xmm8, 16(%rax)3041; AVX-NEXT: vmovdqa %xmm2, 96(%rax)3042; AVX-NEXT: vmovdqa %xmm10, 112(%rax)3043; AVX-NEXT: vmovdqa %xmm12, 64(%rax)3044; AVX-NEXT: vmovdqa %xmm13, 80(%rax)3045; AVX-NEXT: vmovdqa %xmm15, 160(%rax)3046; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3047; AVX-NEXT: vmovaps %xmm0, 176(%rax)3048; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3049; AVX-NEXT: vmovaps %xmm0, 128(%rax)3050; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3051; AVX-NEXT: vmovaps %xmm0, 144(%rax)3052; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3053; AVX-NEXT: vmovaps %xmm0, 224(%rax)3054; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3055; AVX-NEXT: vmovaps %xmm0, 240(%rax)3056; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3057; AVX-NEXT: vmovaps %xmm0, 192(%rax)3058; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3059; AVX-NEXT: vmovaps %xmm0, 208(%rax)3060; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3061; AVX-NEXT: vmovaps %xmm0, 288(%rax)3062; AVX-NEXT: vmovaps (%rsp), %xmm0 # 16-byte Reload3063; AVX-NEXT: vmovaps %xmm0, 304(%rax)3064; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3065; AVX-NEXT: vmovaps %xmm0, 256(%rax)3066; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3067; AVX-NEXT: vmovaps %xmm0, 272(%rax)3068; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3069; AVX-NEXT: vmovaps %xmm0, 352(%rax)3070; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3071; AVX-NEXT: vmovaps %xmm0, 368(%rax)3072; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3073; AVX-NEXT: vmovaps %xmm0, 320(%rax)3074; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3075; AVX-NEXT: vmovaps %xmm0, 336(%rax)3076; AVX-NEXT: addq $120, %rsp3077; AVX-NEXT: vzeroupper3078; AVX-NEXT: retq3079;3080; AVX2-LABEL: store_i16_stride6_vf32:3081; AVX2: # %bb.0:3082; AVX2-NEXT: subq $616, %rsp # imm = 0x2683083; AVX2-NEXT: vmovdqa (%rcx), %xmm133084; AVX2-NEXT: vmovdqa 32(%rcx), %xmm93085; AVX2-NEXT: vpsrldq {{.*#+}} xmm0 = xmm13[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero3086; AVX2-NEXT: vmovdqa (%rdx), %xmm13087; AVX2-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3088; AVX2-NEXT: vmovdqa 32(%rdx), %xmm113089; AVX2-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero3090; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3091; AVX2-NEXT: vpbroadcastq %xmm0, %ymm03092; AVX2-NEXT: vmovdqa (%rsi), %xmm153093; AVX2-NEXT: vmovdqa 32(%rsi), %xmm53094; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm15[0,1,2,1]3095; AVX2-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,5]3096; AVX2-NEXT: vmovdqa (%rdi), %xmm23097; AVX2-NEXT: vmovdqa %xmm2, (%rsp) # 16-byte Spill3098; AVX2-NEXT: vmovdqa 32(%rdi), %xmm143099; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,2,1]3100; AVX2-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,5]3101; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]3102; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]3103; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]3104; AVX2-NEXT: vmovdqa (%r8), %xmm13105; AVX2-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3106; AVX2-NEXT: vmovdqa 32(%r8), %xmm73107; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,1,3,3,4,5,6,7]3108; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]3109; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7]3110; AVX2-NEXT: vmovdqa (%r9), %xmm03111; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3112; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]3113; AVX2-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]3114; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm0[0,0,2,1]3115; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm0 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]3116; AVX2-NEXT: vpblendvb %ymm0, %ymm1, %ymm2, %ymm13117; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3118; AVX2-NEXT: vpsrldq {{.*#+}} xmm1 = xmm9[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero3119; AVX2-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3120; AVX2-NEXT: vpsrldq {{.*#+}} xmm2 = xmm11[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero3121; AVX2-NEXT: vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3122; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]3123; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm5[0,1,2,1]3124; AVX2-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3125; AVX2-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,5]3126; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm14[0,1,2,1]3127; AVX2-NEXT: vmovdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3128; AVX2-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,5]3129; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]3130; AVX2-NEXT: vmovdqa 32(%r9), %xmm43131; AVX2-NEXT: vpbroadcastq %xmm1, %ymm13132; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]3133; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]3134; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm7[2,1,3,3,4,5,6,7]3135; AVX2-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3136; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]3137; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]3138; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm4[0,2,2,3,4,5,6,7]3139; AVX2-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3140; AVX2-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,4,4]3141; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]3142; AVX2-NEXT: vpblendvb %ymm0, %ymm1, %ymm2, %ymm13143; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3144; AVX2-NEXT: vmovdqa 32(%rdx), %ymm23145; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3146; AVX2-NEXT: vmovdqa 32(%rcx), %ymm13147; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3148; AVX2-NEXT: vpsrldq {{.*#+}} ymm1 = ymm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm1[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero3149; AVX2-NEXT: vpsrldq {{.*#+}} ymm2 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero3150; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]3151; AVX2-NEXT: vmovdqa 32(%rsi), %ymm103152; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm10[2,1,2,3,6,5,6,7]3153; AVX2-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3154; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]3155; AVX2-NEXT: vmovdqa 32(%rdi), %ymm83156; AVX2-NEXT: vpshufd {{.*#+}} ymm3 = ymm8[2,1,2,3,6,5,6,7]3157; AVX2-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3158; AVX2-NEXT: vpshuflw {{.*#+}} ymm3 = ymm3[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]3159; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]3160; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]3161; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]3162; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]3163; AVX2-NEXT: vmovdqa 32(%r8), %ymm23164; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3165; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]3166; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]3167; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]3168; AVX2-NEXT: vmovdqa 32(%r9), %ymm23169; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3170; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]3171; AVX2-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]3172; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]3173; AVX2-NEXT: vpblendvb %ymm0, %ymm1, %ymm2, %ymm13174; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3175; AVX2-NEXT: vmovdqa (%rdx), %ymm23176; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3177; AVX2-NEXT: vmovdqa (%rcx), %ymm13178; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3179; AVX2-NEXT: vpsrldq {{.*#+}} ymm1 = ymm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm1[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero3180; AVX2-NEXT: vpsrldq {{.*#+}} ymm2 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero3181; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]3182; AVX2-NEXT: vmovdqa (%rsi), %ymm23183; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3184; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[2,1,2,3,6,5,6,7]3185; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]3186; AVX2-NEXT: vmovdqa (%rdi), %ymm123187; AVX2-NEXT: vpshufd {{.*#+}} ymm3 = ymm12[2,1,2,3,6,5,6,7]3188; AVX2-NEXT: vpshuflw {{.*#+}} ymm3 = ymm3[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]3189; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]3190; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]3191; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]3192; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]3193; AVX2-NEXT: vmovdqa (%r8), %ymm23194; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3195; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]3196; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]3197; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]3198; AVX2-NEXT: vmovdqa (%r9), %ymm23199; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3200; AVX2-NEXT: vpshuflw {{.*#+}} ymm6 = ymm2[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]3201; AVX2-NEXT: vpshufhw {{.*#+}} ymm6 = ymm6[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]3202; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm6[2,2,2,3]3203; AVX2-NEXT: vpblendvb %ymm0, %ymm1, %ymm6, %ymm03204; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3205; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm14[4],xmm5[4],xmm14[5],xmm5[5],xmm14[6],xmm5[6],xmm14[7],xmm5[7]3206; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,1,1,1]3207; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm11[4],xmm9[4],xmm11[5],xmm9[5],xmm11[6],xmm9[6],xmm11[7],xmm9[7]3208; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,2,3,3]3209; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]3210; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]3211; AVX2-NEXT: vmovdqa {{.*#+}} xmm1 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]3212; AVX2-NEXT: vpshufb %xmm1, %xmm7, %xmm63213; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,1,0,1]3214; AVX2-NEXT: vpblendd {{.*#+}} ymm6 = ymm0[0],ymm6[1],ymm0[2,3],ymm6[4],ymm0[5,6],ymm6[7]3215; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]3216; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,1,4,5,6,7]3217; AVX2-NEXT: vpermq {{.*#+}} ymm14 = ymm0[0,1,0,1]3218; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm3 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]3219; AVX2-NEXT: vpblendvb %ymm3, %ymm6, %ymm14, %ymm03220; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3221; AVX2-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload3222; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm6 = xmm0[4],xmm15[4],xmm0[5],xmm15[5],xmm0[6],xmm15[6],xmm0[7],xmm15[7]3223; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm6[1,1,1,1]3224; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload3225; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm14 = xmm9[4],xmm13[4],xmm9[5],xmm13[5],xmm9[6],xmm13[6],xmm9[7],xmm13[7]3226; AVX2-NEXT: vpshufd {{.*#+}} xmm14 = xmm14[1,2,3,3]3227; AVX2-NEXT: vpermq {{.*#+}} ymm14 = ymm14[0,0,2,1]3228; AVX2-NEXT: vpblendd {{.*#+}} ymm6 = ymm14[0,1],ymm6[2],ymm14[3,4],ymm6[5],ymm14[6,7]3229; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload3230; AVX2-NEXT: vpshufb %xmm1, %xmm11, %xmm13231; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]3232; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm6[0],ymm1[1],ymm6[2,3],ymm1[4],ymm6[5,6],ymm1[7]3233; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload3234; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm7[2,3,2,3]3235; AVX2-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,2,2,1,4,5,6,7]3236; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,1,0,1]3237; AVX2-NEXT: vpblendvb %ymm3, %ymm1, %ymm6, %ymm13238; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3239; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm8[4],ymm10[4],ymm8[5],ymm10[5],ymm8[6],ymm10[6],ymm8[7],ymm10[7],ymm8[12],ymm10[12],ymm8[13],ymm10[13],ymm8[14],ymm10[14],ymm8[15],ymm10[15]3240; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[3,3,3,3]3241; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload3242; AVX2-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm6 # 32-byte Folded Reload3243; AVX2-NEXT: # ymm6 = ymm2[4],mem[4],ymm2[5],mem[5],ymm2[6],mem[6],ymm2[7],mem[7],ymm2[12],mem[12],ymm2[13],mem[13],ymm2[14],mem[14],ymm2[15],mem[15]3244; AVX2-NEXT: vpshufd {{.*#+}} ymm6 = ymm6[1,2,3,3,5,6,7,7]3245; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm6[2,2,2,3]3246; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm6[0,1],ymm1[2],ymm6[3,4],ymm1[5],ymm6[6,7]3247; AVX2-NEXT: vmovdqa {{.*#+}} ymm6 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]3248; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload3249; AVX2-NEXT: vpshufb %ymm6, %ymm2, %ymm143250; AVX2-NEXT: vpermq {{.*#+}} ymm14 = ymm14[2,1,2,3]3251; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm14[1],ymm1[2,3],ymm14[4],ymm1[5,6],ymm14[7]3252; AVX2-NEXT: vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload3253; AVX2-NEXT: # ymm14 = mem[2,3,2,3,6,7,6,7]3254; AVX2-NEXT: vpshuflw {{.*#+}} ymm14 = ymm14[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]3255; AVX2-NEXT: vpermq {{.*#+}} ymm14 = ymm14[2,1,2,3]3256; AVX2-NEXT: vpblendvb %ymm3, %ymm1, %ymm14, %ymm13257; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3258; AVX2-NEXT: vmovdqa %ymm3, %ymm143259; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload3260; AVX2-NEXT: vmovdqa %ymm12, %ymm33261; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm12[4],ymm4[4],ymm12[5],ymm4[5],ymm12[6],ymm4[6],ymm12[7],ymm4[7],ymm12[12],ymm4[12],ymm12[13],ymm4[13],ymm12[14],ymm4[14],ymm12[15],ymm4[15]3262; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[3,3,3,3]3263; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload3264; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload3265; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm12 = ymm2[4],ymm5[4],ymm2[5],ymm5[5],ymm2[6],ymm5[6],ymm2[7],ymm5[7],ymm2[12],ymm5[12],ymm2[13],ymm5[13],ymm2[14],ymm5[14],ymm2[15],ymm5[15]3266; AVX2-NEXT: vpshufd {{.*#+}} ymm12 = ymm12[1,2,3,3,5,6,7,7]3267; AVX2-NEXT: vpermq {{.*#+}} ymm12 = ymm12[2,2,2,3]3268; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm12[0,1],ymm1[2],ymm12[3,4],ymm1[5],ymm12[6,7]3269; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload3270; AVX2-NEXT: vpshufb %ymm6, %ymm8, %ymm63271; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm6[2,1,2,3]3272; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm6[1],ymm1[2,3],ymm6[4],ymm1[5,6],ymm6[7]3273; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload3274; AVX2-NEXT: vpshufd {{.*#+}} ymm6 = ymm10[2,3,2,3,6,7,6,7]3275; AVX2-NEXT: vpshuflw {{.*#+}} ymm6 = ymm6[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]3276; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm6[2,1,2,3]3277; AVX2-NEXT: vpblendvb %ymm14, %ymm1, %ymm6, %ymm63278; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm15[0],xmm0[1],xmm15[1],xmm0[2],xmm15[2],xmm0[3],xmm15[3]3279; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm9[0],xmm13[0],xmm9[1],xmm13[1],xmm9[2],xmm13[2],xmm9[3],xmm13[3]3280; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]3281; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,0,2,2]3282; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]3283; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]3284; AVX2-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm11[0],zero,xmm11[1],zero,xmm11[2],zero,xmm11[3],zero3285; AVX2-NEXT: vpbroadcastq %xmm1, %ymm13286; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]3287; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm7[0,0,2,1,4,5,6,7]3288; AVX2-NEXT: vpbroadcastq %xmm0, %ymm123289; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm15 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]3290; AVX2-NEXT: vpblendvb %ymm15, %ymm1, %ymm12, %ymm123291; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3292; AVX2-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload3293; AVX2-NEXT: # xmm1 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]3294; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3295; AVX2-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload3296; AVX2-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]3297; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]3298; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,0,2,2]3299; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]3300; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]3301; AVX2-NEXT: vpmovzxwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3302; AVX2-NEXT: # xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero3303; AVX2-NEXT: vpbroadcastq %xmm1, %ymm13304; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]3305; AVX2-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3306; AVX2-NEXT: # xmm1 = mem[0,0,2,1,4,5,6,7]3307; AVX2-NEXT: vpbroadcastq %xmm1, %ymm13308; AVX2-NEXT: vpblendvb %ymm15, %ymm0, %ymm1, %ymm13309; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3310; AVX2-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload3311; AVX2-NEXT: # ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11]3312; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload3313; AVX2-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm13 # 32-byte Folded Reload3314; AVX2-NEXT: # ymm13 = ymm7[0],mem[0],ymm7[1],mem[1],ymm7[2],mem[2],ymm7[3],mem[3],ymm7[8],mem[8],ymm7[9],mem[9],ymm7[10],mem[10],ymm7[11],mem[11]3315; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]3316; AVX2-NEXT: vpshufd {{.*#+}} ymm13 = ymm13[1,0,2,2,5,4,6,6]3317; AVX2-NEXT: vpermq {{.*#+}} ymm13 = ymm13[2,1,2,3]3318; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm13[1],ymm0[2,3],ymm13[4],ymm0[5,6],ymm13[7]3319; AVX2-NEXT: vpshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Folded Reload3320; AVX2-NEXT: # ymm11 = mem[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]3321; AVX2-NEXT: vpermq {{.*#+}} ymm11 = ymm11[2,2,2,2]3322; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm11[2],ymm0[3,4],ymm11[5],ymm0[6,7]3323; AVX2-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Folded Reload3324; AVX2-NEXT: # ymm9 = mem[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]3325; AVX2-NEXT: vpermq {{.*#+}} ymm9 = ymm9[2,2,2,2]3326; AVX2-NEXT: vpblendvb %ymm15, %ymm0, %ymm9, %ymm03327; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm4 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[8],ymm4[8],ymm3[9],ymm4[9],ymm3[10],ymm4[10],ymm3[11],ymm4[11]3328; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm7 = ymm2[0],ymm5[0],ymm2[1],ymm5[1],ymm2[2],ymm5[2],ymm2[3],ymm5[3],ymm2[8],ymm5[8],ymm2[9],ymm5[9],ymm2[10],ymm5[10],ymm2[11],ymm5[11]3329; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]3330; AVX2-NEXT: vpshufd {{.*#+}} ymm7 = ymm7[1,0,2,2,5,4,6,6]3331; AVX2-NEXT: vpermq {{.*#+}} ymm7 = ymm7[2,1,2,3]3332; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0],ymm7[1],ymm4[2,3],ymm7[4],ymm4[5,6],ymm7[7]3333; AVX2-NEXT: vpshuflw {{.*#+}} ymm5 = ymm8[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]3334; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm5[2,2,2,2]3335; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]3336; AVX2-NEXT: vpshuflw {{.*#+}} ymm3 = ymm10[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]3337; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,2]3338; AVX2-NEXT: vpblendvb %ymm15, %ymm4, %ymm3, %ymm23339; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax3340; AVX2-NEXT: vmovdqa %ymm2, 96(%rax)3341; AVX2-NEXT: vmovdqa %ymm6, 160(%rax)3342; AVX2-NEXT: vmovdqa %ymm0, 288(%rax)3343; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3344; AVX2-NEXT: vmovaps %ymm0, 352(%rax)3345; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3346; AVX2-NEXT: vmovaps %ymm0, 64(%rax)3347; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3348; AVX2-NEXT: vmovaps %ymm0, 128(%rax)3349; AVX2-NEXT: vmovdqa %ymm1, 192(%rax)3350; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3351; AVX2-NEXT: vmovaps %ymm0, 256(%rax)3352; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3353; AVX2-NEXT: vmovaps %ymm0, 320(%rax)3354; AVX2-NEXT: vmovdqa %ymm12, (%rax)3355; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3356; AVX2-NEXT: vmovaps %ymm0, 224(%rax)3357; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3358; AVX2-NEXT: vmovaps %ymm0, 32(%rax)3359; AVX2-NEXT: addq $616, %rsp # imm = 0x2683360; AVX2-NEXT: vzeroupper3361; AVX2-NEXT: retq3362;3363; AVX2-FP-LABEL: store_i16_stride6_vf32:3364; AVX2-FP: # %bb.0:3365; AVX2-FP-NEXT: subq $648, %rsp # imm = 0x2883366; AVX2-FP-NEXT: vmovdqa (%rsi), %xmm13367; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3368; AVX2-FP-NEXT: vmovdqa 32(%rsi), %xmm83369; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7,8,9,6,7,4,5,10,11]3370; AVX2-FP-NEXT: vpshufb %xmm0, %xmm1, %xmm13371; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm23372; AVX2-FP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3373; AVX2-FP-NEXT: vmovdqa 32(%rdi), %xmm53374; AVX2-FP-NEXT: vpshufb %xmm0, %xmm2, %xmm23375; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]3376; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]3377; AVX2-FP-NEXT: vmovdqa (%rcx), %xmm23378; AVX2-FP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3379; AVX2-FP-NEXT: vmovdqa 32(%rcx), %xmm63380; AVX2-FP-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero3381; AVX2-FP-NEXT: vmovdqa (%rdx), %xmm33382; AVX2-FP-NEXT: vmovdqa %xmm3, (%rsp) # 16-byte Spill3383; AVX2-FP-NEXT: vmovdqa 32(%rdx), %xmm73384; AVX2-FP-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero3385; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]3386; AVX2-FP-NEXT: vpbroadcastq %xmm2, %ymm23387; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]3388; AVX2-FP-NEXT: vmovdqa (%r8), %xmm23389; AVX2-FP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3390; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,1,3,3,4,5,6,7]3391; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]3392; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]3393; AVX2-FP-NEXT: vmovdqa (%r9), %xmm33394; AVX2-FP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3395; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]3396; AVX2-FP-NEXT: vpshufb %xmm1, %xmm3, %xmm33397; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]3398; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm10 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]3399; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm2, %ymm3, %ymm23400; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3401; AVX2-FP-NEXT: vpshufb %xmm0, %xmm8, %xmm23402; AVX2-FP-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3403; AVX2-FP-NEXT: vpshufb %xmm0, %xmm5, %xmm03404; AVX2-FP-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3405; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]3406; AVX2-FP-NEXT: vpsrldq {{.*#+}} xmm2 = xmm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero3407; AVX2-FP-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3408; AVX2-FP-NEXT: vpsrldq {{.*#+}} xmm3 = xmm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero3409; AVX2-FP-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3410; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]3411; AVX2-FP-NEXT: vmovdqa 32(%r8), %xmm93412; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]3413; AVX2-FP-NEXT: vpbroadcastq %xmm2, %ymm23414; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2],ymm0[3,4],ymm2[5],ymm0[6,7]3415; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm9[2,1,3,3,4,5,6,7]3416; AVX2-FP-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3417; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]3418; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7]3419; AVX2-FP-NEXT: vmovdqa 32(%r9), %xmm143420; AVX2-FP-NEXT: vpshufb %xmm1, %xmm14, %xmm13421; AVX2-FP-NEXT: vmovdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3422; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]3423; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm0, %ymm1, %ymm03424; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3425; AVX2-FP-NEXT: vmovdqa 32(%rdi), %ymm113426; AVX2-FP-NEXT: vmovdqa 32(%rsi), %ymm133427; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm0 = [u,u,u,u,4,5,10,11,u,u,u,u,u,u,u,u,24,25,22,23,20,21,26,27,u,u,u,u,u,u,u,u]3428; AVX2-FP-NEXT: vpshufb %ymm0, %ymm13, %ymm13429; AVX2-FP-NEXT: vpshufb %ymm0, %ymm11, %ymm23430; AVX2-FP-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3431; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]3432; AVX2-FP-NEXT: vmovdqa 32(%rdx), %ymm123433; AVX2-FP-NEXT: vmovdqa 32(%rcx), %ymm23434; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3435; AVX2-FP-NEXT: vpsrldq {{.*#+}} ymm2 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero3436; AVX2-FP-NEXT: vpsrldq {{.*#+}} ymm3 = ymm12[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm12[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero3437; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]3438; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]3439; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]3440; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]3441; AVX2-FP-NEXT: vmovdqa 32(%r8), %ymm23442; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3443; AVX2-FP-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]3444; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]3445; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]3446; AVX2-FP-NEXT: vmovdqa 32(%r9), %ymm153447; AVX2-FP-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25,16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25]3448; AVX2-FP-NEXT: # ymm4 = mem[0,1,0,1]3449; AVX2-FP-NEXT: vpshufb %ymm4, %ymm15, %ymm23450; AVX2-FP-NEXT: vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3451; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]3452; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm1, %ymm2, %ymm13453; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3454; AVX2-FP-NEXT: vmovdqa (%rdi), %ymm23455; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3456; AVX2-FP-NEXT: vmovdqa (%rsi), %ymm13457; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3458; AVX2-FP-NEXT: vpshufb %ymm0, %ymm1, %ymm13459; AVX2-FP-NEXT: vpshufb %ymm0, %ymm2, %ymm03460; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]3461; AVX2-FP-NEXT: vmovdqa (%rdx), %ymm33462; AVX2-FP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3463; AVX2-FP-NEXT: vmovdqa (%rcx), %ymm13464; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3465; AVX2-FP-NEXT: vpsrldq {{.*#+}} ymm1 = ymm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm1[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero3466; AVX2-FP-NEXT: vpsrldq {{.*#+}} ymm2 = ymm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm3[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero3467; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]3468; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,1,2,3]3469; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]3470; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]3471; AVX2-FP-NEXT: vmovdqa (%r8), %ymm13472; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3473; AVX2-FP-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]3474; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]3475; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7]3476; AVX2-FP-NEXT: vmovdqa (%r9), %ymm13477; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3478; AVX2-FP-NEXT: vpshufb %ymm4, %ymm1, %ymm13479; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]3480; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm0, %ymm1, %ymm03481; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3482; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm5[4],xmm8[4],xmm5[5],xmm8[5],xmm5[6],xmm8[6],xmm5[7],xmm8[7]3483; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,1,1,1]3484; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]3485; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,2,3,3]3486; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]3487; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]3488; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm1 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]3489; AVX2-FP-NEXT: vpshufb %xmm1, %xmm9, %xmm43490; AVX2-FP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,1,0,1]3491; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm4[1],ymm0[2,3],ymm4[4],ymm0[5,6],ymm4[7]3492; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm7 = [8,9,12,13,12,13,10,11,8,9,10,11,12,13,14,15]3493; AVX2-FP-NEXT: vpshufb %xmm7, %xmm14, %xmm43494; AVX2-FP-NEXT: vpermq {{.*#+}} ymm14 = ymm4[0,1,0,1]3495; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm9 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]3496; AVX2-FP-NEXT: vpblendvb %ymm9, %ymm0, %ymm14, %ymm03497; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3498; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload3499; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload3500; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]3501; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,1,1,1]3502; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload3503; AVX2-FP-NEXT: vmovdqa (%rsp), %xmm2 # 16-byte Reload3504; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm14 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]3505; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm14 = xmm14[1,2,3,3]3506; AVX2-FP-NEXT: vpermq {{.*#+}} ymm14 = ymm14[0,0,2,1]3507; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm14[0,1],ymm0[2],ymm14[3,4],ymm0[5],ymm14[6,7]3508; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload3509; AVX2-FP-NEXT: vpshufb %xmm1, %xmm6, %xmm13510; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]3511; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]3512; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload3513; AVX2-FP-NEXT: vpshufb %xmm7, %xmm8, %xmm13514; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]3515; AVX2-FP-NEXT: vpblendvb %ymm9, %ymm0, %ymm1, %ymm03516; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3517; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm11[4],ymm13[4],ymm11[5],ymm13[5],ymm11[6],ymm13[6],ymm11[7],ymm13[7],ymm11[12],ymm13[12],ymm11[13],ymm13[13],ymm11[14],ymm13[14],ymm11[15],ymm13[15]3518; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,3,3,3]3519; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload3520; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm12[4],ymm10[4],ymm12[5],ymm10[5],ymm12[6],ymm10[6],ymm12[7],ymm10[7],ymm12[12],ymm10[12],ymm12[13],ymm10[13],ymm12[14],ymm10[14],ymm12[15],ymm10[15]3521; AVX2-FP-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[1,2,3,3,5,6,7,7]3522; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]3523; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]3524; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm1 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]3525; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload3526; AVX2-FP-NEXT: vpshufb %ymm1, %ymm11, %ymm73527; AVX2-FP-NEXT: vpermq {{.*#+}} ymm7 = ymm7[2,1,2,3]3528; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm7[1],ymm0[2,3],ymm7[4],ymm0[5,6],ymm7[7]3529; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm14 = ymm15[u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,24,25,28,29,28,29,26,27,24,25,26,27,28,29,30,31]3530; AVX2-FP-NEXT: vpermq {{.*#+}} ymm14 = ymm14[2,1,2,3]3531; AVX2-FP-NEXT: vpblendvb %ymm9, %ymm0, %ymm14, %ymm03532; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3533; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload3534; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload3535; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm7[4],ymm9[4],ymm7[5],ymm9[5],ymm7[6],ymm9[6],ymm7[7],ymm9[7],ymm7[12],ymm9[12],ymm7[13],ymm9[13],ymm7[14],ymm9[14],ymm7[15],ymm9[15]3536; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,3,3,3]3537; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload3538; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload3539; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} ymm15 = ymm15[4],ymm14[4],ymm15[5],ymm14[5],ymm15[6],ymm14[6],ymm15[7],ymm14[7],ymm15[12],ymm14[12],ymm15[13],ymm14[13],ymm15[14],ymm14[14],ymm15[15],ymm14[15]3540; AVX2-FP-NEXT: vpshufd {{.*#+}} ymm15 = ymm15[1,2,3,3,5,6,7,7]3541; AVX2-FP-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,2,2,3]3542; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm15[0,1],ymm0[2],ymm15[3,4],ymm0[5],ymm15[6,7]3543; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload3544; AVX2-FP-NEXT: vpshufb %ymm1, %ymm15, %ymm13545; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]3546; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]3547; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload3548; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,24,25,28,29,28,29,26,27,24,25,26,27,28,29,30,31]3549; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]3550; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm15 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]3551; AVX2-FP-NEXT: vpblendvb %ymm15, %ymm0, %ymm1, %ymm03552; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3553; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]3554; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]3555; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]3556; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,0,2,2]3557; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]3558; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]3559; AVX2-FP-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero3560; AVX2-FP-NEXT: vpbroadcastq %xmm1, %ymm13561; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]3562; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm8[0,0,2,1,4,5,6,7]3563; AVX2-FP-NEXT: vpbroadcastq %xmm0, %ymm43564; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm0 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]3565; AVX2-FP-NEXT: vpblendvb %ymm0, %ymm1, %ymm4, %ymm43566; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3567; AVX2-FP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload3568; AVX2-FP-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]3569; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload3570; AVX2-FP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm15 # 16-byte Folded Reload3571; AVX2-FP-NEXT: # xmm15 = xmm2[0],mem[0],xmm2[1],mem[1],xmm2[2],mem[2],xmm2[3],mem[3]3572; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]3573; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm15 = xmm15[1,0,2,2]3574; AVX2-FP-NEXT: vpermq {{.*#+}} ymm15 = ymm15[0,1,0,1]3575; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm15[1],ymm1[2,3],ymm15[4],ymm1[5,6],ymm15[7]3576; AVX2-FP-NEXT: vpmovzxwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload3577; AVX2-FP-NEXT: # xmm15 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero3578; AVX2-FP-NEXT: vpbroadcastq %xmm15, %ymm153579; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm15[2],ymm1[3,4],ymm15[5],ymm1[6,7]3580; AVX2-FP-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload3581; AVX2-FP-NEXT: # xmm15 = mem[0,0,2,1,4,5,6,7]3582; AVX2-FP-NEXT: vpbroadcastq %xmm15, %ymm153583; AVX2-FP-NEXT: vpblendvb %ymm0, %ymm1, %ymm15, %ymm13584; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload3585; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm15 = ymm2[0],ymm13[0],ymm2[1],ymm13[1],ymm2[2],ymm13[2],ymm2[3],ymm13[3],ymm2[8],ymm13[8],ymm2[9],ymm13[9],ymm2[10],ymm13[10],ymm2[11],ymm13[11]3586; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm12 = ymm12[0],ymm10[0],ymm12[1],ymm10[1],ymm12[2],ymm10[2],ymm12[3],ymm10[3],ymm12[8],ymm10[8],ymm12[9],ymm10[9],ymm12[10],ymm10[10],ymm12[11],ymm10[11]3587; AVX2-FP-NEXT: vpermq {{.*#+}} ymm13 = ymm15[2,2,2,3]3588; AVX2-FP-NEXT: vpshufd {{.*#+}} ymm12 = ymm12[1,0,2,2,5,4,6,6]3589; AVX2-FP-NEXT: vpermq {{.*#+}} ymm12 = ymm12[2,1,2,3]3590; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm12 = ymm13[0],ymm12[1],ymm13[2,3],ymm12[4],ymm13[5,6],ymm12[7]3591; AVX2-FP-NEXT: vpshuflw {{.*#+}} ymm11 = ymm11[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]3592; AVX2-FP-NEXT: vpermq {{.*#+}} ymm11 = ymm11[2,2,2,2]3593; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1],ymm11[2],ymm12[3,4],ymm11[5],ymm12[6,7]3594; AVX2-FP-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload3595; AVX2-FP-NEXT: # ymm10 = mem[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]3596; AVX2-FP-NEXT: vpermq {{.*#+}} ymm10 = ymm10[2,2,2,2]3597; AVX2-FP-NEXT: vpblendvb %ymm0, %ymm11, %ymm10, %ymm103598; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm8 = ymm7[0],ymm9[0],ymm7[1],ymm9[1],ymm7[2],ymm9[2],ymm7[3],ymm9[3],ymm7[8],ymm9[8],ymm7[9],ymm9[9],ymm7[10],ymm9[10],ymm7[11],ymm9[11]3599; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload3600; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm5 = ymm2[0],ymm14[0],ymm2[1],ymm14[1],ymm2[2],ymm14[2],ymm2[3],ymm14[3],ymm2[8],ymm14[8],ymm2[9],ymm14[9],ymm2[10],ymm14[10],ymm2[11],ymm14[11]3601; AVX2-FP-NEXT: vpermq {{.*#+}} ymm6 = ymm8[2,2,2,3]3602; AVX2-FP-NEXT: vpshufd {{.*#+}} ymm5 = ymm5[1,0,2,2,5,4,6,6]3603; AVX2-FP-NEXT: vpermq {{.*#+}} ymm5 = ymm5[2,1,2,3]3604; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2,3],ymm5[4],ymm6[5,6],ymm5[7]3605; AVX2-FP-NEXT: vpshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload3606; AVX2-FP-NEXT: # ymm3 = mem[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]3607; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,2]3608; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0,1],ymm3[2],ymm5[3,4],ymm3[5],ymm5[6,7]3609; AVX2-FP-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload3610; AVX2-FP-NEXT: # ymm2 = mem[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]3611; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]3612; AVX2-FP-NEXT: vpblendvb %ymm0, %ymm3, %ymm2, %ymm03613; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %rax3614; AVX2-FP-NEXT: vmovdqa %ymm0, 96(%rax)3615; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3616; AVX2-FP-NEXT: vmovaps %ymm0, 160(%rax)3617; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3618; AVX2-FP-NEXT: vmovaps %ymm0, 128(%rax)3619; AVX2-FP-NEXT: vmovdqa %ymm10, 288(%rax)3620; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3621; AVX2-FP-NEXT: vmovaps %ymm0, 352(%rax)3622; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3623; AVX2-FP-NEXT: vmovaps %ymm0, 320(%rax)3624; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3625; AVX2-FP-NEXT: vmovaps %ymm0, 64(%rax)3626; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3627; AVX2-FP-NEXT: vmovaps %ymm0, 224(%rax)3628; AVX2-FP-NEXT: vmovdqa %ymm1, 192(%rax)3629; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3630; AVX2-FP-NEXT: vmovaps %ymm0, 256(%rax)3631; AVX2-FP-NEXT: vmovdqa %ymm4, (%rax)3632; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3633; AVX2-FP-NEXT: vmovaps %ymm0, 32(%rax)3634; AVX2-FP-NEXT: addq $648, %rsp # imm = 0x2883635; AVX2-FP-NEXT: vzeroupper3636; AVX2-FP-NEXT: retq3637;3638; AVX2-FCP-LABEL: store_i16_stride6_vf32:3639; AVX2-FCP: # %bb.0:3640; AVX2-FCP-NEXT: subq $648, %rsp # imm = 0x2883641; AVX2-FCP-NEXT: vmovdqa (%rsi), %xmm13642; AVX2-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3643; AVX2-FCP-NEXT: vmovdqa 32(%rsi), %xmm143644; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7,8,9,6,7,4,5,10,11]3645; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm1, %xmm13646; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm23647; AVX2-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3648; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %xmm123649; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm2, %xmm23650; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]3651; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]3652; AVX2-FCP-NEXT: vmovdqa (%rcx), %xmm23653; AVX2-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3654; AVX2-FCP-NEXT: vmovdqa 32(%rcx), %xmm63655; AVX2-FCP-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero3656; AVX2-FCP-NEXT: vmovdqa (%rdx), %xmm33657; AVX2-FCP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3658; AVX2-FCP-NEXT: vmovdqa 32(%rdx), %xmm73659; AVX2-FCP-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero3660; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]3661; AVX2-FCP-NEXT: vpbroadcastq %xmm2, %ymm23662; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]3663; AVX2-FCP-NEXT: vmovdqa (%r8), %xmm23664; AVX2-FCP-NEXT: vmovdqa %xmm2, (%rsp) # 16-byte Spill3665; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,1,3,3,4,5,6,7]3666; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]3667; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]3668; AVX2-FCP-NEXT: vmovdqa (%r9), %xmm33669; AVX2-FCP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3670; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]3671; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm3, %xmm33672; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]3673; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm13 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]3674; AVX2-FCP-NEXT: vpblendvb %ymm13, %ymm1, %ymm3, %ymm13675; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3676; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm14, %xmm13677; AVX2-FCP-NEXT: vmovdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3678; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm12, %xmm03679; AVX2-FCP-NEXT: vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3680; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]3681; AVX2-FCP-NEXT: vpsrldq {{.*#+}} xmm1 = xmm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero3682; AVX2-FCP-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3683; AVX2-FCP-NEXT: vpsrldq {{.*#+}} xmm3 = xmm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero3684; AVX2-FCP-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3685; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]3686; AVX2-FCP-NEXT: vmovdqa 32(%r8), %xmm83687; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]3688; AVX2-FCP-NEXT: vpbroadcastq %xmm1, %ymm13689; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]3690; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm8[2,1,3,3,4,5,6,7]3691; AVX2-FCP-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3692; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]3693; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7]3694; AVX2-FCP-NEXT: vmovdqa 32(%r9), %xmm93695; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm9, %xmm13696; AVX2-FCP-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3697; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]3698; AVX2-FCP-NEXT: vpblendvb %ymm13, %ymm0, %ymm1, %ymm03699; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3700; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %ymm23701; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3702; AVX2-FCP-NEXT: vmovdqa 32(%rsi), %ymm13703; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3704; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm0 = [u,u,u,u,4,5,10,11,u,u,u,u,u,u,u,u,24,25,22,23,20,21,26,27,u,u,u,u,u,u,u,u]3705; AVX2-FCP-NEXT: vpshufb %ymm0, %ymm1, %ymm13706; AVX2-FCP-NEXT: vpshufb %ymm0, %ymm2, %ymm23707; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]3708; AVX2-FCP-NEXT: vmovdqa 32(%rdx), %ymm103709; AVX2-FCP-NEXT: vmovdqa 32(%rcx), %ymm43710; AVX2-FCP-NEXT: vpsrldq {{.*#+}} ymm2 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero3711; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3712; AVX2-FCP-NEXT: vpsrldq {{.*#+}} ymm3 = ymm10[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm10[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero3713; AVX2-FCP-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3714; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]3715; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]3716; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]3717; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]3718; AVX2-FCP-NEXT: vmovdqa 32(%r8), %ymm23719; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3720; AVX2-FCP-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]3721; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]3722; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]3723; AVX2-FCP-NEXT: vmovdqa 32(%r9), %ymm33724; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3725; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25,16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25]3726; AVX2-FCP-NEXT: # ymm2 = mem[0,1,0,1]3727; AVX2-FCP-NEXT: vpshufb %ymm2, %ymm3, %ymm33728; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]3729; AVX2-FCP-NEXT: vpblendvb %ymm13, %ymm1, %ymm3, %ymm13730; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3731; AVX2-FCP-NEXT: vmovdqa (%rdi), %ymm33732; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3733; AVX2-FCP-NEXT: vmovdqa (%rsi), %ymm13734; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3735; AVX2-FCP-NEXT: vpshufb %ymm0, %ymm1, %ymm13736; AVX2-FCP-NEXT: vpshufb %ymm0, %ymm3, %ymm03737; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]3738; AVX2-FCP-NEXT: vmovdqa (%rdx), %ymm53739; AVX2-FCP-NEXT: vmovdqa (%rcx), %ymm13740; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3741; AVX2-FCP-NEXT: vpsrldq {{.*#+}} ymm1 = ymm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm1[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero3742; AVX2-FCP-NEXT: vpsrldq {{.*#+}} ymm3 = ymm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm5[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero3743; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm3[0],ymm1[0],ymm3[1],ymm1[1],ymm3[2],ymm1[2],ymm3[3],ymm1[3],ymm3[8],ymm1[8],ymm3[9],ymm1[9],ymm3[10],ymm1[10],ymm3[11],ymm1[11]3744; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,1,2,3]3745; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]3746; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]3747; AVX2-FCP-NEXT: vmovdqa (%r8), %ymm13748; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3749; AVX2-FCP-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]3750; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]3751; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7]3752; AVX2-FCP-NEXT: vmovdqa (%r9), %ymm13753; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3754; AVX2-FCP-NEXT: vpshufb %ymm2, %ymm1, %ymm23755; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]3756; AVX2-FCP-NEXT: vpblendvb %ymm13, %ymm0, %ymm2, %ymm03757; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3758; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]3759; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm2 = [1,2,1,2,0,0,3,3]3760; AVX2-FCP-NEXT: vpermd %ymm0, %ymm2, %ymm03761; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} xmm6 = xmm12[4],xmm14[4],xmm12[5],xmm14[5],xmm12[6],xmm14[6],xmm12[7],xmm14[7]3762; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm6 = ymm6[1,1,1,1]3763; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm6[2],ymm0[3,4],ymm6[5],ymm0[6,7]3764; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm6 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]3765; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm8, %xmm133766; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm13 = ymm13[0,1,0,1]3767; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm13 = ymm0[0],ymm13[1],ymm0[2,3],ymm13[4],ymm0[5,6],ymm13[7]3768; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm3 = [8,9,12,13,12,13,10,11,8,9,10,11,12,13,14,15]3769; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm9, %xmm13770; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm1[0,1,0,1]3771; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm8 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]3772; AVX2-FCP-NEXT: vpblendvb %ymm8, %ymm13, %ymm0, %ymm03773; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3774; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload3775; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload3776; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm14[4],xmm15[4],xmm14[5],xmm15[5],xmm14[6],xmm15[6],xmm14[7],xmm15[7]3777; AVX2-FCP-NEXT: vpermd %ymm0, %ymm2, %ymm03778; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload3779; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3780; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm7[4],xmm1[5],xmm7[5],xmm1[6],xmm7[6],xmm1[7],xmm7[7]3781; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[1,1,1,1]3782; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2],ymm0[3,4],ymm2[5],ymm0[6,7]3783; AVX2-FCP-NEXT: vmovdqa (%rsp), %xmm12 # 16-byte Reload3784; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm12, %xmm23785; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]3786; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6],ymm2[7]3787; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload3788; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm11, %xmm23789; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]3790; AVX2-FCP-NEXT: vpblendvb %ymm8, %ymm0, %ymm2, %ymm03791; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3792; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm10[4],ymm4[4],ymm10[5],ymm4[5],ymm10[6],ymm4[6],ymm10[7],ymm4[7],ymm10[12],ymm4[12],ymm10[13],ymm4[13],ymm10[14],ymm4[14],ymm10[15],ymm4[15]3793; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm2 = [5,6,5,6,5,6,7,7]3794; AVX2-FCP-NEXT: vpermd %ymm0, %ymm2, %ymm03795; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload3796; AVX2-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload3797; AVX2-FCP-NEXT: # ymm3 = ymm3[4],mem[4],ymm3[5],mem[5],ymm3[6],mem[6],ymm3[7],mem[7],ymm3[12],mem[12],ymm3[13],mem[13],ymm3[14],mem[14],ymm3[15],mem[15]3798; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[3,3,3,3]3799; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm3[2],ymm0[3,4],ymm3[5],ymm0[6,7]3800; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm10 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]3801; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload3802; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm4, %ymm63803; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm6 = ymm6[2,1,2,3]3804; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm6[1],ymm0[2,3],ymm6[4],ymm0[5,6],ymm6[7]3805; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm3 = [u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,24,25,28,29,28,29,26,27,24,25,26,27,28,29,30,31]3806; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload3807; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm4, %ymm133808; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm13 = ymm13[2,1,2,3]3809; AVX2-FCP-NEXT: vpblendvb %ymm8, %ymm0, %ymm13, %ymm03810; AVX2-FCP-NEXT: vmovdqa %ymm8, %ymm133811; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3812; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload3813; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm5[4],ymm6[4],ymm5[5],ymm6[5],ymm5[6],ymm6[6],ymm5[7],ymm6[7],ymm5[12],ymm6[12],ymm5[13],ymm6[13],ymm5[14],ymm6[14],ymm5[15],ymm6[15]3814; AVX2-FCP-NEXT: vpermd %ymm0, %ymm2, %ymm03815; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload3816; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload3817; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm4[4],ymm8[4],ymm4[5],ymm8[5],ymm4[6],ymm8[6],ymm4[7],ymm8[7],ymm4[12],ymm8[12],ymm4[13],ymm8[13],ymm4[14],ymm8[14],ymm4[15],ymm8[15]3818; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[3,3,3,3]3819; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2],ymm0[3,4],ymm2[5],ymm0[6,7]3820; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload3821; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm9, %ymm23822; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]3823; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6],ymm2[7]3824; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload3825; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm10, %ymm23826; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]3827; AVX2-FCP-NEXT: vpblendvb %ymm13, %ymm0, %ymm2, %ymm133828; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]3829; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm7[0],xmm1[1],xmm7[1],xmm1[2],xmm7[2],xmm1[3],xmm7[3]3830; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [1,0,2,2,1,0,2,2]3831; AVX2-FCP-NEXT: # ymm3 = mem[0,1,0,1]3832; AVX2-FCP-NEXT: vpermd %ymm0, %ymm3, %ymm03833; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]3834; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]3835; AVX2-FCP-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm12[0],zero,xmm12[1],zero,xmm12[2],zero,xmm12[3],zero3836; AVX2-FCP-NEXT: vpbroadcastq %xmm1, %ymm13837; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]3838; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm11[0,0,2,1,4,5,6,7]3839; AVX2-FCP-NEXT: vpbroadcastq %xmm0, %ymm23840; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm0 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]3841; AVX2-FCP-NEXT: vpblendvb %ymm0, %ymm1, %ymm2, %ymm23842; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3843; AVX2-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload3844; AVX2-FCP-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]3845; AVX2-FCP-NEXT: vpermd %ymm1, %ymm3, %ymm13846; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload3847; AVX2-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload3848; AVX2-FCP-NEXT: # xmm3 = xmm3[0],mem[0],xmm3[1],mem[1],xmm3[2],mem[2],xmm3[3],mem[3]3849; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]3850; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0],ymm1[1],ymm3[2,3],ymm1[4],ymm3[5,6],ymm1[7]3851; AVX2-FCP-NEXT: vpmovzxwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload3852; AVX2-FCP-NEXT: # xmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero3853; AVX2-FCP-NEXT: vpbroadcastq %xmm3, %ymm33854; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm3[2],ymm1[3,4],ymm3[5],ymm1[6,7]3855; AVX2-FCP-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload3856; AVX2-FCP-NEXT: # xmm3 = mem[0,0,2,1,4,5,6,7]3857; AVX2-FCP-NEXT: vpbroadcastq %xmm3, %ymm33858; AVX2-FCP-NEXT: vpblendvb %ymm0, %ymm1, %ymm3, %ymm13859; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload3860; AVX2-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload3861; AVX2-FCP-NEXT: # ymm3 = ymm3[0],mem[0],ymm3[1],mem[1],ymm3[2],mem[2],ymm3[3],mem[3],ymm3[8],mem[8],ymm3[9],mem[9],ymm3[10],mem[10],ymm3[11],mem[11]3862; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload3863; AVX2-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm14 # 32-byte Folded Reload3864; AVX2-FCP-NEXT: # ymm14 = ymm7[0],mem[0],ymm7[1],mem[1],ymm7[2],mem[2],ymm7[3],mem[3],ymm7[8],mem[8],ymm7[9],mem[9],ymm7[10],mem[10],ymm7[11],mem[11]3865; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm15 = [5,4,2,2,5,4,6,6]3866; AVX2-FCP-NEXT: vpermd %ymm3, %ymm15, %ymm33867; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm14 = ymm14[2,2,2,3]3868; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm14[0],ymm3[1],ymm14[2,3],ymm3[4],ymm14[5,6],ymm3[7]3869; AVX2-FCP-NEXT: vpshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Folded Reload3870; AVX2-FCP-NEXT: # ymm12 = mem[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]3871; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm12 = ymm12[2,2,2,2]3872; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm12[2],ymm3[3,4],ymm12[5],ymm3[6,7]3873; AVX2-FCP-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Folded Reload3874; AVX2-FCP-NEXT: # ymm11 = mem[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]3875; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm11 = ymm11[2,2,2,2]3876; AVX2-FCP-NEXT: vpblendvb %ymm0, %ymm3, %ymm11, %ymm33877; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm7 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[2],ymm6[2],ymm5[3],ymm6[3],ymm5[8],ymm6[8],ymm5[9],ymm6[9],ymm5[10],ymm6[10],ymm5[11],ymm6[11]3878; AVX2-FCP-NEXT: vpermd %ymm7, %ymm15, %ymm73879; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm8 = ymm4[0],ymm8[0],ymm4[1],ymm8[1],ymm4[2],ymm8[2],ymm4[3],ymm8[3],ymm4[8],ymm8[8],ymm4[9],ymm8[9],ymm4[10],ymm8[10],ymm4[11],ymm8[11]3880; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm8 = ymm8[2,2,2,3]3881; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0],ymm7[1],ymm8[2,3],ymm7[4],ymm8[5,6],ymm7[7]3882; AVX2-FCP-NEXT: vpshuflw {{.*#+}} ymm5 = ymm9[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]3883; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm5 = ymm5[2,2,2,2]3884; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0,1],ymm5[2],ymm7[3,4],ymm5[5],ymm7[6,7]3885; AVX2-FCP-NEXT: vpshuflw {{.*#+}} ymm4 = ymm10[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]3886; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,2,2,2]3887; AVX2-FCP-NEXT: vpblendvb %ymm0, %ymm5, %ymm4, %ymm03888; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax3889; AVX2-FCP-NEXT: vmovdqa %ymm0, 96(%rax)3890; AVX2-FCP-NEXT: vmovdqa %ymm13, 160(%rax)3891; AVX2-FCP-NEXT: vmovdqa %ymm3, 288(%rax)3892; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3893; AVX2-FCP-NEXT: vmovaps %ymm0, 352(%rax)3894; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3895; AVX2-FCP-NEXT: vmovaps %ymm0, 64(%rax)3896; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3897; AVX2-FCP-NEXT: vmovaps %ymm0, 128(%rax)3898; AVX2-FCP-NEXT: vmovdqa %ymm1, 192(%rax)3899; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3900; AVX2-FCP-NEXT: vmovaps %ymm0, 256(%rax)3901; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3902; AVX2-FCP-NEXT: vmovaps %ymm0, 320(%rax)3903; AVX2-FCP-NEXT: vmovdqa %ymm2, (%rax)3904; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3905; AVX2-FCP-NEXT: vmovaps %ymm0, 224(%rax)3906; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3907; AVX2-FCP-NEXT: vmovaps %ymm0, 32(%rax)3908; AVX2-FCP-NEXT: addq $648, %rsp # imm = 0x2883909; AVX2-FCP-NEXT: vzeroupper3910; AVX2-FCP-NEXT: retq3911;3912; AVX512-LABEL: store_i16_stride6_vf32:3913; AVX512: # %bb.0:3914; AVX512-NEXT: subq $40, %rsp3915; AVX512-NEXT: vmovdqa 32(%rcx), %ymm53916; AVX512-NEXT: vpsrldq {{.*#+}} ymm0 = ymm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm5[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero3917; AVX512-NEXT: vmovdqa 32(%rdx), %ymm23918; AVX512-NEXT: vpsrldq {{.*#+}} ymm1 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero3919; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]3920; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]3921; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3922; AVX512-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm2[4],ymm5[4],ymm2[5],ymm5[5],ymm2[6],ymm5[6],ymm2[7],ymm5[7],ymm2[12],ymm5[12],ymm2[13],ymm5[13],ymm2[14],ymm5[14],ymm2[15],ymm5[15]3923; AVX512-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[1,2,3,3,5,6,7,7]3924; AVX512-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3925; AVX512-NEXT: vmovdqa 32(%rsi), %ymm153926; AVX512-NEXT: vpshufd {{.*#+}} ymm0 = ymm15[2,1,2,3,6,5,6,7]3927; AVX512-NEXT: vpshuflw {{.*#+}} ymm0 = ymm0[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]3928; AVX512-NEXT: vmovdqa 32(%rdi), %ymm73929; AVX512-NEXT: vpshufd {{.*#+}} ymm1 = ymm7[2,1,2,3,6,5,6,7]3930; AVX512-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]3931; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]3932; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,1,2,3]3933; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3934; AVX512-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm7[4],ymm15[4],ymm7[5],ymm15[5],ymm7[6],ymm15[6],ymm7[7],ymm15[7],ymm7[12],ymm15[12],ymm7[13],ymm15[13],ymm7[14],ymm15[14],ymm7[15],ymm15[15]3935; AVX512-NEXT: vpermq {{.*#+}} ymm19 = ymm0[3,3,3,3]3936; AVX512-NEXT: vmovdqa (%rcx), %ymm03937; AVX512-NEXT: vpsrldq {{.*#+}} ymm1 = ymm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm0[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero3938; AVX512-NEXT: vmovdqa (%rdx), %ymm63939; AVX512-NEXT: vpsrldq {{.*#+}} ymm3 = ymm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm6[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero3940; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm3[0],ymm1[0],ymm3[1],ymm1[1],ymm3[2],ymm1[2],ymm3[3],ymm1[3],ymm3[8],ymm1[8],ymm3[9],ymm1[9],ymm3[10],ymm1[10],ymm3[11],ymm1[11]3941; AVX512-NEXT: vmovdqa64 %ymm1, %ymm293942; AVX512-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm6[4],ymm0[4],ymm6[5],ymm0[5],ymm6[6],ymm0[6],ymm6[7],ymm0[7],ymm6[12],ymm0[12],ymm6[13],ymm0[13],ymm6[14],ymm0[14],ymm6[15],ymm0[15]3943; AVX512-NEXT: vmovdqa64 %ymm1, %ymm313944; AVX512-NEXT: vmovdqa (%rsi), %ymm33945; AVX512-NEXT: vpshufd {{.*#+}} ymm1 = ymm3[2,1,2,3,6,5,6,7]3946; AVX512-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]3947; AVX512-NEXT: vmovdqa (%rdi), %ymm43948; AVX512-NEXT: vpshufd {{.*#+}} ymm8 = ymm4[2,1,2,3,6,5,6,7]3949; AVX512-NEXT: vpshuflw {{.*#+}} ymm8 = ymm8[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]3950; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm8[0],ymm1[0],ymm8[1],ymm1[1],ymm8[2],ymm1[2],ymm8[3],ymm1[3],ymm8[8],ymm1[8],ymm8[9],ymm1[9],ymm8[10],ymm1[10],ymm8[11],ymm1[11]3951; AVX512-NEXT: vmovdqa64 %ymm1, %ymm183952; AVX512-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]3953; AVX512-NEXT: vmovdqa64 %ymm1, %ymm163954; AVX512-NEXT: vmovdqa (%rcx), %xmm113955; AVX512-NEXT: vmovdqa (%rdx), %xmm123956; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7]3957; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm6[0],ymm0[0],ymm6[1],ymm0[1],ymm6[2],ymm0[2],ymm6[3],ymm0[3],ymm6[8],ymm0[8],ymm6[9],ymm0[9],ymm6[10],ymm0[10],ymm6[11],ymm0[11]3958; AVX512-NEXT: vpmovsxbd {{.*#+}} zmm21 = [17,18,17,18,0,0,19,19,5,4,2,2,5,4,6,6]3959; AVX512-NEXT: vpermt2d %zmm1, %zmm21, %zmm03960; AVX512-NEXT: vmovdqa (%rsi), %xmm63961; AVX512-NEXT: vmovdqa (%rdi), %xmm93962; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm9[4],xmm6[4],xmm9[5],xmm6[5],xmm9[6],xmm6[6],xmm9[7],xmm6[7]3963; AVX512-NEXT: vmovdqa64 %xmm6, %xmm173964; AVX512-NEXT: vpermq {{.*#+}} ymm1 = ymm1[1,1,1,1]3965; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]3966; AVX512-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]3967; AVX512-NEXT: vmovdqa (%r8), %xmm63968; AVX512-NEXT: vmovdqa {{.*#+}} xmm13 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]3969; AVX512-NEXT: vpshufb %xmm13, %xmm6, %xmm43970; AVX512-NEXT: vmovdqa64 %xmm6, %xmm303971; AVX512-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,1,0,1]3972; AVX512-NEXT: vmovdqa (%r8), %ymm63973; AVX512-NEXT: vpshuflw {{.*#+}} ymm8 = ymm6[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]3974; AVX512-NEXT: vpermq {{.*#+}} ymm20 = ymm8[2,2,2,2]3975; AVX512-NEXT: vmovdqa (%r9), %xmm103976; AVX512-NEXT: vpshufd {{.*#+}} xmm8 = xmm10[2,3,2,3]3977; AVX512-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,2,2,1,4,5,6,7]3978; AVX512-NEXT: vpermq {{.*#+}} ymm22 = ymm8[0,1,0,1]3979; AVX512-NEXT: vmovdqa (%r9), %ymm83980; AVX512-NEXT: vpshuflw {{.*#+}} ymm14 = ymm8[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]3981; AVX512-NEXT: vpermq {{.*#+}} ymm14 = ymm14[2,2,2,2]3982; AVX512-NEXT: movw $18724, %ax # imm = 0x49243983; AVX512-NEXT: kmovw %eax, %k13984; AVX512-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm13985; AVX512-NEXT: vmovdqa32 %zmm1, %zmm0 {%k1}3986; AVX512-NEXT: vinserti64x4 $1, %ymm20, %zmm4, %zmm13987; AVX512-NEXT: movw $9362, %ax # imm = 0x24923988; AVX512-NEXT: kmovw %eax, %k23989; AVX512-NEXT: vmovdqa32 %zmm1, %zmm0 {%k2}3990; AVX512-NEXT: vinserti64x4 $1, %ymm14, %zmm22, %zmm203991; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm22 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]3992; AVX512-NEXT: vpternlogq {{.*#+}} zmm20 = zmm20 ^ (zmm22 & (zmm20 ^ zmm0))3993; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm2[0],ymm5[0],ymm2[1],ymm5[1],ymm2[2],ymm5[2],ymm2[3],ymm5[3],ymm2[8],ymm5[8],ymm2[9],ymm5[9],ymm2[10],ymm5[10],ymm2[11],ymm5[11]3994; AVX512-NEXT: vmovdqa 32(%rcx), %xmm33995; AVX512-NEXT: vmovdqa 32(%rdx), %xmm23996; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]3997; AVX512-NEXT: vpermt2d %zmm1, %zmm21, %zmm03998; AVX512-NEXT: vmovdqa 32(%rsi), %xmm43999; AVX512-NEXT: vmovdqa 32(%rdi), %xmm54000; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]4001; AVX512-NEXT: vpermq {{.*#+}} ymm21 = ymm1[1,1,1,1]4002; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm7[0],ymm15[0],ymm7[1],ymm15[1],ymm7[2],ymm15[2],ymm7[3],ymm15[3],ymm7[8],ymm15[8],ymm7[9],ymm15[9],ymm7[10],ymm15[10],ymm7[11],ymm15[11]4003; AVX512-NEXT: vpermq {{.*#+}} ymm23 = ymm1[2,2,2,3]4004; AVX512-NEXT: vmovdqa 32(%r8), %xmm74005; AVX512-NEXT: vpshufb %xmm13, %xmm7, %xmm14006; AVX512-NEXT: vpermq {{.*#+}} ymm24 = ymm1[0,1,0,1]4007; AVX512-NEXT: vmovdqa 32(%r8), %ymm144008; AVX512-NEXT: vpshuflw {{.*#+}} ymm1 = ymm14[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]4009; AVX512-NEXT: vpermq {{.*#+}} ymm25 = ymm1[2,2,2,2]4010; AVX512-NEXT: vmovdqa 32(%r9), %xmm154011; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm15[2,3,2,3]4012; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,1,4,5,6,7]4013; AVX512-NEXT: vpermq {{.*#+}} ymm26 = ymm1[0,1,0,1]4014; AVX512-NEXT: vmovdqa 32(%r9), %ymm14015; AVX512-NEXT: vpshuflw {{.*#+}} ymm13 = ymm1[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]4016; AVX512-NEXT: vpermq {{.*#+}} ymm13 = ymm13[2,2,2,2]4017; AVX512-NEXT: vinserti64x4 $1, %ymm23, %zmm21, %zmm214018; AVX512-NEXT: vmovdqa32 %zmm21, %zmm0 {%k1}4019; AVX512-NEXT: vinserti64x4 $1, %ymm25, %zmm24, %zmm214020; AVX512-NEXT: vmovdqa32 %zmm21, %zmm0 {%k2}4021; AVX512-NEXT: vinserti64x4 $1, %ymm13, %zmm26, %zmm274022; AVX512-NEXT: vpternlogq {{.*#+}} zmm27 = zmm27 ^ (zmm22 & (zmm27 ^ zmm0))4023; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]4024; AVX512-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero4025; AVX512-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero4026; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]4027; AVX512-NEXT: vpmovsxbd {{.*#+}} zmm26 = [1,0,2,2,1,0,2,2,16,17,16,17,16,17,16,17]4028; AVX512-NEXT: vpermt2d %zmm2, %zmm26, %zmm04029; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm4[0,1,2,1]4030; AVX512-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,5]4031; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm5[0,1,2,1]4032; AVX512-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,5]4033; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]4034; AVX512-NEXT: vpshuflw {{.*#+}} ymm3 = ymm14[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]4035; AVX512-NEXT: vpermq {{.*#+}} ymm21 = ymm3[2,2,2,3]4036; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]4037; AVX512-NEXT: vmovdqa {{.*#+}} ymm4 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]4038; AVX512-NEXT: vpshufb %ymm4, %ymm14, %ymm54039; AVX512-NEXT: vpermq {{.*#+}} ymm22 = ymm5[2,1,2,3]4040; AVX512-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]4041; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm28 = zmm3[0,1,2,3],zmm2[0,1,0,1]4042; AVX512-NEXT: vpshuflw {{.*#+}} ymm2 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]4043; AVX512-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]4044; AVX512-NEXT: vpermq {{.*#+}} ymm23 = ymm2[2,2,2,3]4045; AVX512-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[2,3,2,3,6,7,6,7]4046; AVX512-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]4047; AVX512-NEXT: vpermq {{.*#+}} ymm25 = ymm1[2,1,2,3]4048; AVX512-NEXT: vpermq {{.*#+}} ymm3 = ymm29[2,2,2,2]4049; AVX512-NEXT: vpshufd {{.*#+}} ymm24 = ymm31[1,2,3,3,5,6,7,7]4050; AVX512-NEXT: vpermq {{.*#+}} ymm5 = ymm18[2,1,2,3]4051; AVX512-NEXT: vpermq {{.*#+}} ymm2 = ymm16[3,3,3,3]4052; AVX512-NEXT: vpshufb %ymm4, %ymm6, %ymm14053; AVX512-NEXT: vpshuflw {{.*#+}} ymm4 = ymm6[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]4054; AVX512-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]4055; AVX512-NEXT: vpermq {{.*#+}} ymm14 = ymm1[2,1,2,3]4056; AVX512-NEXT: vmovdqa32 %zmm0, %zmm28 {%k2}4057; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]4058; AVX512-NEXT: vpsrldq {{.*#+}} xmm1 = xmm11[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero4059; AVX512-NEXT: vpsrldq {{.*#+}} xmm6 = xmm12[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero4060; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm6[0],xmm1[0],xmm6[1],xmm1[1],xmm6[2],xmm1[2],xmm6[3],xmm1[3]4061; AVX512-NEXT: vpshuflw {{.*#+}} ymm6 = ymm8[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]4062; AVX512-NEXT: vpshufhw {{.*#+}} ymm6 = ymm6[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]4063; AVX512-NEXT: vpermq {{.*#+}} ymm6 = ymm6[2,2,2,3]4064; AVX512-NEXT: vpshufd {{.*#+}} ymm8 = ymm8[2,3,2,3,6,7,6,7]4065; AVX512-NEXT: vpshuflw {{.*#+}} ymm8 = ymm8[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]4066; AVX512-NEXT: vpermq {{.*#+}} ymm8 = ymm8[2,1,2,3]4067; AVX512-NEXT: vpermt2d %zmm1, %zmm26, %zmm04068; AVX512-NEXT: vmovdqa64 %xmm17, %xmm124069; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm17[0,1,2,1]4070; AVX512-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,5]4071; AVX512-NEXT: vpshufd {{.*#+}} xmm11 = xmm9[0,1,2,1]4072; AVX512-NEXT: vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,7,6,5]4073; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm11[4],xmm1[4],xmm11[5],xmm1[5],xmm11[6],xmm1[6],xmm11[7],xmm1[7]4074; AVX512-NEXT: vpmovzxwd {{.*#+}} xmm11 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero4075; AVX512-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[2,1,3,3,4,5,6,7]4076; AVX512-NEXT: vpermq {{.*#+}} ymm7 = ymm7[0,0,2,1]4077; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm12[0],xmm9[1],xmm12[1],xmm9[2],xmm12[2],xmm9[3],xmm12[3]4078; AVX512-NEXT: vpshuflw {{.*#+}} xmm12 = xmm15[0,0,2,1,4,5,6,7]4079; AVX512-NEXT: vpbroadcastq %xmm12, %ymm124080; AVX512-NEXT: vpshuflw {{.*#+}} xmm13 = xmm15[0,2,2,3,4,5,6,7]4081; AVX512-NEXT: vpshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,4,4,4,4]4082; AVX512-NEXT: vpermq {{.*#+}} ymm13 = ymm13[0,0,2,1]4083; AVX512-NEXT: vpermq {{.*#+}} ymm9 = ymm9[0,0,2,1]4084; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm9[0,1,2,3],zmm1[0,1,0,1]4085; AVX512-NEXT: vmovdqa32 %zmm0, %zmm1 {%k2}4086; AVX512-NEXT: vmovdqa64 %xmm30, %xmm94087; AVX512-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm30[0],zero,xmm30[1],zero,xmm30[2],zero,xmm30[3],zero4088; AVX512-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[2,1,3,3,4,5,6,7]4089; AVX512-NEXT: vpermq {{.*#+}} ymm9 = ymm9[0,0,2,1]4090; AVX512-NEXT: vpshuflw {{.*#+}} xmm15 = xmm10[0,0,2,1,4,5,6,7]4091; AVX512-NEXT: vpbroadcastq %xmm15, %ymm154092; AVX512-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[0,2,2,3,4,5,6,7]4093; AVX512-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,4,4,4,4]4094; AVX512-NEXT: vpermq {{.*#+}} ymm10 = ymm10[0,0,2,1]4095; AVX512-NEXT: vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %ymm17 # 32-byte Folded Reload4096; AVX512-NEXT: # ymm17 = mem[2,2,2,3]4097; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload4098; AVX512-NEXT: vinserti64x4 $1, %ymm17, %zmm16, %zmm164099; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload4100; AVX512-NEXT: vinserti64x4 $1, %ymm19, %zmm17, %zmm174101; AVX512-NEXT: vmovdqa32 %zmm16, %zmm17 {%k1}4102; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax4103; AVX512-NEXT: vinserti64x4 $1, %ymm22, %zmm21, %zmm164104; AVX512-NEXT: movw $-28087, %cx # imm = 0x92494105; AVX512-NEXT: kmovw %ecx, %k24106; AVX512-NEXT: vmovdqa32 %zmm16, %zmm17 {%k2}4107; AVX512-NEXT: vinserti64x4 $1, %ymm25, %zmm23, %zmm164108; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm18 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]4109; AVX512-NEXT: vpternlogq {{.*#+}} zmm16 = zmm16 ^ (zmm18 & (zmm16 ^ zmm17))4110; AVX512-NEXT: vpermq {{.*#+}} ymm17 = ymm24[2,2,2,3]4111; AVX512-NEXT: vinserti64x4 $1, %ymm17, %zmm3, %zmm34112; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm5, %zmm24113; AVX512-NEXT: vmovdqa32 %zmm3, %zmm2 {%k1}4114; AVX512-NEXT: vinserti64x4 $1, %ymm14, %zmm4, %zmm34115; AVX512-NEXT: vmovdqa32 %zmm3, %zmm2 {%k2}4116; AVX512-NEXT: vinserti64x4 $1, %ymm8, %zmm6, %zmm34117; AVX512-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 ^ (zmm18 & (zmm3 ^ zmm2))4118; AVX512-NEXT: vpbroadcastq %xmm11, %ymm24119; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm2, %zmm24120; AVX512-NEXT: vmovdqa32 %zmm2, %zmm28 {%k1}4121; AVX512-NEXT: vinserti64x4 $1, %ymm13, %zmm12, %zmm24122; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm4 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]4123; AVX512-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm4 & (zmm2 ^ zmm28))4124; AVX512-NEXT: vpbroadcastq %xmm0, %ymm04125; AVX512-NEXT: vinserti64x4 $1, %ymm9, %zmm0, %zmm04126; AVX512-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}4127; AVX512-NEXT: vinserti64x4 $1, %ymm10, %zmm15, %zmm04128; AVX512-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm4 & (zmm0 ^ zmm1))4129; AVX512-NEXT: vmovdqa64 %zmm0, (%rax)4130; AVX512-NEXT: vmovdqa64 %zmm2, 192(%rax)4131; AVX512-NEXT: vmovdqa64 %zmm27, 256(%rax)4132; AVX512-NEXT: vmovdqa64 %zmm20, 64(%rax)4133; AVX512-NEXT: vmovdqa64 %zmm3, 128(%rax)4134; AVX512-NEXT: vmovdqa64 %zmm16, 320(%rax)4135; AVX512-NEXT: addq $40, %rsp4136; AVX512-NEXT: vzeroupper4137; AVX512-NEXT: retq4138;4139; AVX512-FCP-LABEL: store_i16_stride6_vf32:4140; AVX512-FCP: # %bb.0:4141; AVX512-FCP-NEXT: subq $72, %rsp4142; AVX512-FCP-NEXT: vmovdqa 32(%rcx), %ymm14143; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4144; AVX512-FCP-NEXT: vmovdqa 32(%rdx), %ymm24145; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4146; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15]4147; AVX512-FCP-NEXT: vpsrldq {{.*#+}} ymm1 = ymm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm1[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero4148; AVX512-FCP-NEXT: vpsrldq {{.*#+}} ymm2 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero4149; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]4150; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm4 = [4,5,4,5,4,5,4,5,21,22,21,22,21,22,23,23]4151; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm4, %zmm14152; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %ymm54153; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4154; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm0 = [u,u,u,u,4,5,10,11,u,u,u,u,u,u,u,u,24,25,22,23,20,21,26,27,u,u,u,u,u,u,u,u]4155; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm5, %ymm24156; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm64157; AVX512-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4158; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm6, %ymm34159; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]4160; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm6[4],ymm5[4],ymm6[5],ymm5[5],ymm6[6],ymm5[6],ymm6[7],ymm5[7],ymm6[12],ymm5[12],ymm6[13],ymm5[13],ymm6[14],ymm5[14],ymm6[15],ymm5[15]4161; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm5 = [2,1,2,3,11,11,11,11]4162; AVX512-FCP-NEXT: vpermt2q %zmm3, %zmm5, %zmm24163; AVX512-FCP-NEXT: movw $18724, %ax # imm = 0x49244164; AVX512-FCP-NEXT: kmovw %eax, %k14165; AVX512-FCP-NEXT: vmovdqa32 %zmm1, %zmm2 {%k1}4166; AVX512-FCP-NEXT: vmovdqa 32(%r8), %ymm74167; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm1 = ymm7[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]4168; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm8 = [12,1,2,13,4,5,14,7]4169; AVX512-FCP-NEXT: vmovdqa %ymm2, %ymm34170; AVX512-FCP-NEXT: vpermt2d %ymm1, %ymm8, %ymm34171; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm9 = [8,21,10,11,20,13,14,23]4172; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]4173; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm7, %ymm64174; AVX512-FCP-NEXT: vmovdqa64 %ymm7, %ymm304175; AVX512-FCP-NEXT: vpermt2d %zmm6, %zmm9, %zmm24176; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm24177; AVX512-FCP-NEXT: vmovdqu64 %zmm2, (%rsp) # 64-byte Spill4178; AVX512-FCP-NEXT: vmovdqa 32(%r9), %ymm74179; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm2 = [u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,24,25,28,29,28,29,26,27,24,25,26,27,28,29,30,31]4180; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm7, %ymm64181; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25,16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25]4182; AVX512-FCP-NEXT: # ymm3 = mem[0,1,0,1]4183; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm7, %ymm104184; AVX512-FCP-NEXT: vmovdqa64 %ymm7, %ymm314185; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm19 = [2,2,0,3,10,0,10,11]4186; AVX512-FCP-NEXT: vpermi2q %zmm6, %zmm10, %zmm194187; AVX512-FCP-NEXT: vmovdqa (%rcx), %ymm74188; AVX512-FCP-NEXT: vmovdqa (%rdx), %ymm104189; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm6 = ymm10[4],ymm7[4],ymm10[5],ymm7[5],ymm10[6],ymm7[6],ymm10[7],ymm7[7],ymm10[12],ymm7[12],ymm10[13],ymm7[13],ymm10[14],ymm7[14],ymm10[15],ymm7[15]4190; AVX512-FCP-NEXT: vpsrldq {{.*#+}} ymm12 = ymm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm7[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero4191; AVX512-FCP-NEXT: vmovdqa64 %ymm7, %ymm284192; AVX512-FCP-NEXT: vpsrldq {{.*#+}} ymm13 = ymm10[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm10[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero4193; AVX512-FCP-NEXT: vmovdqa64 %ymm10, %ymm274194; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm12 = ymm13[0],ymm12[0],ymm13[1],ymm12[1],ymm13[2],ymm12[2],ymm13[3],ymm12[3],ymm13[8],ymm12[8],ymm13[9],ymm12[9],ymm13[10],ymm12[10],ymm13[11],ymm12[11]4195; AVX512-FCP-NEXT: vpermt2d %zmm6, %zmm4, %zmm124196; AVX512-FCP-NEXT: vmovdqa (%rsi), %ymm64197; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm6, %ymm44198; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm144199; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm14, %ymm04200; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm4[0],ymm0[1],ymm4[1],ymm0[2],ymm4[2],ymm0[3],ymm4[3],ymm0[8],ymm4[8],ymm0[9],ymm4[9],ymm0[10],ymm4[10],ymm0[11],ymm4[11]4201; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm14[4],ymm6[4],ymm14[5],ymm6[5],ymm14[6],ymm6[6],ymm14[7],ymm6[7],ymm14[12],ymm6[12],ymm14[13],ymm6[13],ymm14[14],ymm6[14],ymm14[15],ymm6[15]4202; AVX512-FCP-NEXT: vmovdqa64 %ymm6, %ymm174203; AVX512-FCP-NEXT: vpermt2q %zmm4, %zmm5, %zmm04204; AVX512-FCP-NEXT: vmovdqa32 %zmm12, %zmm0 {%k1}4205; AVX512-FCP-NEXT: vmovdqa (%r8), %ymm134206; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm4 = ymm13[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]4207; AVX512-FCP-NEXT: vpermi2d %ymm4, %ymm0, %ymm84208; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm13, %ymm14209; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm9, %zmm04210; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm8, %zmm294211; AVX512-FCP-NEXT: vmovdqa (%r9), %ymm44212; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm4, %ymm04213; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm4, %ymm14214; AVX512-FCP-NEXT: vmovdqa64 %ymm4, %ymm264215; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm20 = [2,2,2,3,10,9,10,11]4216; AVX512-FCP-NEXT: vpermi2q %zmm0, %zmm1, %zmm204217; AVX512-FCP-NEXT: vmovdqa 32(%r9), %xmm34218; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]4219; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm14220; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm3[0,0,2,1,4,5,6,7]4221; AVX512-FCP-NEXT: vmovdqa64 %xmm3, %xmm164222; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm23 = [0,0,0,0,8,8,0,9]4223; AVX512-FCP-NEXT: vpermi2q %zmm1, %zmm2, %zmm234224; AVX512-FCP-NEXT: vmovdqa (%r9), %xmm114225; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm11, %xmm04226; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm11[0,0,2,1,4,5,6,7]4227; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm18 = [0,0,0,0,8,8,0,9]4228; AVX512-FCP-NEXT: vpermi2q %zmm0, %zmm1, %zmm184229; AVX512-FCP-NEXT: vmovdqa 32(%rcx), %xmm104230; AVX512-FCP-NEXT: vmovdqa 32(%rdx), %xmm94231; AVX512-FCP-NEXT: vpsrldq {{.*#+}} xmm0 = xmm10[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero4232; AVX512-FCP-NEXT: vpsrldq {{.*#+}} xmm1 = xmm9[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero4233; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]4234; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm21 = [1,0,2,2,1,0,2,2,16,17,16,17,16,17,16,17]4235; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]4236; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm21, %zmm14237; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %xmm84238; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm6 = [0,1,2,3,4,5,6,7,8,9,6,7,4,5,10,11]4239; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm8, %xmm04240; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %xmm74241; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm7, %xmm154242; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7]4243; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm22 = [8,8,0,9,0,1,0,1]4244; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]4245; AVX512-FCP-NEXT: vpermt2q %zmm2, %zmm22, %zmm04246; AVX512-FCP-NEXT: movw $9362, %ax # imm = 0x24924247; AVX512-FCP-NEXT: kmovw %eax, %k24248; AVX512-FCP-NEXT: vmovdqa32 %zmm1, %zmm0 {%k2}4249; AVX512-FCP-NEXT: vmovdqa64 %ymm0, %ymm244250; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm25 = [16,9,10,17,12,13,18,15]4251; AVX512-FCP-NEXT: vmovdqa 32(%r8), %xmm154252; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm15[2,1,3,3,4,5,6,7]4253; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm25, %zmm04254; AVX512-FCP-NEXT: vmovdqa (%rcx), %xmm54255; AVX512-FCP-NEXT: vmovdqa (%rdx), %xmm44256; AVX512-FCP-NEXT: vpsrldq {{.*#+}} xmm1 = xmm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero4257; AVX512-FCP-NEXT: vpsrldq {{.*#+}} xmm12 = xmm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero4258; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm12[0],xmm1[0],xmm12[1],xmm1[1],xmm12[2],xmm1[2],xmm12[3],xmm1[3]4259; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]4260; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm21, %zmm124261; AVX512-FCP-NEXT: vmovdqa (%rsi), %xmm24262; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm14263; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm2, %xmm34264; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm1, %xmm64265; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]4266; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]4267; AVX512-FCP-NEXT: vpermt2q %zmm6, %zmm22, %zmm34268; AVX512-FCP-NEXT: vpmovzxwd {{.*#+}} xmm6 = xmm15[0],zero,xmm15[1],zero,xmm15[2],zero,xmm15[3],zero4269; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm22 = [0,1,8,3,4,9,6,7]4270; AVX512-FCP-NEXT: vpermt2d %ymm6, %ymm22, %ymm244271; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm24, %zmm214272; AVX512-FCP-NEXT: vmovdqa32 %zmm12, %zmm3 {%k2}4273; AVX512-FCP-NEXT: vmovdqa (%r8), %xmm64274; AVX512-FCP-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero4275; AVX512-FCP-NEXT: vpermi2d %ymm0, %ymm3, %ymm224276; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm6[2,1,3,3,4,5,6,7]4277; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm25, %zmm34278; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm22, %zmm224279; AVX512-FCP-NEXT: vmovdqa64 %ymm17, %ymm04280; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm14[0],ymm0[0],ymm14[1],ymm0[1],ymm14[2],ymm0[2],ymm14[3],ymm0[3],ymm14[8],ymm0[8],ymm14[9],ymm0[9],ymm14[10],ymm0[10],ymm14[11],ymm0[11]4281; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]4282; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm3 = [1,1,1,1,10,10,10,11]4283; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm3, %zmm14284; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]4285; AVX512-FCP-NEXT: vmovdqa64 %ymm28, %ymm24286; AVX512-FCP-NEXT: vmovdqa64 %ymm27, %ymm44287; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm4[0],ymm2[0],ymm4[1],ymm2[1],ymm4[2],ymm2[2],ymm4[3],ymm2[3],ymm4[8],ymm2[8],ymm4[9],ymm2[9],ymm4[10],ymm2[10],ymm4[11],ymm2[11]4288; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm4 = [17,18,17,18,0,0,19,19,5,4,2,2,5,4,6,6]4289; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm4, %zmm24290; AVX512-FCP-NEXT: vmovdqa32 %zmm1, %zmm2 {%k1}4291; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]4292; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm6, %xmm14293; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm5 = [0,9,2,3,8,5,6,11]4294; AVX512-FCP-NEXT: vmovdqa %ymm2, %ymm64295; AVX512-FCP-NEXT: vpermt2d %ymm1, %ymm5, %ymm64296; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm12 = [8,9,20,11,12,21,14,15]4297; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm1 = ymm13[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]4298; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm12, %zmm24299; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm6, %zmm64300; AVX512-FCP-NEXT: vmovdqa64 %ymm26, %ymm14301; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm14 = ymm1[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]4302; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm1 = [8,9,12,13,12,13,10,11,8,9,10,11,12,13,14,15]4303; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm11, %xmm24304; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm11 = [0,0,0,1,0,10,10,0]4305; AVX512-FCP-NEXT: vpermt2q %zmm14, %zmm11, %zmm24306; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload4307; AVX512-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload4308; AVX512-FCP-NEXT: # ymm14 = ymm14[0],mem[0],ymm14[1],mem[1],ymm14[2],mem[2],ymm14[3],mem[3],ymm14[8],mem[8],ymm14[9],mem[9],ymm14[10],mem[10],ymm14[11],mem[11]4309; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm7 = xmm7[4],xmm8[4],xmm7[5],xmm8[5],xmm7[6],xmm8[6],xmm7[7],xmm8[7]4310; AVX512-FCP-NEXT: vpermt2q %zmm14, %zmm3, %zmm74311; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm9[4],xmm10[4],xmm9[5],xmm10[5],xmm9[6],xmm10[6],xmm9[7],xmm10[7]4312; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload4313; AVX512-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload4314; AVX512-FCP-NEXT: # ymm8 = ymm8[0],mem[0],ymm8[1],mem[1],ymm8[2],mem[2],ymm8[3],mem[3],ymm8[8],mem[8],ymm8[9],mem[9],ymm8[10],mem[10],ymm8[11],mem[11]4315; AVX512-FCP-NEXT: vpermt2d %zmm3, %zmm4, %zmm84316; AVX512-FCP-NEXT: vmovdqa32 %zmm7, %zmm8 {%k1}4317; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm15, %xmm04318; AVX512-FCP-NEXT: vpermi2d %ymm0, %ymm8, %ymm54319; AVX512-FCP-NEXT: vmovdqa64 %ymm30, %ymm04320; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm0 = ymm0[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]4321; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm12, %zmm84322; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm8, %zmm5, %zmm04323; AVX512-FCP-NEXT: vmovdqa64 %xmm16, %xmm34324; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm3, %xmm14325; AVX512-FCP-NEXT: vmovdqa64 %ymm31, %ymm34326; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm3 = ymm3[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]4327; AVX512-FCP-NEXT: vpermt2q %zmm3, %zmm11, %zmm14328; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]4329; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm3 & (zmm1 ^ zmm0))4330; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax4331; AVX512-FCP-NEXT: vmovdqa64 %zmm1, 256(%rax)4332; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm3 & (zmm2 ^ zmm6))4333; AVX512-FCP-NEXT: vmovdqa64 %zmm2, 64(%rax)4334; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm0 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]4335; AVX512-FCP-NEXT: vpternlogq $184, (%rsp), %zmm0, %zmm19 # 64-byte Folded Reload4336; AVX512-FCP-NEXT: # zmm19 = zmm19 ^ (zmm0 & (zmm19 ^ mem))4337; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm20 = zmm20 ^ (zmm0 & (zmm20 ^ zmm29))4338; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]4339; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm23 = zmm23 ^ (zmm0 & (zmm23 ^ zmm21))4340; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm18 = zmm18 ^ (zmm0 & (zmm18 ^ zmm22))4341; AVX512-FCP-NEXT: vmovdqa64 %zmm18, (%rax)4342; AVX512-FCP-NEXT: vmovdqa64 %zmm23, 192(%rax)4343; AVX512-FCP-NEXT: vmovdqa64 %zmm20, 128(%rax)4344; AVX512-FCP-NEXT: vmovdqa64 %zmm19, 320(%rax)4345; AVX512-FCP-NEXT: addq $72, %rsp4346; AVX512-FCP-NEXT: vzeroupper4347; AVX512-FCP-NEXT: retq4348;4349; AVX512DQ-LABEL: store_i16_stride6_vf32:4350; AVX512DQ: # %bb.0:4351; AVX512DQ-NEXT: subq $152, %rsp4352; AVX512DQ-NEXT: vmovdqa (%rcx), %xmm104353; AVX512DQ-NEXT: vmovdqa 32(%rcx), %xmm24354; AVX512DQ-NEXT: vmovdqa (%rdx), %xmm44355; AVX512DQ-NEXT: vmovdqa 32(%rdx), %xmm34356; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]4357; AVX512DQ-NEXT: vpsrldq {{.*#+}} xmm1 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero4358; AVX512DQ-NEXT: vmovdqa64 %xmm2, %xmm304359; AVX512DQ-NEXT: vpsrldq {{.*#+}} xmm2 = xmm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero4360; AVX512DQ-NEXT: vmovdqa64 %xmm3, %xmm254361; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]4362; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm2 = [1,0,2,2,1,0,2,2,16,17,16,17,16,17,16,17]4363; AVX512DQ-NEXT: vpermt2d %zmm1, %zmm2, %zmm04364; AVX512DQ-NEXT: vmovdqa (%rsi), %xmm84365; AVX512DQ-NEXT: vmovdqa 32(%rsi), %xmm54366; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm5[0,1,2,1]4367; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,5]4368; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm94369; AVX512DQ-NEXT: vmovdqa 32(%rdi), %xmm134370; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm3 = xmm13[0,1,2,1]4371; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,5]4372; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]4373; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]4374; AVX512DQ-NEXT: vmovdqa64 %xmm5, %xmm234375; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]4376; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm27 = zmm3[0,1,2,3],zmm1[0,1,0,1]4377; AVX512DQ-NEXT: movw $9362, %ax # imm = 0x24924378; AVX512DQ-NEXT: kmovw %eax, %k14379; AVX512DQ-NEXT: vmovdqa32 %zmm0, %zmm27 {%k1}4380; AVX512DQ-NEXT: vmovdqa (%r8), %xmm154381; AVX512DQ-NEXT: vmovdqa 32(%r8), %xmm14382; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm1[2,1,3,3,4,5,6,7]4383; AVX512DQ-NEXT: vmovdqa64 %xmm1, %xmm264384; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]4385; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4386; AVX512DQ-NEXT: vmovdqa (%r9), %xmm54387; AVX512DQ-NEXT: vmovdqa 32(%r9), %xmm14388; AVX512DQ-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4389; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm1[0,0,2,1,4,5,6,7]4390; AVX512DQ-NEXT: vpbroadcastq %xmm0, %ymm04391; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4392; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm1[0,2,2,3,4,5,6,7]4393; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]4394; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]4395; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4396; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm10[0],xmm4[1],xmm10[1],xmm4[2],xmm10[2],xmm4[3],xmm10[3]4397; AVX512DQ-NEXT: vpsrldq {{.*#+}} xmm1 = xmm10[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero4398; AVX512DQ-NEXT: vpsrldq {{.*#+}} xmm3 = xmm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero4399; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]4400; AVX512DQ-NEXT: vpermt2d %zmm1, %zmm2, %zmm04401; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm8[0,1,2,1]4402; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,5]4403; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm9[0,1,2,1]4404; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,5]4405; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]4406; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]4407; AVX512DQ-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]4408; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm28 = zmm2[0,1,2,3],zmm1[0,1,0,1]4409; AVX512DQ-NEXT: vmovdqa32 %zmm0, %zmm28 {%k1}4410; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm15[2,1,3,3,4,5,6,7]4411; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]4412; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4413; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm5[0,0,2,1,4,5,6,7]4414; AVX512DQ-NEXT: vpbroadcastq %xmm0, %ymm04415; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4416; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm5[0,2,2,3,4,5,6,7]4417; AVX512DQ-NEXT: vmovdqa64 %xmm5, %xmm184418; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]4419; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]4420; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4421; AVX512DQ-NEXT: vmovdqa 32(%rsi), %ymm124422; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm0 = ymm12[2,1,2,3,6,5,6,7]4423; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm0 = ymm0[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]4424; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm144425; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm1 = ymm14[2,1,2,3,6,5,6,7]4426; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]4427; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]4428; AVX512DQ-NEXT: vpermq {{.*#+}} ymm22 = ymm0[2,1,2,3]4429; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm14[4],ymm12[4],ymm14[5],ymm12[5],ymm14[6],ymm12[6],ymm14[7],ymm12[7],ymm14[12],ymm12[12],ymm14[13],ymm12[13],ymm14[14],ymm12[14],ymm14[15],ymm12[15]4430; AVX512DQ-NEXT: vpermq {{.*#+}} ymm21 = ymm0[3,3,3,3]4431; AVX512DQ-NEXT: vmovdqa 32(%rcx), %ymm74432; AVX512DQ-NEXT: vpsrldq {{.*#+}} ymm0 = ymm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm7[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero4433; AVX512DQ-NEXT: vmovdqa 32(%rdx), %ymm54434; AVX512DQ-NEXT: vpsrldq {{.*#+}} ymm1 = ymm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm5[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero4435; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]4436; AVX512DQ-NEXT: vpermq {{.*#+}} ymm20 = ymm0[2,2,2,2]4437; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm5[4],ymm7[4],ymm5[5],ymm7[5],ymm5[6],ymm7[6],ymm5[7],ymm7[7],ymm5[12],ymm7[12],ymm5[13],ymm7[13],ymm5[14],ymm7[14],ymm5[15],ymm7[15]4438; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm19 = ymm0[1,2,3,3,5,6,7,7]4439; AVX512DQ-NEXT: vmovdqa (%rsi), %ymm34440; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm0 = ymm3[2,1,2,3,6,5,6,7]4441; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm0 = ymm0[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]4442; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm24443; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm1 = ymm2[2,1,2,3,6,5,6,7]4444; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]4445; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]4446; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm174447; AVX512DQ-NEXT: vmovdqa (%rcx), %ymm14448; AVX512DQ-NEXT: vpsrldq {{.*#+}} ymm6 = ymm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm1[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero4449; AVX512DQ-NEXT: vmovdqa (%rdx), %ymm04450; AVX512DQ-NEXT: vpsrldq {{.*#+}} ymm11 = ymm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm0[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero4451; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm6 = ymm11[0],ymm6[0],ymm11[1],ymm6[1],ymm11[2],ymm6[2],ymm11[3],ymm6[3],ymm11[8],ymm6[8],ymm11[9],ymm6[9],ymm11[10],ymm6[10],ymm11[11],ymm6[11]4452; AVX512DQ-NEXT: vmovdqa64 %ymm6, %ymm164453; AVX512DQ-NEXT: vmovdqa 32(%r8), %ymm64454; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm11 = ymm6[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]4455; AVX512DQ-NEXT: vpermq {{.*#+}} ymm24 = ymm11[2,2,2,3]4456; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm10 = xmm4[4],xmm10[4],xmm4[5],xmm10[5],xmm4[6],xmm10[6],xmm4[7],xmm10[7]4457; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm11 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]4458; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm4 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]4459; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm31 = [17,18,17,18,0,0,19,19,5,4,2,2,5,4,6,6]4460; AVX512DQ-NEXT: vpermt2d %zmm10, %zmm31, %zmm44461; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm8 = xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]4462; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm9 = ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15]4463; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11]4464; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm1 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]4465; AVX512DQ-NEXT: vpshufb %ymm1, %ymm6, %ymm24466; AVX512DQ-NEXT: vpermq {{.*#+}} ymm29 = ymm2[2,1,2,3]4467; AVX512DQ-NEXT: vmovdqa64 %xmm30, %xmm24468; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm34469; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]4470; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm5 = ymm5[0],ymm7[0],ymm5[1],ymm7[1],ymm5[2],ymm7[2],ymm5[3],ymm7[3],ymm5[8],ymm7[8],ymm5[9],ymm7[9],ymm5[10],ymm7[10],ymm5[11],ymm7[11]4471; AVX512DQ-NEXT: vmovdqa 32(%r9), %ymm74472; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm3 = ymm7[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]4473; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]4474; AVX512DQ-NEXT: vpermq {{.*#+}} ymm30 = ymm3[2,2,2,3]4475; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = ymm8[1,1,1,1]4476; AVX512DQ-NEXT: vpermt2d %zmm2, %zmm31, %zmm54477; AVX512DQ-NEXT: vmovdqa64 %xmm23, %xmm24478; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm13[4],xmm2[4],xmm13[5],xmm2[5],xmm13[6],xmm2[6],xmm13[7],xmm2[7]4479; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm8 = ymm14[0],ymm12[0],ymm14[1],ymm12[1],ymm14[2],ymm12[2],ymm14[3],ymm12[3],ymm14[8],ymm12[8],ymm14[9],ymm12[9],ymm14[10],ymm12[10],ymm14[11],ymm12[11]4480; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]4481; AVX512DQ-NEXT: vpmovzxwd {{.*#+}} xmm10 = xmm15[0],zero,xmm15[1],zero,xmm15[2],zero,xmm15[3],zero4482; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm12 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]4483; AVX512DQ-NEXT: vpshufb %xmm12, %xmm15, %xmm134484; AVX512DQ-NEXT: vpermq {{.*#+}} ymm13 = ymm13[0,1,0,1]4485; AVX512DQ-NEXT: vmovdqa (%r8), %ymm144486; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm15 = ymm14[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]4487; AVX512DQ-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,2,2,2]4488; AVX512DQ-NEXT: movw $18724, %ax # imm = 0x49244489; AVX512DQ-NEXT: kmovw %eax, %k24490; AVX512DQ-NEXT: vinserti32x8 $1, %ymm0, %zmm3, %zmm4 {%k2}4491; AVX512DQ-NEXT: vinserti32x8 $1, %ymm15, %zmm13, %zmm4 {%k1}4492; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm2[1,1,1,1]4493; AVX512DQ-NEXT: vpermq {{.*#+}} ymm2 = ymm8[2,2,2,3]4494; AVX512DQ-NEXT: vinserti32x8 $1, %ymm2, %zmm0, %zmm5 {%k2}4495; AVX512DQ-NEXT: vmovdqa64 %xmm26, %xmm24496; AVX512DQ-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm26[0],zero,xmm26[1],zero,xmm26[2],zero,xmm26[3],zero4497; AVX512DQ-NEXT: vpshufb %xmm12, %xmm2, %xmm24498; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm3 = ymm7[2,3,2,3,6,7,6,7]4499; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm3 = ymm3[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]4500; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,1,2,3]4501; AVX512DQ-NEXT: vpermq {{.*#+}} ymm8 = ymm17[2,1,2,3]4502; AVX512DQ-NEXT: vpermq {{.*#+}} ymm9 = ymm9[3,3,3,3]4503; AVX512DQ-NEXT: vpermq {{.*#+}} ymm12 = ymm16[2,2,2,2]4504; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm11 = ymm11[1,2,3,3,5,6,7,7]4505; AVX512DQ-NEXT: vpshufb %ymm1, %ymm14, %ymm14506; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm13 = ymm14[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]4507; AVX512DQ-NEXT: vpermq {{.*#+}} ymm13 = ymm13[2,2,2,3]4508; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]4509; AVX512DQ-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]4510; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm6 = ymm6[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]4511; AVX512DQ-NEXT: vpermq {{.*#+}} ymm6 = ymm6[2,2,2,2]4512; AVX512DQ-NEXT: vinserti32x8 $1, %ymm6, %zmm2, %zmm5 {%k1}4513; AVX512DQ-NEXT: vmovdqa (%r9), %ymm24514; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm6 = xmm18[2,3,2,3]4515; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,2,2,1,4,5,6,7]4516; AVX512DQ-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,1,0,1]4517; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm14 = ymm2[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]4518; AVX512DQ-NEXT: vpermq {{.*#+}} ymm14 = ymm14[2,2,2,2]4519; AVX512DQ-NEXT: vinserti64x4 $1, %ymm14, %zmm6, %zmm64520; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm14 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]4521; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm14 & (zmm6 ^ zmm4))4522; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm4 = ymm2[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]4523; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm4 = ymm4[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]4524; AVX512DQ-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]4525; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[2,3,2,3,6,7,6,7]4526; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]4527; AVX512DQ-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]4528; AVX512DQ-NEXT: vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload4529; AVX512DQ-NEXT: # xmm15 = mem[2,3,2,3]4530; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm15 = xmm15[0,2,2,1,4,5,6,7]4531; AVX512DQ-NEXT: vpermq {{.*#+}} ymm15 = ymm15[0,1,0,1]4532; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm7 = ymm7[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]4533; AVX512DQ-NEXT: vpermq {{.*#+}} ymm7 = ymm7[2,2,2,2]4534; AVX512DQ-NEXT: vinserti64x4 $1, %ymm7, %zmm15, %zmm74535; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm14 & (zmm7 ^ zmm5))4536; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax4537; AVX512DQ-NEXT: vpbroadcastq %xmm0, %ymm04538; AVX512DQ-NEXT: vinserti32x8 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm27 {%k2} # 32-byte Folded Reload4539; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload4540; AVX512DQ-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload4541; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]4542; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm5 & (zmm0 ^ zmm27))4543; AVX512DQ-NEXT: vpbroadcastq %xmm10, %ymm104544; AVX512DQ-NEXT: vinserti32x8 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm28 {%k2} # 32-byte Folded Reload4545; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload4546; AVX512DQ-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm10 # 32-byte Folded Reload4547; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm5 & (zmm10 ^ zmm28))4548; AVX512DQ-NEXT: vinserti64x4 $1, %ymm21, %zmm22, %zmm54549; AVX512DQ-NEXT: vpermq {{.*#+}} ymm14 = ymm19[2,2,2,3]4550; AVX512DQ-NEXT: vinserti32x8 $1, %ymm14, %zmm20, %zmm5 {%k2}4551; AVX512DQ-NEXT: movw $-28087, %cx # imm = 0x92494552; AVX512DQ-NEXT: kmovw %ecx, %k14553; AVX512DQ-NEXT: vinserti32x8 $1, %ymm29, %zmm24, %zmm5 {%k1}4554; AVX512DQ-NEXT: vinserti64x4 $1, %ymm3, %zmm30, %zmm34555; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm14 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]4556; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 ^ (zmm14 & (zmm3 ^ zmm5))4557; AVX512DQ-NEXT: vinserti64x4 $1, %ymm9, %zmm8, %zmm54558; AVX512DQ-NEXT: vpermq {{.*#+}} ymm8 = ymm11[2,2,2,3]4559; AVX512DQ-NEXT: vinserti32x8 $1, %ymm8, %zmm12, %zmm5 {%k2}4560; AVX512DQ-NEXT: vinserti32x8 $1, %ymm1, %zmm13, %zmm5 {%k1}4561; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm4, %zmm14562; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm14 & (zmm1 ^ zmm5))4563; AVX512DQ-NEXT: vmovdqa64 %zmm7, 256(%rax)4564; AVX512DQ-NEXT: vmovdqa64 %zmm6, 64(%rax)4565; AVX512DQ-NEXT: vmovdqa64 %zmm10, (%rax)4566; AVX512DQ-NEXT: vmovdqa64 %zmm0, 192(%rax)4567; AVX512DQ-NEXT: vmovdqa64 %zmm1, 128(%rax)4568; AVX512DQ-NEXT: vmovdqa64 %zmm3, 320(%rax)4569; AVX512DQ-NEXT: addq $152, %rsp4570; AVX512DQ-NEXT: vzeroupper4571; AVX512DQ-NEXT: retq4572;4573; AVX512DQ-FCP-LABEL: store_i16_stride6_vf32:4574; AVX512DQ-FCP: # %bb.0:4575; AVX512DQ-FCP-NEXT: subq $72, %rsp4576; AVX512DQ-FCP-NEXT: vmovdqa 32(%rcx), %ymm14577; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4578; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdx), %ymm24579; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4580; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15]4581; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} ymm1 = ymm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm1[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero4582; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} ymm2 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero4583; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]4584; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm4 = [4,5,4,5,4,5,4,5,21,22,21,22,21,22,23,23]4585; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm4, %zmm14586; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %ymm54587; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4588; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm0 = [u,u,u,u,4,5,10,11,u,u,u,u,u,u,u,u,24,25,22,23,20,21,26,27,u,u,u,u,u,u,u,u]4589; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm5, %ymm24590; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm64591; AVX512DQ-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4592; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm6, %ymm34593; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]4594; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm6[4],ymm5[4],ymm6[5],ymm5[5],ymm6[6],ymm5[6],ymm6[7],ymm5[7],ymm6[12],ymm5[12],ymm6[13],ymm5[13],ymm6[14],ymm5[14],ymm6[15],ymm5[15]4595; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm5 = [2,1,2,3,11,11,11,11]4596; AVX512DQ-FCP-NEXT: vpermt2q %zmm3, %zmm5, %zmm24597; AVX512DQ-FCP-NEXT: movw $18724, %ax # imm = 0x49244598; AVX512DQ-FCP-NEXT: kmovw %eax, %k14599; AVX512DQ-FCP-NEXT: vmovdqa32 %zmm1, %zmm2 {%k1}4600; AVX512DQ-FCP-NEXT: vmovdqa 32(%r8), %ymm74601; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm1 = ymm7[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]4602; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm8 = [12,1,2,13,4,5,14,7]4603; AVX512DQ-FCP-NEXT: vmovdqa %ymm2, %ymm34604; AVX512DQ-FCP-NEXT: vpermt2d %ymm1, %ymm8, %ymm34605; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm9 = [8,21,10,11,20,13,14,23]4606; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]4607; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm7, %ymm64608; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm7, %ymm304609; AVX512DQ-FCP-NEXT: vpermt2d %zmm6, %zmm9, %zmm24610; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm24611; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm2, (%rsp) # 64-byte Spill4612; AVX512DQ-FCP-NEXT: vmovdqa 32(%r9), %ymm74613; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm2 = [u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,24,25,28,29,28,29,26,27,24,25,26,27,28,29,30,31]4614; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm7, %ymm64615; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25,16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25]4616; AVX512DQ-FCP-NEXT: # ymm3 = mem[0,1,0,1]4617; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm7, %ymm104618; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm7, %ymm314619; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm19 = [2,2,0,3,10,0,10,11]4620; AVX512DQ-FCP-NEXT: vpermi2q %zmm6, %zmm10, %zmm194621; AVX512DQ-FCP-NEXT: vmovdqa (%rcx), %ymm74622; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %ymm104623; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm6 = ymm10[4],ymm7[4],ymm10[5],ymm7[5],ymm10[6],ymm7[6],ymm10[7],ymm7[7],ymm10[12],ymm7[12],ymm10[13],ymm7[13],ymm10[14],ymm7[14],ymm10[15],ymm7[15]4624; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} ymm12 = ymm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm7[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero4625; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm7, %ymm284626; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} ymm13 = ymm10[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm10[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero4627; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm10, %ymm274628; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm12 = ymm13[0],ymm12[0],ymm13[1],ymm12[1],ymm13[2],ymm12[2],ymm13[3],ymm12[3],ymm13[8],ymm12[8],ymm13[9],ymm12[9],ymm13[10],ymm12[10],ymm13[11],ymm12[11]4629; AVX512DQ-FCP-NEXT: vpermt2d %zmm6, %zmm4, %zmm124630; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %ymm64631; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm6, %ymm44632; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm144633; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm14, %ymm04634; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm4[0],ymm0[1],ymm4[1],ymm0[2],ymm4[2],ymm0[3],ymm4[3],ymm0[8],ymm4[8],ymm0[9],ymm4[9],ymm0[10],ymm4[10],ymm0[11],ymm4[11]4635; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm14[4],ymm6[4],ymm14[5],ymm6[5],ymm14[6],ymm6[6],ymm14[7],ymm6[7],ymm14[12],ymm6[12],ymm14[13],ymm6[13],ymm14[14],ymm6[14],ymm14[15],ymm6[15]4636; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm6, %ymm174637; AVX512DQ-FCP-NEXT: vpermt2q %zmm4, %zmm5, %zmm04638; AVX512DQ-FCP-NEXT: vmovdqa32 %zmm12, %zmm0 {%k1}4639; AVX512DQ-FCP-NEXT: vmovdqa (%r8), %ymm134640; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm4 = ymm13[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]4641; AVX512DQ-FCP-NEXT: vpermi2d %ymm4, %ymm0, %ymm84642; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm13, %ymm14643; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm9, %zmm04644; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm8, %zmm294645; AVX512DQ-FCP-NEXT: vmovdqa (%r9), %ymm44646; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm4, %ymm04647; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm4, %ymm14648; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm4, %ymm264649; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm20 = [2,2,2,3,10,9,10,11]4650; AVX512DQ-FCP-NEXT: vpermi2q %zmm0, %zmm1, %zmm204651; AVX512DQ-FCP-NEXT: vmovdqa 32(%r9), %xmm34652; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]4653; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm14654; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm3[0,0,2,1,4,5,6,7]4655; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm3, %xmm164656; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm23 = [0,0,0,0,8,8,0,9]4657; AVX512DQ-FCP-NEXT: vpermi2q %zmm1, %zmm2, %zmm234658; AVX512DQ-FCP-NEXT: vmovdqa (%r9), %xmm114659; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm11, %xmm04660; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm11[0,0,2,1,4,5,6,7]4661; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm18 = [0,0,0,0,8,8,0,9]4662; AVX512DQ-FCP-NEXT: vpermi2q %zmm0, %zmm1, %zmm184663; AVX512DQ-FCP-NEXT: vmovdqa 32(%rcx), %xmm104664; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdx), %xmm94665; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} xmm0 = xmm10[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero4666; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} xmm1 = xmm9[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero4667; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]4668; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm21 = [1,0,2,2,1,0,2,2,16,17,16,17,16,17,16,17]4669; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]4670; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm21, %zmm14671; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %xmm84672; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm6 = [0,1,2,3,4,5,6,7,8,9,6,7,4,5,10,11]4673; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm8, %xmm04674; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %xmm74675; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm7, %xmm154676; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7]4677; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm22 = [8,8,0,9,0,1,0,1]4678; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]4679; AVX512DQ-FCP-NEXT: vpermt2q %zmm2, %zmm22, %zmm04680; AVX512DQ-FCP-NEXT: movw $9362, %ax # imm = 0x24924681; AVX512DQ-FCP-NEXT: kmovw %eax, %k24682; AVX512DQ-FCP-NEXT: vmovdqa32 %zmm1, %zmm0 {%k2}4683; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm0, %ymm244684; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm25 = [16,9,10,17,12,13,18,15]4685; AVX512DQ-FCP-NEXT: vmovdqa 32(%r8), %xmm154686; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm15[2,1,3,3,4,5,6,7]4687; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm25, %zmm04688; AVX512DQ-FCP-NEXT: vmovdqa (%rcx), %xmm54689; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %xmm44690; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} xmm1 = xmm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero4691; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} xmm12 = xmm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero4692; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm12[0],xmm1[0],xmm12[1],xmm1[1],xmm12[2],xmm1[2],xmm12[3],xmm1[3]4693; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]4694; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm21, %zmm124695; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %xmm24696; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm14697; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm2, %xmm34698; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm1, %xmm64699; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]4700; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]4701; AVX512DQ-FCP-NEXT: vpermt2q %zmm6, %zmm22, %zmm34702; AVX512DQ-FCP-NEXT: vpmovzxwd {{.*#+}} xmm6 = xmm15[0],zero,xmm15[1],zero,xmm15[2],zero,xmm15[3],zero4703; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm22 = [0,1,8,3,4,9,6,7]4704; AVX512DQ-FCP-NEXT: vpermt2d %ymm6, %ymm22, %ymm244705; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm24, %zmm214706; AVX512DQ-FCP-NEXT: vmovdqa32 %zmm12, %zmm3 {%k2}4707; AVX512DQ-FCP-NEXT: vmovdqa (%r8), %xmm64708; AVX512DQ-FCP-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero4709; AVX512DQ-FCP-NEXT: vpermi2d %ymm0, %ymm3, %ymm224710; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm6[2,1,3,3,4,5,6,7]4711; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm25, %zmm34712; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm22, %zmm224713; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm17, %ymm04714; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm14[0],ymm0[0],ymm14[1],ymm0[1],ymm14[2],ymm0[2],ymm14[3],ymm0[3],ymm14[8],ymm0[8],ymm14[9],ymm0[9],ymm14[10],ymm0[10],ymm14[11],ymm0[11]4715; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]4716; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm3 = [1,1,1,1,10,10,10,11]4717; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm3, %zmm14718; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]4719; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm28, %ymm24720; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm27, %ymm44721; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm4[0],ymm2[0],ymm4[1],ymm2[1],ymm4[2],ymm2[2],ymm4[3],ymm2[3],ymm4[8],ymm2[8],ymm4[9],ymm2[9],ymm4[10],ymm2[10],ymm4[11],ymm2[11]4722; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm4 = [17,18,17,18,0,0,19,19,5,4,2,2,5,4,6,6]4723; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm4, %zmm24724; AVX512DQ-FCP-NEXT: vmovdqa32 %zmm1, %zmm2 {%k1}4725; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]4726; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm6, %xmm14727; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm5 = [0,9,2,3,8,5,6,11]4728; AVX512DQ-FCP-NEXT: vmovdqa %ymm2, %ymm64729; AVX512DQ-FCP-NEXT: vpermt2d %ymm1, %ymm5, %ymm64730; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm12 = [8,9,20,11,12,21,14,15]4731; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm1 = ymm13[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]4732; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm12, %zmm24733; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm6, %zmm64734; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm26, %ymm14735; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm14 = ymm1[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]4736; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm1 = [8,9,12,13,12,13,10,11,8,9,10,11,12,13,14,15]4737; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm11, %xmm24738; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm11 = [0,0,0,1,0,10,10,0]4739; AVX512DQ-FCP-NEXT: vpermt2q %zmm14, %zmm11, %zmm24740; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload4741; AVX512DQ-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload4742; AVX512DQ-FCP-NEXT: # ymm14 = ymm14[0],mem[0],ymm14[1],mem[1],ymm14[2],mem[2],ymm14[3],mem[3],ymm14[8],mem[8],ymm14[9],mem[9],ymm14[10],mem[10],ymm14[11],mem[11]4743; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm7 = xmm7[4],xmm8[4],xmm7[5],xmm8[5],xmm7[6],xmm8[6],xmm7[7],xmm8[7]4744; AVX512DQ-FCP-NEXT: vpermt2q %zmm14, %zmm3, %zmm74745; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm9[4],xmm10[4],xmm9[5],xmm10[5],xmm9[6],xmm10[6],xmm9[7],xmm10[7]4746; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload4747; AVX512DQ-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload4748; AVX512DQ-FCP-NEXT: # ymm8 = ymm8[0],mem[0],ymm8[1],mem[1],ymm8[2],mem[2],ymm8[3],mem[3],ymm8[8],mem[8],ymm8[9],mem[9],ymm8[10],mem[10],ymm8[11],mem[11]4749; AVX512DQ-FCP-NEXT: vpermt2d %zmm3, %zmm4, %zmm84750; AVX512DQ-FCP-NEXT: vmovdqa32 %zmm7, %zmm8 {%k1}4751; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm15, %xmm04752; AVX512DQ-FCP-NEXT: vpermi2d %ymm0, %ymm8, %ymm54753; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm30, %ymm04754; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm0 = ymm0[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]4755; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm12, %zmm84756; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm8, %zmm5, %zmm04757; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm16, %xmm34758; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm3, %xmm14759; AVX512DQ-FCP-NEXT: vmovdqa64 %ymm31, %ymm34760; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm3 = ymm3[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]4761; AVX512DQ-FCP-NEXT: vpermt2q %zmm3, %zmm11, %zmm14762; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]4763; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm3 & (zmm1 ^ zmm0))4764; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax4765; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, 256(%rax)4766; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm3 & (zmm2 ^ zmm6))4767; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, 64(%rax)4768; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm0 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]4769; AVX512DQ-FCP-NEXT: vpternlogq $184, (%rsp), %zmm0, %zmm19 # 64-byte Folded Reload4770; AVX512DQ-FCP-NEXT: # zmm19 = zmm19 ^ (zmm0 & (zmm19 ^ mem))4771; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm20 = zmm20 ^ (zmm0 & (zmm20 ^ zmm29))4772; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]4773; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm23 = zmm23 ^ (zmm0 & (zmm23 ^ zmm21))4774; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm18 = zmm18 ^ (zmm0 & (zmm18 ^ zmm22))4775; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm18, (%rax)4776; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm23, 192(%rax)4777; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm20, 128(%rax)4778; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm19, 320(%rax)4779; AVX512DQ-FCP-NEXT: addq $72, %rsp4780; AVX512DQ-FCP-NEXT: vzeroupper4781; AVX512DQ-FCP-NEXT: retq4782;4783; AVX512BW-LABEL: store_i16_stride6_vf32:4784; AVX512BW: # %bb.0:4785; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax4786; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm24787; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm34788; AVX512BW-NEXT: vmovdqa64 (%rdx), %zmm44789; AVX512BW-NEXT: vmovdqa64 (%rcx), %zmm54790; AVX512BW-NEXT: vmovdqa64 (%r8), %zmm14791; AVX512BW-NEXT: vmovdqa64 (%r9), %zmm04792; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm6 = [0,0,0,32,3,35,0,0,1,33,4,36,0,0,2,34,0,0,0,32,3,35,0,0,1,33,4,36,0,0,2,34]4793; AVX512BW-NEXT: # zmm6 = mem[0,1,2,3,0,1,2,3]4794; AVX512BW-NEXT: vpermi2w %zmm5, %zmm4, %zmm64795; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [0,32,3,35,0,0,1,33,4,36,0,0,2,34,5,37,0,32,3,35,0,0,1,33,4,36,0,0,2,34,5,37]4796; AVX512BW-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]4797; AVX512BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm74798; AVX512BW-NEXT: movw $9362, %cx # imm = 0x24924799; AVX512BW-NEXT: kmovd %ecx, %k24800; AVX512BW-NEXT: vmovdqa32 %zmm6, %zmm7 {%k2}4801; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm8 = [0,1,2,3,32,0,6,7,8,9,33,0,12,13,14,15,34,0,18,19,20,21,35,0,24,25,26,27,36,0,30,31]4802; AVX512BW-NEXT: vpermi2w %zmm1, %zmm7, %zmm84803; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm6 = [0,1,2,3,4,32,6,7,8,9,10,33,12,13,14,15,16,34,18,19,20,21,22,35,24,25,26,27,28,36,30,31]4804; AVX512BW-NEXT: vpermi2w %zmm0, %zmm8, %zmm64805; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [8,40,0,0,6,38,9,41,0,0,7,39,10,42,0,0,8,40,0,0,6,38,9,41,0,0,7,39,10,42,0,0]4806; AVX512BW-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]4807; AVX512BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm74808; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm8 = [5,37,8,40,0,0,6,38,9,41,0,0,7,39,10,42,5,37,8,40,0,0,6,38,9,41,0,0,7,39,10,42]4809; AVX512BW-NEXT: # zmm8 = mem[0,1,2,3,0,1,2,3]4810; AVX512BW-NEXT: vpermi2w %zmm5, %zmm4, %zmm84811; AVX512BW-NEXT: movw $18724, %cx # imm = 0x49244812; AVX512BW-NEXT: kmovd %ecx, %k14813; AVX512BW-NEXT: vmovdqa32 %zmm7, %zmm8 {%k1}4814; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm7 = [0,1,37,0,4,5,6,7,38,0,10,11,12,13,39,0,16,17,18,19,40,0,22,23,24,25,41,0,28,29,30,31]4815; AVX512BW-NEXT: vpermi2w %zmm1, %zmm8, %zmm74816; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm8 = [0,1,2,37,4,5,6,7,8,38,10,11,12,13,14,39,16,17,18,19,20,40,22,23,24,25,26,41,28,29,30,31]4817; AVX512BW-NEXT: vpermi2w %zmm0, %zmm7, %zmm84818; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0]4819; AVX512BW-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]4820; AVX512BW-NEXT: vpermi2w %zmm5, %zmm4, %zmm74821; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm9 = [0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45]4822; AVX512BW-NEXT: # zmm9 = mem[0,1,2,3,0,1,2,3]4823; AVX512BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm94824; AVX512BW-NEXT: vmovdqa32 %zmm7, %zmm9 {%k1}4825; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm7 = [42,0,2,3,4,5,43,0,8,9,10,11,44,0,14,15,16,17,45,0,20,21,22,23,46,0,26,27,28,29,47,0]4826; AVX512BW-NEXT: vpermi2w %zmm1, %zmm9, %zmm74827; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm9 = [0,42,2,3,4,5,6,43,8,9,10,11,12,44,14,15,16,17,18,45,20,21,22,23,24,46,26,27,28,29,30,47]4828; AVX512BW-NEXT: vpermi2w %zmm0, %zmm7, %zmm94829; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [0,0,16,48,19,51,0,0,17,49,20,52,0,0,18,50,0,0,16,48,19,51,0,0,17,49,20,52,0,0,18,50]4830; AVX512BW-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]4831; AVX512BW-NEXT: vpermi2w %zmm5, %zmm4, %zmm74832; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm10 = [16,48,19,51,0,0,17,49,20,52,0,0,18,50,21,53,16,48,19,51,0,0,17,49,20,52,0,0,18,50,21,53]4833; AVX512BW-NEXT: # zmm10 = mem[0,1,2,3,0,1,2,3]4834; AVX512BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm104835; AVX512BW-NEXT: vmovdqa32 %zmm7, %zmm10 {%k2}4836; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm7 = [0,1,2,3,48,0,6,7,8,9,49,0,12,13,14,15,50,0,18,19,20,21,51,0,24,25,26,27,52,0,30,31]4837; AVX512BW-NEXT: vpermi2w %zmm1, %zmm10, %zmm74838; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm10 = [0,1,2,3,4,48,6,7,8,9,10,49,12,13,14,15,16,50,18,19,20,21,22,51,24,25,26,27,28,52,30,31]4839; AVX512BW-NEXT: vpermi2w %zmm0, %zmm7, %zmm104840; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [24,56,0,0,22,54,25,57,0,0,23,55,26,58,0,0,24,56,0,0,22,54,25,57,0,0,23,55,26,58,0,0]4841; AVX512BW-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]4842; AVX512BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm74843; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm11 = [21,53,24,56,0,0,22,54,25,57,0,0,23,55,26,58,21,53,24,56,0,0,22,54,25,57,0,0,23,55,26,58]4844; AVX512BW-NEXT: # zmm11 = mem[0,1,2,3,0,1,2,3]4845; AVX512BW-NEXT: vpermi2w %zmm5, %zmm4, %zmm114846; AVX512BW-NEXT: vmovdqa32 %zmm7, %zmm11 {%k1}4847; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm7 = [0,1,53,0,4,5,6,7,54,0,10,11,12,13,55,0,16,17,18,19,56,0,22,23,24,25,57,0,28,29,30,31]4848; AVX512BW-NEXT: vpermi2w %zmm1, %zmm11, %zmm74849; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm11 = [0,1,2,53,4,5,6,7,8,54,10,11,12,13,14,55,16,17,18,19,20,56,22,23,24,25,26,57,28,29,30,31]4850; AVX512BW-NEXT: vpermi2w %zmm0, %zmm7, %zmm114851; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0]4852; AVX512BW-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]4853; AVX512BW-NEXT: vpermi2w %zmm5, %zmm4, %zmm74854; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm4 = [0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61]4855; AVX512BW-NEXT: # zmm4 = mem[0,1,2,3,0,1,2,3]4856; AVX512BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm44857; AVX512BW-NEXT: vmovdqa32 %zmm7, %zmm4 {%k1}4858; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm2 = [58,0,2,3,4,5,59,0,8,9,10,11,60,0,14,15,16,17,61,0,20,21,22,23,62,0,26,27,28,29,63,0]4859; AVX512BW-NEXT: vpermi2w %zmm1, %zmm4, %zmm24860; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm1 = [0,58,2,3,4,5,6,59,8,9,10,11,12,60,14,15,16,17,18,61,20,21,22,23,24,62,26,27,28,29,30,63]4861; AVX512BW-NEXT: vpermi2w %zmm0, %zmm2, %zmm14862; AVX512BW-NEXT: vmovdqa64 %zmm1, 320(%rax)4863; AVX512BW-NEXT: vmovdqa64 %zmm11, 256(%rax)4864; AVX512BW-NEXT: vmovdqa64 %zmm10, 192(%rax)4865; AVX512BW-NEXT: vmovdqa64 %zmm9, 128(%rax)4866; AVX512BW-NEXT: vmovdqa64 %zmm8, 64(%rax)4867; AVX512BW-NEXT: vmovdqa64 %zmm6, (%rax)4868; AVX512BW-NEXT: vzeroupper4869; AVX512BW-NEXT: retq4870;4871; AVX512BW-FCP-LABEL: store_i16_stride6_vf32:4872; AVX512BW-FCP: # %bb.0:4873; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax4874; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm24875; AVX512BW-FCP-NEXT: vmovdqa64 (%rsi), %zmm34876; AVX512BW-FCP-NEXT: vmovdqa64 (%rdx), %zmm44877; AVX512BW-FCP-NEXT: vmovdqa64 (%rcx), %zmm54878; AVX512BW-FCP-NEXT: vmovdqa64 (%r8), %zmm14879; AVX512BW-FCP-NEXT: vmovdqa64 (%r9), %zmm04880; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm6 = [0,0,0,32,3,35,0,0,1,33,4,36,0,0,2,34,0,0,0,32,3,35,0,0,1,33,4,36,0,0,2,34]4881; AVX512BW-FCP-NEXT: # zmm6 = mem[0,1,2,3,0,1,2,3]4882; AVX512BW-FCP-NEXT: vpermi2w %zmm5, %zmm4, %zmm64883; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [0,32,3,35,0,0,1,33,4,36,0,0,2,34,5,37,0,32,3,35,0,0,1,33,4,36,0,0,2,34,5,37]4884; AVX512BW-FCP-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]4885; AVX512BW-FCP-NEXT: vpermi2w %zmm3, %zmm2, %zmm74886; AVX512BW-FCP-NEXT: movw $9362, %cx # imm = 0x24924887; AVX512BW-FCP-NEXT: kmovd %ecx, %k24888; AVX512BW-FCP-NEXT: vmovdqa32 %zmm6, %zmm7 {%k2}4889; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm8 = [0,1,2,3,32,0,6,7,8,9,33,0,12,13,14,15,34,0,18,19,20,21,35,0,24,25,26,27,36,0,30,31]4890; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm7, %zmm84891; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm6 = [0,1,2,3,4,32,6,7,8,9,10,33,12,13,14,15,16,34,18,19,20,21,22,35,24,25,26,27,28,36,30,31]4892; AVX512BW-FCP-NEXT: vpermi2w %zmm0, %zmm8, %zmm64893; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [8,40,0,0,6,38,9,41,0,0,7,39,10,42,0,0,8,40,0,0,6,38,9,41,0,0,7,39,10,42,0,0]4894; AVX512BW-FCP-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]4895; AVX512BW-FCP-NEXT: vpermi2w %zmm3, %zmm2, %zmm74896; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm8 = [5,37,8,40,0,0,6,38,9,41,0,0,7,39,10,42,5,37,8,40,0,0,6,38,9,41,0,0,7,39,10,42]4897; AVX512BW-FCP-NEXT: # zmm8 = mem[0,1,2,3,0,1,2,3]4898; AVX512BW-FCP-NEXT: vpermi2w %zmm5, %zmm4, %zmm84899; AVX512BW-FCP-NEXT: movw $18724, %cx # imm = 0x49244900; AVX512BW-FCP-NEXT: kmovd %ecx, %k14901; AVX512BW-FCP-NEXT: vmovdqa32 %zmm7, %zmm8 {%k1}4902; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm7 = [0,1,37,0,4,5,6,7,38,0,10,11,12,13,39,0,16,17,18,19,40,0,22,23,24,25,41,0,28,29,30,31]4903; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm8, %zmm74904; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm8 = [0,1,2,37,4,5,6,7,8,38,10,11,12,13,14,39,16,17,18,19,20,40,22,23,24,25,26,41,28,29,30,31]4905; AVX512BW-FCP-NEXT: vpermi2w %zmm0, %zmm7, %zmm84906; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0]4907; AVX512BW-FCP-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]4908; AVX512BW-FCP-NEXT: vpermi2w %zmm5, %zmm4, %zmm74909; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm9 = [0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45]4910; AVX512BW-FCP-NEXT: # zmm9 = mem[0,1,2,3,0,1,2,3]4911; AVX512BW-FCP-NEXT: vpermi2w %zmm3, %zmm2, %zmm94912; AVX512BW-FCP-NEXT: vmovdqa32 %zmm7, %zmm9 {%k1}4913; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm7 = [42,0,2,3,4,5,43,0,8,9,10,11,44,0,14,15,16,17,45,0,20,21,22,23,46,0,26,27,28,29,47,0]4914; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm9, %zmm74915; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm9 = [0,42,2,3,4,5,6,43,8,9,10,11,12,44,14,15,16,17,18,45,20,21,22,23,24,46,26,27,28,29,30,47]4916; AVX512BW-FCP-NEXT: vpermi2w %zmm0, %zmm7, %zmm94917; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [0,0,16,48,19,51,0,0,17,49,20,52,0,0,18,50,0,0,16,48,19,51,0,0,17,49,20,52,0,0,18,50]4918; AVX512BW-FCP-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]4919; AVX512BW-FCP-NEXT: vpermi2w %zmm5, %zmm4, %zmm74920; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm10 = [16,48,19,51,0,0,17,49,20,52,0,0,18,50,21,53,16,48,19,51,0,0,17,49,20,52,0,0,18,50,21,53]4921; AVX512BW-FCP-NEXT: # zmm10 = mem[0,1,2,3,0,1,2,3]4922; AVX512BW-FCP-NEXT: vpermi2w %zmm3, %zmm2, %zmm104923; AVX512BW-FCP-NEXT: vmovdqa32 %zmm7, %zmm10 {%k2}4924; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm7 = [0,1,2,3,48,0,6,7,8,9,49,0,12,13,14,15,50,0,18,19,20,21,51,0,24,25,26,27,52,0,30,31]4925; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm10, %zmm74926; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm10 = [0,1,2,3,4,48,6,7,8,9,10,49,12,13,14,15,16,50,18,19,20,21,22,51,24,25,26,27,28,52,30,31]4927; AVX512BW-FCP-NEXT: vpermi2w %zmm0, %zmm7, %zmm104928; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [24,56,0,0,22,54,25,57,0,0,23,55,26,58,0,0,24,56,0,0,22,54,25,57,0,0,23,55,26,58,0,0]4929; AVX512BW-FCP-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]4930; AVX512BW-FCP-NEXT: vpermi2w %zmm3, %zmm2, %zmm74931; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm11 = [21,53,24,56,0,0,22,54,25,57,0,0,23,55,26,58,21,53,24,56,0,0,22,54,25,57,0,0,23,55,26,58]4932; AVX512BW-FCP-NEXT: # zmm11 = mem[0,1,2,3,0,1,2,3]4933; AVX512BW-FCP-NEXT: vpermi2w %zmm5, %zmm4, %zmm114934; AVX512BW-FCP-NEXT: vmovdqa32 %zmm7, %zmm11 {%k1}4935; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm7 = [0,1,53,0,4,5,6,7,54,0,10,11,12,13,55,0,16,17,18,19,56,0,22,23,24,25,57,0,28,29,30,31]4936; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm11, %zmm74937; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm11 = [0,1,2,53,4,5,6,7,8,54,10,11,12,13,14,55,16,17,18,19,20,56,22,23,24,25,26,57,28,29,30,31]4938; AVX512BW-FCP-NEXT: vpermi2w %zmm0, %zmm7, %zmm114939; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0]4940; AVX512BW-FCP-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]4941; AVX512BW-FCP-NEXT: vpermi2w %zmm5, %zmm4, %zmm74942; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm4 = [0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61]4943; AVX512BW-FCP-NEXT: # zmm4 = mem[0,1,2,3,0,1,2,3]4944; AVX512BW-FCP-NEXT: vpermi2w %zmm3, %zmm2, %zmm44945; AVX512BW-FCP-NEXT: vmovdqa32 %zmm7, %zmm4 {%k1}4946; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm2 = [58,0,2,3,4,5,59,0,8,9,10,11,60,0,14,15,16,17,61,0,20,21,22,23,62,0,26,27,28,29,63,0]4947; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm4, %zmm24948; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm1 = [0,58,2,3,4,5,6,59,8,9,10,11,12,60,14,15,16,17,18,61,20,21,22,23,24,62,26,27,28,29,30,63]4949; AVX512BW-FCP-NEXT: vpermi2w %zmm0, %zmm2, %zmm14950; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, 320(%rax)4951; AVX512BW-FCP-NEXT: vmovdqa64 %zmm11, 256(%rax)4952; AVX512BW-FCP-NEXT: vmovdqa64 %zmm10, 192(%rax)4953; AVX512BW-FCP-NEXT: vmovdqa64 %zmm9, 128(%rax)4954; AVX512BW-FCP-NEXT: vmovdqa64 %zmm8, 64(%rax)4955; AVX512BW-FCP-NEXT: vmovdqa64 %zmm6, (%rax)4956; AVX512BW-FCP-NEXT: vzeroupper4957; AVX512BW-FCP-NEXT: retq4958;4959; AVX512DQ-BW-LABEL: store_i16_stride6_vf32:4960; AVX512DQ-BW: # %bb.0:4961; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %rax4962; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm24963; AVX512DQ-BW-NEXT: vmovdqa64 (%rsi), %zmm34964; AVX512DQ-BW-NEXT: vmovdqa64 (%rdx), %zmm44965; AVX512DQ-BW-NEXT: vmovdqa64 (%rcx), %zmm54966; AVX512DQ-BW-NEXT: vmovdqa64 (%r8), %zmm14967; AVX512DQ-BW-NEXT: vmovdqa64 (%r9), %zmm04968; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm6 = [0,0,0,32,3,35,0,0,1,33,4,36,0,0,2,34,0,0,0,32,3,35,0,0,1,33,4,36,0,0,2,34]4969; AVX512DQ-BW-NEXT: # zmm6 = mem[0,1,2,3,0,1,2,3]4970; AVX512DQ-BW-NEXT: vpermi2w %zmm5, %zmm4, %zmm64971; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [0,32,3,35,0,0,1,33,4,36,0,0,2,34,5,37,0,32,3,35,0,0,1,33,4,36,0,0,2,34,5,37]4972; AVX512DQ-BW-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]4973; AVX512DQ-BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm74974; AVX512DQ-BW-NEXT: movw $9362, %cx # imm = 0x24924975; AVX512DQ-BW-NEXT: kmovd %ecx, %k24976; AVX512DQ-BW-NEXT: vmovdqa32 %zmm6, %zmm7 {%k2}4977; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm8 = [0,1,2,3,32,0,6,7,8,9,33,0,12,13,14,15,34,0,18,19,20,21,35,0,24,25,26,27,36,0,30,31]4978; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm7, %zmm84979; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm6 = [0,1,2,3,4,32,6,7,8,9,10,33,12,13,14,15,16,34,18,19,20,21,22,35,24,25,26,27,28,36,30,31]4980; AVX512DQ-BW-NEXT: vpermi2w %zmm0, %zmm8, %zmm64981; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [8,40,0,0,6,38,9,41,0,0,7,39,10,42,0,0,8,40,0,0,6,38,9,41,0,0,7,39,10,42,0,0]4982; AVX512DQ-BW-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]4983; AVX512DQ-BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm74984; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm8 = [5,37,8,40,0,0,6,38,9,41,0,0,7,39,10,42,5,37,8,40,0,0,6,38,9,41,0,0,7,39,10,42]4985; AVX512DQ-BW-NEXT: # zmm8 = mem[0,1,2,3,0,1,2,3]4986; AVX512DQ-BW-NEXT: vpermi2w %zmm5, %zmm4, %zmm84987; AVX512DQ-BW-NEXT: movw $18724, %cx # imm = 0x49244988; AVX512DQ-BW-NEXT: kmovd %ecx, %k14989; AVX512DQ-BW-NEXT: vmovdqa32 %zmm7, %zmm8 {%k1}4990; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm7 = [0,1,37,0,4,5,6,7,38,0,10,11,12,13,39,0,16,17,18,19,40,0,22,23,24,25,41,0,28,29,30,31]4991; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm8, %zmm74992; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm8 = [0,1,2,37,4,5,6,7,8,38,10,11,12,13,14,39,16,17,18,19,20,40,22,23,24,25,26,41,28,29,30,31]4993; AVX512DQ-BW-NEXT: vpermi2w %zmm0, %zmm7, %zmm84994; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0]4995; AVX512DQ-BW-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]4996; AVX512DQ-BW-NEXT: vpermi2w %zmm5, %zmm4, %zmm74997; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm9 = [0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45]4998; AVX512DQ-BW-NEXT: # zmm9 = mem[0,1,2,3,0,1,2,3]4999; AVX512DQ-BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm95000; AVX512DQ-BW-NEXT: vmovdqa32 %zmm7, %zmm9 {%k1}5001; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm7 = [42,0,2,3,4,5,43,0,8,9,10,11,44,0,14,15,16,17,45,0,20,21,22,23,46,0,26,27,28,29,47,0]5002; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm9, %zmm75003; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm9 = [0,42,2,3,4,5,6,43,8,9,10,11,12,44,14,15,16,17,18,45,20,21,22,23,24,46,26,27,28,29,30,47]5004; AVX512DQ-BW-NEXT: vpermi2w %zmm0, %zmm7, %zmm95005; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [0,0,16,48,19,51,0,0,17,49,20,52,0,0,18,50,0,0,16,48,19,51,0,0,17,49,20,52,0,0,18,50]5006; AVX512DQ-BW-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]5007; AVX512DQ-BW-NEXT: vpermi2w %zmm5, %zmm4, %zmm75008; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm10 = [16,48,19,51,0,0,17,49,20,52,0,0,18,50,21,53,16,48,19,51,0,0,17,49,20,52,0,0,18,50,21,53]5009; AVX512DQ-BW-NEXT: # zmm10 = mem[0,1,2,3,0,1,2,3]5010; AVX512DQ-BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm105011; AVX512DQ-BW-NEXT: vmovdqa32 %zmm7, %zmm10 {%k2}5012; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm7 = [0,1,2,3,48,0,6,7,8,9,49,0,12,13,14,15,50,0,18,19,20,21,51,0,24,25,26,27,52,0,30,31]5013; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm10, %zmm75014; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm10 = [0,1,2,3,4,48,6,7,8,9,10,49,12,13,14,15,16,50,18,19,20,21,22,51,24,25,26,27,28,52,30,31]5015; AVX512DQ-BW-NEXT: vpermi2w %zmm0, %zmm7, %zmm105016; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [24,56,0,0,22,54,25,57,0,0,23,55,26,58,0,0,24,56,0,0,22,54,25,57,0,0,23,55,26,58,0,0]5017; AVX512DQ-BW-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]5018; AVX512DQ-BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm75019; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm11 = [21,53,24,56,0,0,22,54,25,57,0,0,23,55,26,58,21,53,24,56,0,0,22,54,25,57,0,0,23,55,26,58]5020; AVX512DQ-BW-NEXT: # zmm11 = mem[0,1,2,3,0,1,2,3]5021; AVX512DQ-BW-NEXT: vpermi2w %zmm5, %zmm4, %zmm115022; AVX512DQ-BW-NEXT: vmovdqa32 %zmm7, %zmm11 {%k1}5023; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm7 = [0,1,53,0,4,5,6,7,54,0,10,11,12,13,55,0,16,17,18,19,56,0,22,23,24,25,57,0,28,29,30,31]5024; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm11, %zmm75025; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm11 = [0,1,2,53,4,5,6,7,8,54,10,11,12,13,14,55,16,17,18,19,20,56,22,23,24,25,26,57,28,29,30,31]5026; AVX512DQ-BW-NEXT: vpermi2w %zmm0, %zmm7, %zmm115027; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0]5028; AVX512DQ-BW-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]5029; AVX512DQ-BW-NEXT: vpermi2w %zmm5, %zmm4, %zmm75030; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm4 = [0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61]5031; AVX512DQ-BW-NEXT: # zmm4 = mem[0,1,2,3,0,1,2,3]5032; AVX512DQ-BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm45033; AVX512DQ-BW-NEXT: vmovdqa32 %zmm7, %zmm4 {%k1}5034; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm2 = [58,0,2,3,4,5,59,0,8,9,10,11,60,0,14,15,16,17,61,0,20,21,22,23,62,0,26,27,28,29,63,0]5035; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm4, %zmm25036; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm1 = [0,58,2,3,4,5,6,59,8,9,10,11,12,60,14,15,16,17,18,61,20,21,22,23,24,62,26,27,28,29,30,63]5037; AVX512DQ-BW-NEXT: vpermi2w %zmm0, %zmm2, %zmm15038; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, 320(%rax)5039; AVX512DQ-BW-NEXT: vmovdqa64 %zmm11, 256(%rax)5040; AVX512DQ-BW-NEXT: vmovdqa64 %zmm10, 192(%rax)5041; AVX512DQ-BW-NEXT: vmovdqa64 %zmm9, 128(%rax)5042; AVX512DQ-BW-NEXT: vmovdqa64 %zmm8, 64(%rax)5043; AVX512DQ-BW-NEXT: vmovdqa64 %zmm6, (%rax)5044; AVX512DQ-BW-NEXT: vzeroupper5045; AVX512DQ-BW-NEXT: retq5046;5047; AVX512DQ-BW-FCP-LABEL: store_i16_stride6_vf32:5048; AVX512DQ-BW-FCP: # %bb.0:5049; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax5050; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm25051; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rsi), %zmm35052; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdx), %zmm45053; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rcx), %zmm55054; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%r8), %zmm15055; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%r9), %zmm05056; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm6 = [0,0,0,32,3,35,0,0,1,33,4,36,0,0,2,34,0,0,0,32,3,35,0,0,1,33,4,36,0,0,2,34]5057; AVX512DQ-BW-FCP-NEXT: # zmm6 = mem[0,1,2,3,0,1,2,3]5058; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm5, %zmm4, %zmm65059; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [0,32,3,35,0,0,1,33,4,36,0,0,2,34,5,37,0,32,3,35,0,0,1,33,4,36,0,0,2,34,5,37]5060; AVX512DQ-BW-FCP-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]5061; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm3, %zmm2, %zmm75062; AVX512DQ-BW-FCP-NEXT: movw $9362, %cx # imm = 0x24925063; AVX512DQ-BW-FCP-NEXT: kmovd %ecx, %k25064; AVX512DQ-BW-FCP-NEXT: vmovdqa32 %zmm6, %zmm7 {%k2}5065; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm8 = [0,1,2,3,32,0,6,7,8,9,33,0,12,13,14,15,34,0,18,19,20,21,35,0,24,25,26,27,36,0,30,31]5066; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm7, %zmm85067; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm6 = [0,1,2,3,4,32,6,7,8,9,10,33,12,13,14,15,16,34,18,19,20,21,22,35,24,25,26,27,28,36,30,31]5068; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm0, %zmm8, %zmm65069; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [8,40,0,0,6,38,9,41,0,0,7,39,10,42,0,0,8,40,0,0,6,38,9,41,0,0,7,39,10,42,0,0]5070; AVX512DQ-BW-FCP-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]5071; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm3, %zmm2, %zmm75072; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm8 = [5,37,8,40,0,0,6,38,9,41,0,0,7,39,10,42,5,37,8,40,0,0,6,38,9,41,0,0,7,39,10,42]5073; AVX512DQ-BW-FCP-NEXT: # zmm8 = mem[0,1,2,3,0,1,2,3]5074; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm5, %zmm4, %zmm85075; AVX512DQ-BW-FCP-NEXT: movw $18724, %cx # imm = 0x49245076; AVX512DQ-BW-FCP-NEXT: kmovd %ecx, %k15077; AVX512DQ-BW-FCP-NEXT: vmovdqa32 %zmm7, %zmm8 {%k1}5078; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm7 = [0,1,37,0,4,5,6,7,38,0,10,11,12,13,39,0,16,17,18,19,40,0,22,23,24,25,41,0,28,29,30,31]5079; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm8, %zmm75080; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm8 = [0,1,2,37,4,5,6,7,8,38,10,11,12,13,14,39,16,17,18,19,20,40,22,23,24,25,26,41,28,29,30,31]5081; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm0, %zmm7, %zmm85082; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0]5083; AVX512DQ-BW-FCP-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]5084; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm5, %zmm4, %zmm75085; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm9 = [0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45]5086; AVX512DQ-BW-FCP-NEXT: # zmm9 = mem[0,1,2,3,0,1,2,3]5087; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm3, %zmm2, %zmm95088; AVX512DQ-BW-FCP-NEXT: vmovdqa32 %zmm7, %zmm9 {%k1}5089; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm7 = [42,0,2,3,4,5,43,0,8,9,10,11,44,0,14,15,16,17,45,0,20,21,22,23,46,0,26,27,28,29,47,0]5090; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm9, %zmm75091; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm9 = [0,42,2,3,4,5,6,43,8,9,10,11,12,44,14,15,16,17,18,45,20,21,22,23,24,46,26,27,28,29,30,47]5092; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm0, %zmm7, %zmm95093; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [0,0,16,48,19,51,0,0,17,49,20,52,0,0,18,50,0,0,16,48,19,51,0,0,17,49,20,52,0,0,18,50]5094; AVX512DQ-BW-FCP-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]5095; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm5, %zmm4, %zmm75096; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm10 = [16,48,19,51,0,0,17,49,20,52,0,0,18,50,21,53,16,48,19,51,0,0,17,49,20,52,0,0,18,50,21,53]5097; AVX512DQ-BW-FCP-NEXT: # zmm10 = mem[0,1,2,3,0,1,2,3]5098; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm3, %zmm2, %zmm105099; AVX512DQ-BW-FCP-NEXT: vmovdqa32 %zmm7, %zmm10 {%k2}5100; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm7 = [0,1,2,3,48,0,6,7,8,9,49,0,12,13,14,15,50,0,18,19,20,21,51,0,24,25,26,27,52,0,30,31]5101; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm10, %zmm75102; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm10 = [0,1,2,3,4,48,6,7,8,9,10,49,12,13,14,15,16,50,18,19,20,21,22,51,24,25,26,27,28,52,30,31]5103; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm0, %zmm7, %zmm105104; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [24,56,0,0,22,54,25,57,0,0,23,55,26,58,0,0,24,56,0,0,22,54,25,57,0,0,23,55,26,58,0,0]5105; AVX512DQ-BW-FCP-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]5106; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm3, %zmm2, %zmm75107; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm11 = [21,53,24,56,0,0,22,54,25,57,0,0,23,55,26,58,21,53,24,56,0,0,22,54,25,57,0,0,23,55,26,58]5108; AVX512DQ-BW-FCP-NEXT: # zmm11 = mem[0,1,2,3,0,1,2,3]5109; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm5, %zmm4, %zmm115110; AVX512DQ-BW-FCP-NEXT: vmovdqa32 %zmm7, %zmm11 {%k1}5111; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm7 = [0,1,53,0,4,5,6,7,54,0,10,11,12,13,55,0,16,17,18,19,56,0,22,23,24,25,57,0,28,29,30,31]5112; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm11, %zmm75113; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm11 = [0,1,2,53,4,5,6,7,8,54,10,11,12,13,14,55,16,17,18,19,20,56,22,23,24,25,26,57,28,29,30,31]5114; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm0, %zmm7, %zmm115115; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0]5116; AVX512DQ-BW-FCP-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]5117; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm5, %zmm4, %zmm75118; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm4 = [0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61]5119; AVX512DQ-BW-FCP-NEXT: # zmm4 = mem[0,1,2,3,0,1,2,3]5120; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm3, %zmm2, %zmm45121; AVX512DQ-BW-FCP-NEXT: vmovdqa32 %zmm7, %zmm4 {%k1}5122; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm2 = [58,0,2,3,4,5,59,0,8,9,10,11,60,0,14,15,16,17,61,0,20,21,22,23,62,0,26,27,28,29,63,0]5123; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm4, %zmm25124; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm1 = [0,58,2,3,4,5,6,59,8,9,10,11,12,60,14,15,16,17,18,61,20,21,22,23,24,62,26,27,28,29,30,63]5125; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm0, %zmm2, %zmm15126; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, 320(%rax)5127; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm11, 256(%rax)5128; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm10, 192(%rax)5129; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm9, 128(%rax)5130; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm8, 64(%rax)5131; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm6, (%rax)5132; AVX512DQ-BW-FCP-NEXT: vzeroupper5133; AVX512DQ-BW-FCP-NEXT: retq5134 %in.vec0 = load <32 x i16>, ptr %in.vecptr0, align 645135 %in.vec1 = load <32 x i16>, ptr %in.vecptr1, align 645136 %in.vec2 = load <32 x i16>, ptr %in.vecptr2, align 645137 %in.vec3 = load <32 x i16>, ptr %in.vecptr3, align 645138 %in.vec4 = load <32 x i16>, ptr %in.vecptr4, align 645139 %in.vec5 = load <32 x i16>, ptr %in.vecptr5, align 645140 %1 = shufflevector <32 x i16> %in.vec0, <32 x i16> %in.vec1, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>5141 %2 = shufflevector <32 x i16> %in.vec2, <32 x i16> %in.vec3, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>5142 %3 = shufflevector <32 x i16> %in.vec4, <32 x i16> %in.vec5, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>5143 %4 = shufflevector <64 x i16> %1, <64 x i16> %2, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>5144 %5 = shufflevector <64 x i16> %3, <64 x i16> poison, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>5145 %6 = shufflevector <128 x i16> %4, <128 x i16> %5, <192 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127, i32 128, i32 129, i32 130, i32 131, i32 132, i32 133, i32 134, i32 135, i32 136, i32 137, i32 138, i32 139, i32 140, i32 141, i32 142, i32 143, i32 144, i32 145, i32 146, i32 147, i32 148, i32 149, i32 150, i32 151, i32 152, i32 153, i32 154, i32 155, i32 156, i32 157, i32 158, i32 159, i32 160, i32 161, i32 162, i32 163, i32 164, i32 165, i32 166, i32 167, i32 168, i32 169, i32 170, i32 171, i32 172, i32 173, i32 174, i32 175, i32 176, i32 177, i32 178, i32 179, i32 180, i32 181, i32 182, i32 183, i32 184, i32 185, i32 186, i32 187, i32 188, i32 189, i32 190, i32 191>5146 %interleaved.vec = shufflevector <192 x i16> %6, <192 x i16> poison, <192 x i32> <i32 0, i32 32, i32 64, i32 96, i32 128, i32 160, i32 1, i32 33, i32 65, i32 97, i32 129, i32 161, i32 2, i32 34, i32 66, i32 98, i32 130, i32 162, i32 3, i32 35, i32 67, i32 99, i32 131, i32 163, i32 4, i32 36, i32 68, i32 100, i32 132, i32 164, i32 5, i32 37, i32 69, i32 101, i32 133, i32 165, i32 6, i32 38, i32 70, i32 102, i32 134, i32 166, i32 7, i32 39, i32 71, i32 103, i32 135, i32 167, i32 8, i32 40, i32 72, i32 104, i32 136, i32 168, i32 9, i32 41, i32 73, i32 105, i32 137, i32 169, i32 10, i32 42, i32 74, i32 106, i32 138, i32 170, i32 11, i32 43, i32 75, i32 107, i32 139, i32 171, i32 12, i32 44, i32 76, i32 108, i32 140, i32 172, i32 13, i32 45, i32 77, i32 109, i32 141, i32 173, i32 14, i32 46, i32 78, i32 110, i32 142, i32 174, i32 15, i32 47, i32 79, i32 111, i32 143, i32 175, i32 16, i32 48, i32 80, i32 112, i32 144, i32 176, i32 17, i32 49, i32 81, i32 113, i32 145, i32 177, i32 18, i32 50, i32 82, i32 114, i32 146, i32 178, i32 19, i32 51, i32 83, i32 115, i32 147, i32 179, i32 20, i32 52, i32 84, i32 116, i32 148, i32 180, i32 21, i32 53, i32 85, i32 117, i32 149, i32 181, i32 22, i32 54, i32 86, i32 118, i32 150, i32 182, i32 23, i32 55, i32 87, i32 119, i32 151, i32 183, i32 24, i32 56, i32 88, i32 120, i32 152, i32 184, i32 25, i32 57, i32 89, i32 121, i32 153, i32 185, i32 26, i32 58, i32 90, i32 122, i32 154, i32 186, i32 27, i32 59, i32 91, i32 123, i32 155, i32 187, i32 28, i32 60, i32 92, i32 124, i32 156, i32 188, i32 29, i32 61, i32 93, i32 125, i32 157, i32 189, i32 30, i32 62, i32 94, i32 126, i32 158, i32 190, i32 31, i32 63, i32 95, i32 127, i32 159, i32 191>5147 store <192 x i16> %interleaved.vec, ptr %out.vec, align 645148 ret void5149}5150 5151define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %in.vecptr3, ptr %in.vecptr4, ptr %in.vecptr5, ptr %out.vec) nounwind {5152; SSE-LABEL: store_i16_stride6_vf64:5153; SSE: # %bb.0:5154; SSE-NEXT: subq $808, %rsp # imm = 0x3285155; SSE-NEXT: movdqa (%rdi), %xmm105156; SSE-NEXT: movdqa 16(%rdi), %xmm115157; SSE-NEXT: movdqa (%rsi), %xmm45158; SSE-NEXT: movdqa 16(%rsi), %xmm15159; SSE-NEXT: movdqa (%rdx), %xmm125160; SSE-NEXT: movdqa 16(%rdx), %xmm25161; SSE-NEXT: movdqa (%rcx), %xmm65162; SSE-NEXT: movdqa 16(%rcx), %xmm35163; SSE-NEXT: movdqa (%r8), %xmm95164; SSE-NEXT: movdqa (%r9), %xmm85165; SSE-NEXT: movdqa %xmm12, %xmm05166; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3]5167; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5168; SSE-NEXT: movdqa %xmm10, %xmm75169; SSE-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3]5170; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5171; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[2,3],xmm0[3,3]5172; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm9[2,1,3,3,4,5,6,7]5173; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[1,2],xmm5[0,1]5174; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[2,0,1,3]5175; SSE-NEXT: movaps {{.*#+}} xmm14 = [65535,0,65535,65535,65535,65535,65535,0]5176; SSE-NEXT: andps %xmm14, %xmm75177; SSE-NEXT: movdqa %xmm8, %xmm55178; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5179; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[0,2,2,3,4,5,6,7]5180; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,1,2,1]5181; SSE-NEXT: movaps %xmm14, %xmm05182; SSE-NEXT: andnps %xmm8, %xmm05183; SSE-NEXT: orps %xmm7, %xmm05184; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5185; SSE-NEXT: punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm6[4],xmm12[5],xmm6[5],xmm12[6],xmm6[6],xmm12[7],xmm6[7]5186; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5187; SSE-NEXT: punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]5188; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5189; SSE-NEXT: movdqa %xmm10, %xmm45190; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,3],xmm12[3,3]5191; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm9[0,1,2,3,6,5,7,7]5192; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[1,2],xmm6[2,3]5193; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0,1,3]5194; SSE-NEXT: andps %xmm14, %xmm45195; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm5[0,1,2,3,4,6,6,7]5196; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[2,1,2,3]5197; SSE-NEXT: movaps %xmm14, %xmm05198; SSE-NEXT: andnps %xmm6, %xmm05199; SSE-NEXT: orps %xmm4, %xmm05200; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5201; SSE-NEXT: movdqa %xmm2, %xmm05202; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]5203; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5204; SSE-NEXT: movdqa %xmm11, %xmm45205; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]5206; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5207; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,3],xmm0[3,3]5208; SSE-NEXT: movdqa 16(%r8), %xmm55209; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm5[2,1,3,3,4,5,6,7]5210; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5211; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[1,2],xmm6[0,1]5212; SSE-NEXT: movdqa 16(%r9), %xmm75213; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm7[0,2,2,3,4,5,6,7]5214; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5215; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,1,2,1]5216; SSE-NEXT: movaps %xmm14, %xmm05217; SSE-NEXT: andnps %xmm6, %xmm05218; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0,1,3]5219; SSE-NEXT: andps %xmm14, %xmm45220; SSE-NEXT: orps %xmm4, %xmm05221; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5222; SSE-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]5223; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5224; SSE-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm1[4],xmm11[5],xmm1[5],xmm11[6],xmm1[6],xmm11[7],xmm1[7]5225; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5226; SSE-NEXT: movdqa %xmm11, %xmm15227; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,3],xmm2[3,3]5228; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm5[0,1,2,3,6,5,7,7]5229; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,2],xmm3[2,3]5230; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm7[0,1,2,3,4,6,6,7]5231; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,1,2,3]5232; SSE-NEXT: movaps %xmm14, %xmm05233; SSE-NEXT: andnps %xmm3, %xmm05234; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0,1,3]5235; SSE-NEXT: andps %xmm14, %xmm15236; SSE-NEXT: orps %xmm1, %xmm05237; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5238; SSE-NEXT: movdqa 32(%rdx), %xmm25239; SSE-NEXT: movdqa 32(%rcx), %xmm15240; SSE-NEXT: movdqa %xmm2, %xmm05241; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]5242; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5243; SSE-NEXT: movdqa 32(%rdi), %xmm35244; SSE-NEXT: movdqa 32(%rsi), %xmm65245; SSE-NEXT: movdqa %xmm3, %xmm75246; SSE-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]5247; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5248; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[2,3],xmm0[3,3]5249; SSE-NEXT: movdqa 32(%r8), %xmm45250; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm4[2,1,3,3,4,5,6,7]5251; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5252; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[1,2],xmm8[0,1]5253; SSE-NEXT: movdqa 32(%r9), %xmm55254; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm5[0,2,2,3,4,5,6,7]5255; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5256; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,1,2,1]5257; SSE-NEXT: movaps %xmm14, %xmm05258; SSE-NEXT: andnps %xmm8, %xmm05259; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[2,0,1,3]5260; SSE-NEXT: andps %xmm14, %xmm75261; SSE-NEXT: orps %xmm7, %xmm05262; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5263; SSE-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]5264; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5265; SSE-NEXT: movdqa %xmm3, %xmm15266; SSE-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7]5267; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5268; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,3],xmm2[3,3]5269; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm4[0,1,2,3,6,5,7,7]5270; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,2],xmm6[2,3]5271; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm5[0,1,2,3,4,6,6,7]5272; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[2,1,2,3]5273; SSE-NEXT: movaps %xmm14, %xmm05274; SSE-NEXT: andnps %xmm6, %xmm05275; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0,1,3]5276; SSE-NEXT: andps %xmm14, %xmm15277; SSE-NEXT: orps %xmm1, %xmm05278; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5279; SSE-NEXT: movdqa 48(%rdx), %xmm25280; SSE-NEXT: movdqa 48(%rcx), %xmm15281; SSE-NEXT: movdqa %xmm2, %xmm05282; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]5283; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5284; SSE-NEXT: movdqa 48(%rdi), %xmm35285; SSE-NEXT: movdqa 48(%rsi), %xmm75286; SSE-NEXT: movdqa %xmm3, %xmm85287; SSE-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]5288; SSE-NEXT: movdqa %xmm8, (%rsp) # 16-byte Spill5289; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[2,3],xmm0[3,3]5290; SSE-NEXT: movdqa 48(%r8), %xmm65291; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm6[2,1,3,3,4,5,6,7]5292; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[1,2],xmm11[0,1]5293; SSE-NEXT: movdqa 48(%r9), %xmm45294; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm4[0,2,2,3,4,5,6,7]5295; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5296; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm11[0,1,2,1]5297; SSE-NEXT: movaps %xmm14, %xmm05298; SSE-NEXT: andnps %xmm11, %xmm05299; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[2,0,1,3]5300; SSE-NEXT: andps %xmm14, %xmm85301; SSE-NEXT: orps %xmm8, %xmm05302; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5303; SSE-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]5304; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5305; SSE-NEXT: movdqa %xmm3, %xmm15306; SSE-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm7[4],xmm1[5],xmm7[5],xmm1[6],xmm7[6],xmm1[7],xmm7[7]5307; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5308; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,3],xmm2[3,3]5309; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm6[0,1,2,3,6,5,7,7]5310; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,2],xmm7[2,3]5311; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm4[0,1,2,3,4,6,6,7]5312; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[2,1,2,3]5313; SSE-NEXT: movaps %xmm14, %xmm05314; SSE-NEXT: andnps %xmm7, %xmm05315; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0,1,3]5316; SSE-NEXT: andps %xmm14, %xmm15317; SSE-NEXT: orps %xmm1, %xmm05318; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5319; SSE-NEXT: movdqa 64(%rdx), %xmm25320; SSE-NEXT: movdqa 64(%rcx), %xmm15321; SSE-NEXT: movdqa %xmm2, %xmm05322; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]5323; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5324; SSE-NEXT: movdqa 64(%rdi), %xmm35325; SSE-NEXT: movdqa 64(%rsi), %xmm85326; SSE-NEXT: movdqa %xmm3, %xmm115327; SSE-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm8[0],xmm11[1],xmm8[1],xmm11[2],xmm8[2],xmm11[3],xmm8[3]5328; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5329; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[2,3],xmm0[3,3]5330; SSE-NEXT: movdqa 64(%r8), %xmm75331; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm7[2,1,3,3,4,5,6,7]5332; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[1,2],xmm12[0,1]5333; SSE-NEXT: movdqa 64(%r9), %xmm45334; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm4[0,2,2,3,4,5,6,7]5335; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5336; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm12[0,1,2,1]5337; SSE-NEXT: movaps %xmm14, %xmm05338; SSE-NEXT: andnps %xmm12, %xmm05339; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[2,0,1,3]5340; SSE-NEXT: andps %xmm14, %xmm115341; SSE-NEXT: orps %xmm11, %xmm05342; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5343; SSE-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]5344; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5345; SSE-NEXT: movdqa %xmm3, %xmm15346; SSE-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7]5347; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5348; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,3],xmm2[3,3]5349; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm7[0,1,2,3,6,5,7,7]5350; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,2],xmm8[2,3]5351; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm4[0,1,2,3,4,6,6,7]5352; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[2,1,2,3]5353; SSE-NEXT: movaps %xmm14, %xmm05354; SSE-NEXT: andnps %xmm8, %xmm05355; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0,1,3]5356; SSE-NEXT: andps %xmm14, %xmm15357; SSE-NEXT: orps %xmm1, %xmm05358; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5359; SSE-NEXT: movdqa 80(%rdx), %xmm25360; SSE-NEXT: movdqa 80(%rcx), %xmm15361; SSE-NEXT: movdqa %xmm2, %xmm05362; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]5363; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5364; SSE-NEXT: movdqa 80(%rdi), %xmm35365; SSE-NEXT: movdqa 80(%rsi), %xmm115366; SSE-NEXT: movdqa %xmm3, %xmm125367; SSE-NEXT: punpcklwd {{.*#+}} xmm12 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]5368; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5369; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[2,3],xmm0[3,3]5370; SSE-NEXT: movdqa 80(%r8), %xmm85371; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm8[2,1,3,3,4,5,6,7]5372; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[1,2],xmm15[0,1]5373; SSE-NEXT: movdqa 80(%r9), %xmm45374; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm4[0,2,2,3,4,5,6,7]5375; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5376; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm15[0,1,2,1]5377; SSE-NEXT: movaps %xmm14, %xmm05378; SSE-NEXT: andnps %xmm15, %xmm05379; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[2,0,1,3]5380; SSE-NEXT: andps %xmm14, %xmm125381; SSE-NEXT: orps %xmm12, %xmm05382; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5383; SSE-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]5384; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5385; SSE-NEXT: movdqa %xmm3, %xmm15386; SSE-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm11[4],xmm1[5],xmm11[5],xmm1[6],xmm11[6],xmm1[7],xmm11[7]5387; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5388; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,3],xmm2[3,3]5389; SSE-NEXT: pshufhw {{.*#+}} xmm11 = xmm8[0,1,2,3,6,5,7,7]5390; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,2],xmm11[2,3]5391; SSE-NEXT: pshufhw {{.*#+}} xmm11 = xmm4[0,1,2,3,4,6,6,7]5392; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm11[2,1,2,3]5393; SSE-NEXT: movaps %xmm14, %xmm05394; SSE-NEXT: andnps %xmm11, %xmm05395; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0,1,3]5396; SSE-NEXT: andps %xmm14, %xmm15397; SSE-NEXT: orps %xmm1, %xmm05398; SSE-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5399; SSE-NEXT: movdqa 96(%rdx), %xmm25400; SSE-NEXT: movdqa 96(%rcx), %xmm15401; SSE-NEXT: movdqa %xmm2, %xmm05402; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]5403; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5404; SSE-NEXT: movdqa 96(%rdi), %xmm35405; SSE-NEXT: movdqa 96(%rsi), %xmm125406; SSE-NEXT: movdqa %xmm3, %xmm155407; SSE-NEXT: punpcklwd {{.*#+}} xmm15 = xmm15[0],xmm12[0],xmm15[1],xmm12[1],xmm15[2],xmm12[2],xmm15[3],xmm12[3]5408; SSE-NEXT: movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5409; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[2,3],xmm0[3,3]5410; SSE-NEXT: movdqa 96(%r8), %xmm115411; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm11[2,1,3,3,4,5,6,7]5412; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[1,2],xmm0[0,1]5413; SSE-NEXT: movdqa 96(%r9), %xmm105414; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm10[0,2,2,3,4,5,6,7]5415; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5416; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1]5417; SSE-NEXT: movaps %xmm14, %xmm135418; SSE-NEXT: andnps %xmm0, %xmm135419; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[2,0,1,3]5420; SSE-NEXT: andps %xmm14, %xmm155421; SSE-NEXT: orps %xmm15, %xmm135422; SSE-NEXT: movaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5423; SSE-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]5424; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5425; SSE-NEXT: movdqa %xmm3, %xmm05426; SSE-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm12[4],xmm0[5],xmm12[5],xmm0[6],xmm12[6],xmm0[7],xmm12[7]5427; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5428; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,3],xmm2[3,3]5429; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm11[0,1,2,3,6,5,7,7]5430; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,2],xmm1[2,3]5431; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm10[0,1,2,3,4,6,6,7]5432; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]5433; SSE-NEXT: movaps %xmm14, %xmm125434; SSE-NEXT: andnps %xmm1, %xmm125435; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0,1,3]5436; SSE-NEXT: andps %xmm14, %xmm05437; SSE-NEXT: orps %xmm0, %xmm125438; SSE-NEXT: movaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5439; SSE-NEXT: movdqa 112(%rdx), %xmm45440; SSE-NEXT: movdqa 112(%rcx), %xmm55441; SSE-NEXT: movdqa %xmm4, %xmm25442; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]5443; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5444; SSE-NEXT: movdqa 112(%rdi), %xmm05445; SSE-NEXT: movdqa 112(%rsi), %xmm15446; SSE-NEXT: movdqa %xmm0, %xmm155447; SSE-NEXT: punpcklwd {{.*#+}} xmm15 = xmm15[0],xmm1[0],xmm15[1],xmm1[1],xmm15[2],xmm1[2],xmm15[3],xmm1[3]5448; SSE-NEXT: movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5449; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[2,3],xmm2[3,3]5450; SSE-NEXT: movdqa 112(%r8), %xmm35451; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm3[2,1,3,3,4,5,6,7]5452; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5453; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[1,2],xmm13[0,1]5454; SSE-NEXT: movdqa 112(%r9), %xmm25455; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm2[0,2,2,3,4,5,6,7]5456; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm13[0,1,2,1]5457; SSE-NEXT: movaps %xmm14, %xmm125458; SSE-NEXT: andnps %xmm13, %xmm125459; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[2,0,1,3]5460; SSE-NEXT: andps %xmm14, %xmm155461; SSE-NEXT: orps %xmm15, %xmm125462; SSE-NEXT: movaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5463; SSE-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]5464; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5465; SSE-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]5466; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5467; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,3],xmm4[3,3]5468; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm3[0,1,2,3,6,5,7,7]5469; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,2],xmm1[2,3]5470; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0,1,3]5471; SSE-NEXT: andps %xmm14, %xmm05472; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm2[0,1,2,3,4,6,6,7]5473; SSE-NEXT: movdqa %xmm2, %xmm155474; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]5475; SSE-NEXT: andnps %xmm1, %xmm145476; SSE-NEXT: orps %xmm0, %xmm145477; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload5478; SSE-NEXT: movaps %xmm4, %xmm05479; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload5480; SSE-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]5481; SSE-NEXT: movdqa %xmm9, %xmm15482; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm3[1,3]5483; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[0,2]5484; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload5485; SSE-NEXT: movdqa %xmm2, %xmm135486; SSE-NEXT: pslldq {{.*#+}} xmm13 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm13[0,1,2,3,4,5]5487; SSE-NEXT: movdqa {{.*#+}} xmm12 = [65535,65535,65535,65535,65535,0,65535,65535]5488; SSE-NEXT: movdqa %xmm12, %xmm15489; SSE-NEXT: pandn %xmm13, %xmm15490; SSE-NEXT: andps %xmm12, %xmm05491; SSE-NEXT: por %xmm0, %xmm15492; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5493; SSE-NEXT: unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm4[1]5494; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm9[1,1,1,1,4,5,6,7]5495; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm4[1,1]5496; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm3[0,2]5497; SSE-NEXT: movdqa {{.*#+}} xmm10 = [65535,65535,65535,0,65535,65535,65535,65535]5498; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm2[0,0,1,1]5499; SSE-NEXT: movdqa %xmm10, %xmm15500; SSE-NEXT: pandn %xmm13, %xmm15501; SSE-NEXT: andps %xmm10, %xmm05502; SSE-NEXT: por %xmm0, %xmm15503; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5504; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload5505; SSE-NEXT: movaps %xmm4, %xmm05506; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload5507; SSE-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]5508; SSE-NEXT: movdqa %xmm9, %xmm135509; SSE-NEXT: shufps {{.*#+}} xmm13 = xmm13[2,1],xmm3[1,3]5510; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm13[0,2]5511; SSE-NEXT: movdqa %xmm2, %xmm15512; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm2[2,2,3,3]5513; SSE-NEXT: pslld $16, %xmm15514; SSE-NEXT: movdqa %xmm1, %xmm25515; SSE-NEXT: movdqa %xmm12, %xmm15516; SSE-NEXT: pandn %xmm2, %xmm15517; SSE-NEXT: andps %xmm12, %xmm05518; SSE-NEXT: por %xmm0, %xmm15519; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5520; SSE-NEXT: unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm4[1]5521; SSE-NEXT: psrldq {{.*#+}} xmm9 = xmm9[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero5522; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[1,1],xmm4[1,1]5523; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[2,0],xmm3[0,2]5524; SSE-NEXT: movdqa %xmm10, %xmm05525; SSE-NEXT: pandn %xmm13, %xmm05526; SSE-NEXT: andps %xmm10, %xmm95527; SSE-NEXT: por %xmm9, %xmm05528; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5529; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload5530; SSE-NEXT: movaps %xmm9, %xmm05531; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload5532; SSE-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm5[0]5533; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload5534; SSE-NEXT: movaps %xmm3, %xmm25535; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm5[1,3]5536; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[0,2]5537; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload5538; SSE-NEXT: movdqa %xmm4, %xmm25539; SSE-NEXT: pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2,3,4,5]5540; SSE-NEXT: movdqa %xmm12, %xmm15541; SSE-NEXT: pandn %xmm2, %xmm15542; SSE-NEXT: andps %xmm12, %xmm05543; SSE-NEXT: por %xmm0, %xmm15544; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5545; SSE-NEXT: unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm9[1]5546; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm3[1,1,1,1,4,5,6,7]5547; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm9[1,1]5548; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm5[0,2]5549; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,0,1,1]5550; SSE-NEXT: movdqa %xmm4, %xmm55551; SSE-NEXT: movdqa %xmm10, %xmm15552; SSE-NEXT: pandn %xmm2, %xmm15553; SSE-NEXT: andps %xmm10, %xmm05554; SSE-NEXT: por %xmm0, %xmm15555; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5556; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload5557; SSE-NEXT: movaps %xmm9, %xmm05558; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload5559; SSE-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm4[0]5560; SSE-NEXT: movaps %xmm3, %xmm25561; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,1],xmm4[1,3]5562; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[0,2]5563; SSE-NEXT: movdqa %xmm5, %xmm15564; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm5[2,2,3,3]5565; SSE-NEXT: pslld $16, %xmm15566; SSE-NEXT: movdqa %xmm12, %xmm55567; SSE-NEXT: pandn %xmm1, %xmm55568; SSE-NEXT: andps %xmm12, %xmm05569; SSE-NEXT: por %xmm0, %xmm55570; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5571; SSE-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm9[1]5572; SSE-NEXT: movdqa %xmm3, %xmm05573; SSE-NEXT: psrldq {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero5574; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm9[1,1]5575; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm4[0,2]5576; SSE-NEXT: movdqa %xmm10, %xmm15577; SSE-NEXT: pandn %xmm2, %xmm15578; SSE-NEXT: andps %xmm10, %xmm05579; SSE-NEXT: por %xmm0, %xmm15580; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5581; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload5582; SSE-NEXT: movaps %xmm5, %xmm05583; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload5584; SSE-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm4[0]5585; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload5586; SSE-NEXT: movaps %xmm1, %xmm25587; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm4[1,3]5588; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[0,2]5589; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload5590; SSE-NEXT: movdqa %xmm3, %xmm25591; SSE-NEXT: pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2,3,4,5]5592; SSE-NEXT: movdqa %xmm12, %xmm95593; SSE-NEXT: pandn %xmm2, %xmm95594; SSE-NEXT: andps %xmm12, %xmm05595; SSE-NEXT: por %xmm0, %xmm95596; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5597; SSE-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm5[1]5598; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm1[1,1,1,1,4,5,6,7]5599; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm5[1,1]5600; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm4[0,2]5601; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,0,1,1]5602; SSE-NEXT: movdqa %xmm10, %xmm45603; SSE-NEXT: pandn %xmm2, %xmm45604; SSE-NEXT: andps %xmm10, %xmm05605; SSE-NEXT: por %xmm0, %xmm45606; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5607; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload5608; SSE-NEXT: movaps %xmm5, %xmm05609; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload5610; SSE-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm4[0]5611; SSE-NEXT: movaps %xmm1, %xmm25612; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,1],xmm4[1,3]5613; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[0,2]5614; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,2,3,3]5615; SSE-NEXT: pslld $16, %xmm35616; SSE-NEXT: movdqa %xmm12, %xmm95617; SSE-NEXT: pandn %xmm3, %xmm95618; SSE-NEXT: andps %xmm12, %xmm05619; SSE-NEXT: por %xmm0, %xmm95620; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5621; SSE-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm5[1]5622; SSE-NEXT: movdqa %xmm1, %xmm05623; SSE-NEXT: psrldq {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero5624; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm5[1,1]5625; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm4[0,2]5626; SSE-NEXT: movdqa %xmm10, %xmm15627; SSE-NEXT: pandn %xmm2, %xmm15628; SSE-NEXT: andps %xmm10, %xmm05629; SSE-NEXT: por %xmm0, %xmm15630; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5631; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload5632; SSE-NEXT: movaps %xmm4, %xmm05633; SSE-NEXT: movaps (%rsp), %xmm3 # 16-byte Reload5634; SSE-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]5635; SSE-NEXT: movdqa %xmm6, %xmm25636; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm3[1,3]5637; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[0,2]5638; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload5639; SSE-NEXT: movdqa %xmm1, %xmm25640; SSE-NEXT: pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2,3,4,5]5641; SSE-NEXT: movdqa %xmm12, %xmm55642; SSE-NEXT: pandn %xmm2, %xmm55643; SSE-NEXT: andps %xmm12, %xmm05644; SSE-NEXT: por %xmm0, %xmm55645; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5646; SSE-NEXT: unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm4[1]5647; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm6[1,1,1,1,4,5,6,7]5648; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm4[1,1]5649; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm3[0,2]5650; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,0,1,1]5651; SSE-NEXT: movdqa %xmm10, %xmm35652; SSE-NEXT: pandn %xmm2, %xmm35653; SSE-NEXT: andps %xmm10, %xmm05654; SSE-NEXT: por %xmm0, %xmm35655; SSE-NEXT: movdqa %xmm3, (%rsp) # 16-byte Spill5656; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload5657; SSE-NEXT: movaps %xmm4, %xmm05658; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload5659; SSE-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]5660; SSE-NEXT: movdqa %xmm6, %xmm25661; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,1],xmm3[1,3]5662; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[0,2]5663; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,2,3,3]5664; SSE-NEXT: pslld $16, %xmm15665; SSE-NEXT: movdqa %xmm12, %xmm55666; SSE-NEXT: pandn %xmm1, %xmm55667; SSE-NEXT: andps %xmm12, %xmm05668; SSE-NEXT: por %xmm0, %xmm55669; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5670; SSE-NEXT: unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm4[1]5671; SSE-NEXT: psrldq {{.*#+}} xmm6 = xmm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero5672; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[1,1],xmm4[1,1]5673; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[2,0],xmm3[0,2]5674; SSE-NEXT: movdqa %xmm10, %xmm05675; SSE-NEXT: pandn %xmm2, %xmm05676; SSE-NEXT: andps %xmm10, %xmm65677; SSE-NEXT: por %xmm6, %xmm05678; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5679; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload5680; SSE-NEXT: movaps %xmm4, %xmm05681; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload5682; SSE-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]5683; SSE-NEXT: movdqa %xmm7, %xmm25684; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm3[1,3]5685; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[0,2]5686; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload5687; SSE-NEXT: movdqa %xmm1, %xmm25688; SSE-NEXT: pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2,3,4,5]5689; SSE-NEXT: movdqa %xmm12, %xmm55690; SSE-NEXT: pandn %xmm2, %xmm55691; SSE-NEXT: andps %xmm12, %xmm05692; SSE-NEXT: por %xmm0, %xmm55693; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5694; SSE-NEXT: unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm4[1]5695; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm7[1,1,1,1,4,5,6,7]5696; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm4[1,1]5697; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm3[0,2]5698; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,0,1,1]5699; SSE-NEXT: movdqa %xmm10, %xmm35700; SSE-NEXT: pandn %xmm2, %xmm35701; SSE-NEXT: andps %xmm10, %xmm05702; SSE-NEXT: por %xmm0, %xmm35703; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5704; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload5705; SSE-NEXT: movaps %xmm4, %xmm05706; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload5707; SSE-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]5708; SSE-NEXT: movdqa %xmm7, %xmm25709; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,1],xmm3[1,3]5710; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[0,2]5711; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,2,3,3]5712; SSE-NEXT: pslld $16, %xmm15713; SSE-NEXT: movdqa %xmm12, %xmm55714; SSE-NEXT: pandn %xmm1, %xmm55715; SSE-NEXT: andps %xmm12, %xmm05716; SSE-NEXT: por %xmm0, %xmm55717; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5718; SSE-NEXT: unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm4[1]5719; SSE-NEXT: psrldq {{.*#+}} xmm7 = xmm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero5720; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[1,1],xmm4[1,1]5721; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[2,0],xmm3[0,2]5722; SSE-NEXT: movdqa %xmm10, %xmm05723; SSE-NEXT: pandn %xmm2, %xmm05724; SSE-NEXT: andps %xmm10, %xmm75725; SSE-NEXT: por %xmm7, %xmm05726; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5727; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload5728; SSE-NEXT: movaps %xmm4, %xmm05729; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload5730; SSE-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]5731; SSE-NEXT: movdqa %xmm8, %xmm25732; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm3[1,3]5733; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[0,2]5734; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload5735; SSE-NEXT: movdqa %xmm1, %xmm25736; SSE-NEXT: pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2,3,4,5]5737; SSE-NEXT: movdqa %xmm12, %xmm55738; SSE-NEXT: pandn %xmm2, %xmm55739; SSE-NEXT: andps %xmm12, %xmm05740; SSE-NEXT: por %xmm0, %xmm55741; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5742; SSE-NEXT: unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm4[1]5743; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm8[1,1,1,1,4,5,6,7]5744; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm4[1,1]5745; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm3[0,2]5746; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,0,1,1]5747; SSE-NEXT: movdqa %xmm10, %xmm35748; SSE-NEXT: pandn %xmm2, %xmm35749; SSE-NEXT: andps %xmm10, %xmm05750; SSE-NEXT: por %xmm0, %xmm35751; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5752; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload5753; SSE-NEXT: movaps %xmm4, %xmm05754; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload5755; SSE-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]5756; SSE-NEXT: movdqa %xmm8, %xmm25757; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,1],xmm3[1,3]5758; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[0,2]5759; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,2,3,3]5760; SSE-NEXT: pslld $16, %xmm15761; SSE-NEXT: movdqa %xmm12, %xmm55762; SSE-NEXT: pandn %xmm1, %xmm55763; SSE-NEXT: andps %xmm12, %xmm05764; SSE-NEXT: por %xmm0, %xmm55765; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5766; SSE-NEXT: unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm4[1]5767; SSE-NEXT: psrldq {{.*#+}} xmm8 = xmm8[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero5768; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[1,1],xmm4[1,1]5769; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[2,0],xmm3[0,2]5770; SSE-NEXT: movdqa %xmm10, %xmm05771; SSE-NEXT: pandn %xmm2, %xmm05772; SSE-NEXT: andps %xmm10, %xmm85773; SSE-NEXT: por %xmm8, %xmm05774; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5775; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload5776; SSE-NEXT: movaps %xmm4, %xmm05777; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload5778; SSE-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]5779; SSE-NEXT: movdqa %xmm11, %xmm25780; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm3[1,3]5781; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[0,2]5782; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload5783; SSE-NEXT: movdqa %xmm1, %xmm25784; SSE-NEXT: pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2,3,4,5]5785; SSE-NEXT: movdqa %xmm12, %xmm135786; SSE-NEXT: pandn %xmm2, %xmm135787; SSE-NEXT: andps %xmm12, %xmm05788; SSE-NEXT: por %xmm0, %xmm135789; SSE-NEXT: unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm4[1]5790; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm11[1,1,1,1,4,5,6,7]5791; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm4[1,1]5792; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm3[0,2]5793; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,0,1,1]5794; SSE-NEXT: movdqa %xmm10, %xmm95795; SSE-NEXT: pandn %xmm2, %xmm95796; SSE-NEXT: andps %xmm10, %xmm05797; SSE-NEXT: por %xmm0, %xmm95798; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload5799; SSE-NEXT: movaps %xmm7, %xmm05800; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload5801; SSE-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]5802; SSE-NEXT: movdqa %xmm11, %xmm25803; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,1],xmm3[1,3]5804; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[0,2]5805; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[2,2,3,3]5806; SSE-NEXT: pslld $16, %xmm15807; SSE-NEXT: movdqa %xmm12, %xmm65808; SSE-NEXT: pandn %xmm1, %xmm65809; SSE-NEXT: andps %xmm12, %xmm05810; SSE-NEXT: por %xmm0, %xmm65811; SSE-NEXT: unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm7[1]5812; SSE-NEXT: psrldq {{.*#+}} xmm11 = xmm11[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero5813; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[1,1],xmm7[1,1]5814; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[2,0],xmm3[0,2]5815; SSE-NEXT: movdqa %xmm10, %xmm45816; SSE-NEXT: pandn %xmm5, %xmm45817; SSE-NEXT: andps %xmm10, %xmm115818; SSE-NEXT: por %xmm11, %xmm45819; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload5820; SSE-NEXT: movaps %xmm7, %xmm05821; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload5822; SSE-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]5823; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload5824; SSE-NEXT: movaps %xmm1, %xmm55825; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,1],xmm3[1,3]5826; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm5[0,2]5827; SSE-NEXT: movdqa %xmm15, %xmm55828; SSE-NEXT: pslldq {{.*#+}} xmm5 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm5[0,1,2,3,4,5]5829; SSE-NEXT: movdqa %xmm12, %xmm115830; SSE-NEXT: pandn %xmm5, %xmm115831; SSE-NEXT: andps %xmm12, %xmm05832; SSE-NEXT: por %xmm0, %xmm115833; SSE-NEXT: movaps %xmm7, %xmm05834; SSE-NEXT: unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm7[1]5835; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm1[1,1,1,1,4,5,6,7]5836; SSE-NEXT: movaps %xmm1, %xmm75837; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[1,1],xmm0[1,1]5838; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[2,0],xmm3[0,2]5839; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm15[0,0,1,1]5840; SSE-NEXT: movdqa %xmm15, %xmm85841; SSE-NEXT: movdqa %xmm10, %xmm155842; SSE-NEXT: pandn %xmm1, %xmm155843; SSE-NEXT: andps %xmm10, %xmm55844; SSE-NEXT: por %xmm5, %xmm155845; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload5846; SSE-NEXT: movaps %xmm3, %xmm15847; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload5848; SSE-NEXT: movlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]5849; SSE-NEXT: movaps %xmm7, %xmm55850; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[2,1],xmm2[1,3]5851; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm5[0,2]5852; SSE-NEXT: andps %xmm12, %xmm15853; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm8[2,2,3,3]5854; SSE-NEXT: pslld $16, %xmm85855; SSE-NEXT: pandn %xmm8, %xmm125856; SSE-NEXT: por %xmm1, %xmm125857; SSE-NEXT: unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm3[1]5858; SSE-NEXT: psrldq {{.*#+}} xmm7 = xmm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero5859; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[1,1],xmm3[1,1]5860; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[2,0],xmm2[0,2]5861; SSE-NEXT: andps %xmm10, %xmm75862; SSE-NEXT: pandn %xmm5, %xmm105863; SSE-NEXT: por %xmm7, %xmm105864; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax5865; SSE-NEXT: movdqa %xmm10, 736(%rax)5866; SSE-NEXT: movdqa %xmm12, 720(%rax)5867; SSE-NEXT: movdqa %xmm15, 688(%rax)5868; SSE-NEXT: movdqa %xmm11, 672(%rax)5869; SSE-NEXT: movdqa %xmm4, 640(%rax)5870; SSE-NEXT: movdqa %xmm6, 624(%rax)5871; SSE-NEXT: movdqa %xmm9, 592(%rax)5872; SSE-NEXT: movdqa %xmm13, 576(%rax)5873; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5874; SSE-NEXT: movaps %xmm0, 544(%rax)5875; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5876; SSE-NEXT: movaps %xmm0, 528(%rax)5877; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5878; SSE-NEXT: movaps %xmm0, 496(%rax)5879; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5880; SSE-NEXT: movaps %xmm0, 480(%rax)5881; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5882; SSE-NEXT: movaps %xmm0, 448(%rax)5883; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5884; SSE-NEXT: movaps %xmm0, 432(%rax)5885; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5886; SSE-NEXT: movaps %xmm0, 400(%rax)5887; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5888; SSE-NEXT: movaps %xmm0, 384(%rax)5889; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5890; SSE-NEXT: movaps %xmm0, 352(%rax)5891; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5892; SSE-NEXT: movaps %xmm0, 336(%rax)5893; SSE-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload5894; SSE-NEXT: movaps %xmm0, 304(%rax)5895; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5896; SSE-NEXT: movaps %xmm0, 288(%rax)5897; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5898; SSE-NEXT: movaps %xmm0, 256(%rax)5899; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5900; SSE-NEXT: movaps %xmm0, 240(%rax)5901; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5902; SSE-NEXT: movaps %xmm0, 208(%rax)5903; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5904; SSE-NEXT: movaps %xmm0, 192(%rax)5905; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5906; SSE-NEXT: movaps %xmm0, 160(%rax)5907; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5908; SSE-NEXT: movaps %xmm0, 144(%rax)5909; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5910; SSE-NEXT: movaps %xmm0, 112(%rax)5911; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5912; SSE-NEXT: movaps %xmm0, 96(%rax)5913; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5914; SSE-NEXT: movaps %xmm0, 64(%rax)5915; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5916; SSE-NEXT: movaps %xmm0, 48(%rax)5917; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5918; SSE-NEXT: movaps %xmm0, 16(%rax)5919; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5920; SSE-NEXT: movaps %xmm0, (%rax)5921; SSE-NEXT: movaps %xmm14, 752(%rax)5922; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5923; SSE-NEXT: movaps %xmm0, 704(%rax)5924; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5925; SSE-NEXT: movaps %xmm0, 656(%rax)5926; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5927; SSE-NEXT: movaps %xmm0, 608(%rax)5928; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5929; SSE-NEXT: movaps %xmm0, 560(%rax)5930; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5931; SSE-NEXT: movaps %xmm0, 512(%rax)5932; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5933; SSE-NEXT: movaps %xmm0, 464(%rax)5934; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5935; SSE-NEXT: movaps %xmm0, 416(%rax)5936; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5937; SSE-NEXT: movaps %xmm0, 368(%rax)5938; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5939; SSE-NEXT: movaps %xmm0, 320(%rax)5940; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5941; SSE-NEXT: movaps %xmm0, 272(%rax)5942; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5943; SSE-NEXT: movaps %xmm0, 224(%rax)5944; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5945; SSE-NEXT: movaps %xmm0, 176(%rax)5946; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5947; SSE-NEXT: movaps %xmm0, 128(%rax)5948; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5949; SSE-NEXT: movaps %xmm0, 80(%rax)5950; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5951; SSE-NEXT: movaps %xmm0, 32(%rax)5952; SSE-NEXT: addq $808, %rsp # imm = 0x3285953; SSE-NEXT: retq5954;5955; AVX-LABEL: store_i16_stride6_vf64:5956; AVX: # %bb.0:5957; AVX-NEXT: subq $504, %rsp # imm = 0x1F85958; AVX-NEXT: vmovdqa 80(%rcx), %xmm15959; AVX-NEXT: vmovdqa 80(%rdx), %xmm25960; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]5961; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,2,3,3]5962; AVX-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]5963; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm4[0,0,1,1]5964; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm25965; AVX-NEXT: vmovdqa 80(%rsi), %xmm35966; AVX-NEXT: vmovdqa 80(%rdi), %xmm55967; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]5968; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[2,3,2,3]5969; AVX-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]5970; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm5[0,1,0,1]5971; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm6, %ymm35972; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm3[0,1],ymm2[2],ymm3[3,4],ymm2[5],ymm3[6,7]5973; AVX-NEXT: vmovdqa 80(%r8), %xmm25974; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[2,1,3,3,4,5,6,7]5975; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,1,2,1]5976; AVX-NEXT: vblendps {{.*#+}} xmm7 = xmm3[0],xmm6[1,2],xmm3[3]5977; AVX-NEXT: vmovdqa 80(%r9), %xmm35978; AVX-NEXT: vpshuflw {{.*#+}} xmm8 = xmm3[0,2,2,3,4,5,6,7]5979; AVX-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[0,1,2,1]5980; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0],xmm8[1],xmm7[2,3,4,5,6],xmm8[7]5981; AVX-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5982; AVX-NEXT: vextractf128 $1, %ymm6, %xmm65983; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3],xmm2[4,5],xmm6[6,7]5984; AVX-NEXT: vpslld $16, %xmm3, %xmm75985; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm7[5],xmm6[6,7]5986; AVX-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5987; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]5988; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm55989; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[1,1,2,2]5990; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]5991; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm45992; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]5993; AVX-NEXT: vpsrldq {{.*#+}} xmm5 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero5994; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm4[0],xmm5[1],xmm4[2,3]5995; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[2,2,3,3]5996; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2],xmm6[3],xmm5[4,5,6,7]5997; AVX-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5998; AVX-NEXT: vextractf128 $1, %ymm4, %xmm45999; AVX-NEXT: vpshufhw {{.*#+}} xmm5 = xmm2[0,1,2,3,6,5,7,7]6000; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[2,1,2,3]6001; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3,4,5],xmm5[6,7]6002; AVX-NEXT: vpshufhw {{.*#+}} xmm5 = xmm3[0,1,2,3,4,6,6,7]6003; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[2,1,2,3]6004; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm5[1],xmm4[2,3,4,5,6],xmm5[7]6005; AVX-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6006; AVX-NEXT: vmovdqa 64(%rsi), %xmm86007; AVX-NEXT: vmovdqa 64(%rdi), %xmm96008; AVX-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]6009; AVX-NEXT: vmovdqa 64(%rcx), %xmm106010; AVX-NEXT: vmovdqa 64(%rdx), %xmm116011; AVX-NEXT: vpunpckhwd {{.*#+}} xmm7 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]6012; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm7[1,1,2,2]6013; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm7[2,2,3,3]6014; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm56015; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[2,3,2,3]6016; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm4, %ymm66017; AVX-NEXT: vblendps {{.*#+}} ymm12 = ymm5[0,1],ymm6[2],ymm5[3,4],ymm6[5],ymm5[6,7]6018; AVX-NEXT: vmovdqa 64(%r8), %xmm56019; AVX-NEXT: vpsrldq {{.*#+}} xmm6 = xmm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero6020; AVX-NEXT: vblendps {{.*#+}} xmm13 = xmm12[0],xmm6[1],xmm12[2,3]6021; AVX-NEXT: vmovdqa 64(%r9), %xmm66022; AVX-NEXT: vpshufd {{.*#+}} xmm14 = xmm6[2,2,3,3]6023; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0,1,2],xmm14[3],xmm13[4,5,6,7]6024; AVX-NEXT: vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6025; AVX-NEXT: vextractf128 $1, %ymm12, %xmm126026; AVX-NEXT: vpshufhw {{.*#+}} xmm13 = xmm5[0,1,2,3,6,5,7,7]6027; AVX-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[2,1,2,3]6028; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm13[0,1],xmm12[2,3,4,5],xmm13[6,7]6029; AVX-NEXT: vpshufhw {{.*#+}} xmm13 = xmm6[0,1,2,3,4,6,6,7]6030; AVX-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[2,1,2,3]6031; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0],xmm13[1],xmm12[2,3,4,5,6],xmm13[7]6032; AVX-NEXT: vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6033; AVX-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[0,0,1,1]6034; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,2]6035; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm12, %ymm06036; AVX-NEXT: vpshufd {{.*#+}} xmm12 = xmm1[0,1,0,1]6037; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm12, %ymm16038; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]6039; AVX-NEXT: vinsertps {{.*#+}} xmm1 = xmm0[0,1],xmm2[0],xmm0[3]6040; AVX-NEXT: vpslldq {{.*#+}} xmm12 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1,2,3,4,5]6041; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm12[5],xmm1[6,7]6042; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6043; AVX-NEXT: vextractf128 $1, %ymm0, %xmm06044; AVX-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero6045; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]6046; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[0,0,1,1]6047; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3],xmm0[4,5,6,7]6048; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6049; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]6050; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,0,1,1]6051; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,2]6052; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm16053; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]6054; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,1,0,1]6055; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm36056; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm3[0],ymm1[1],ymm3[2,3],ymm1[4],ymm3[5,6],ymm1[7]6057; AVX-NEXT: vinsertps {{.*#+}} xmm3 = xmm1[0,1],xmm5[0],xmm1[3]6058; AVX-NEXT: vpslldq {{.*#+}} xmm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm6[0,1,2,3,4,5]6059; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm8[5],xmm3[6,7]6060; AVX-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6061; AVX-NEXT: vextractf128 $1, %ymm1, %xmm16062; AVX-NEXT: vpmovzxwd {{.*#+}} xmm3 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero6063; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5,6,7]6064; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm6[0,0,1,1]6065; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm3[3],xmm1[4,5,6,7]6066; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6067; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]6068; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm7[0,0,1,1]6069; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm06070; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]6071; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm4[0,1,0,1]6072; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm16073; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]6074; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm5[2,1,3,3,4,5,6,7]6075; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]6076; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0],xmm0[1,2],xmm1[3]6077; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm6[0,2,2,3,4,5,6,7]6078; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,2,1]6079; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2,3,4,5,6],xmm2[7]6080; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6081; AVX-NEXT: vextractf128 $1, %ymm0, %xmm06082; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm5[4,5],xmm0[6,7]6083; AVX-NEXT: vpslld $16, %xmm6, %xmm16084; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm1[5],xmm0[6,7]6085; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6086; AVX-NEXT: vmovdqa 48(%rcx), %xmm16087; AVX-NEXT: vmovdqa 48(%rdx), %xmm26088; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]6089; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,2,3,3]6090; AVX-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]6091; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm4[0,0,1,1]6092; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm26093; AVX-NEXT: vmovdqa 48(%rsi), %xmm36094; AVX-NEXT: vmovdqa 48(%rdi), %xmm56095; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]6096; AVX-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]6097; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]6098; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[0,1,0,1]6099; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm3, %ymm36100; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm3[0,1],ymm2[2],ymm3[3,4],ymm2[5],ymm3[6,7]6101; AVX-NEXT: vmovdqa 48(%r8), %xmm26102; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[2,1,3,3,4,5,6,7]6103; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,1,2,1]6104; AVX-NEXT: vblendps {{.*#+}} xmm7 = xmm3[0],xmm6[1,2],xmm3[3]6105; AVX-NEXT: vmovdqa 48(%r9), %xmm36106; AVX-NEXT: vpshuflw {{.*#+}} xmm8 = xmm3[0,2,2,3,4,5,6,7]6107; AVX-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[0,1,2,1]6108; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0],xmm8[1],xmm7[2,3,4,5,6],xmm8[7]6109; AVX-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6110; AVX-NEXT: vextractf128 $1, %ymm6, %xmm66111; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3],xmm2[4,5],xmm6[6,7]6112; AVX-NEXT: vpslld $16, %xmm3, %xmm76113; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm7[5],xmm6[6,7]6114; AVX-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6115; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]6116; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm56117; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[1,1,2,2]6118; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]6119; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm46120; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]6121; AVX-NEXT: vpsrldq {{.*#+}} xmm5 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero6122; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm4[0],xmm5[1],xmm4[2,3]6123; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[2,2,3,3]6124; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2],xmm6[3],xmm5[4,5,6,7]6125; AVX-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6126; AVX-NEXT: vextractf128 $1, %ymm4, %xmm46127; AVX-NEXT: vpshufhw {{.*#+}} xmm5 = xmm2[0,1,2,3,6,5,7,7]6128; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[2,1,2,3]6129; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3,4,5],xmm5[6,7]6130; AVX-NEXT: vpshufhw {{.*#+}} xmm5 = xmm3[0,1,2,3,4,6,6,7]6131; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[2,1,2,3]6132; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm5[1],xmm4[2,3,4,5,6],xmm5[7]6133; AVX-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6134; AVX-NEXT: vmovdqa 32(%rcx), %xmm86135; AVX-NEXT: vmovdqa 32(%rdx), %xmm96136; AVX-NEXT: vpunpckhwd {{.*#+}} xmm6 = xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]6137; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm6[1,1,2,2]6138; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm6[2,2,3,3]6139; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm4, %ymm46140; AVX-NEXT: vmovdqa 32(%rsi), %xmm106141; AVX-NEXT: vmovdqa 32(%rdi), %xmm116142; AVX-NEXT: vpunpckhwd {{.*#+}} xmm7 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]6143; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm7[2,3,2,3]6144; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm7, %ymm56145; AVX-NEXT: vblendps {{.*#+}} ymm12 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]6146; AVX-NEXT: vmovdqa 32(%r8), %xmm46147; AVX-NEXT: vpsrldq {{.*#+}} xmm5 = xmm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero6148; AVX-NEXT: vblendps {{.*#+}} xmm13 = xmm12[0],xmm5[1],xmm12[2,3]6149; AVX-NEXT: vmovdqa 32(%r9), %xmm56150; AVX-NEXT: vpshufd {{.*#+}} xmm14 = xmm5[2,2,3,3]6151; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0,1,2],xmm14[3],xmm13[4,5,6,7]6152; AVX-NEXT: vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6153; AVX-NEXT: vextractf128 $1, %ymm12, %xmm126154; AVX-NEXT: vpshufhw {{.*#+}} xmm13 = xmm4[0,1,2,3,6,5,7,7]6155; AVX-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[2,1,2,3]6156; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm13[0,1],xmm12[2,3,4,5],xmm13[6,7]6157; AVX-NEXT: vpshufhw {{.*#+}} xmm13 = xmm5[0,1,2,3,4,6,6,7]6158; AVX-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[2,1,2,3]6159; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0],xmm13[1],xmm12[2,3,4,5,6],xmm13[7]6160; AVX-NEXT: vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6161; AVX-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[0,0,1,1]6162; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,2]6163; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm12, %ymm06164; AVX-NEXT: vpshufd {{.*#+}} xmm12 = xmm1[0,1,0,1]6165; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm12, %ymm16166; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]6167; AVX-NEXT: vinsertps {{.*#+}} xmm1 = xmm0[0,1],xmm2[0],xmm0[3]6168; AVX-NEXT: vpslldq {{.*#+}} xmm12 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1,2,3,4,5]6169; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm12[5],xmm1[6,7]6170; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6171; AVX-NEXT: vextractf128 $1, %ymm0, %xmm06172; AVX-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero6173; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]6174; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[0,0,1,1]6175; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3],xmm0[4,5,6,7]6176; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6177; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]6178; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,0,1,1]6179; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,2]6180; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm16181; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]6182; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,1,0,1]6183; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm36184; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm3[0],ymm1[1],ymm3[2,3],ymm1[4],ymm3[5,6],ymm1[7]6185; AVX-NEXT: vinsertps {{.*#+}} xmm3 = xmm1[0,1],xmm4[0],xmm1[3]6186; AVX-NEXT: vpslldq {{.*#+}} xmm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm5[0,1,2,3,4,5]6187; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm8[5],xmm3[6,7]6188; AVX-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6189; AVX-NEXT: vextractf128 $1, %ymm1, %xmm16190; AVX-NEXT: vpmovzxwd {{.*#+}} xmm3 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero6191; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5,6,7]6192; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm5[0,0,1,1]6193; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm3[3],xmm1[4,5,6,7]6194; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6195; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]6196; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm6[0,0,1,1]6197; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm06198; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]6199; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm7[0,1,0,1]6200; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm16201; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]6202; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm4[2,1,3,3,4,5,6,7]6203; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]6204; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0],xmm0[1,2],xmm1[3]6205; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm5[0,2,2,3,4,5,6,7]6206; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,2,1]6207; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2,3,4,5,6],xmm2[7]6208; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6209; AVX-NEXT: vextractf128 $1, %ymm0, %xmm06210; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm4[4,5],xmm0[6,7]6211; AVX-NEXT: vpslld $16, %xmm5, %xmm16212; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm1[5],xmm0[6,7]6213; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6214; AVX-NEXT: vmovdqa 112(%rcx), %xmm16215; AVX-NEXT: vmovdqa 112(%rdx), %xmm26216; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]6217; AVX-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]6218; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]6219; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm4[0,0,1,1]6220; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm26221; AVX-NEXT: vmovdqa 112(%rsi), %xmm36222; AVX-NEXT: vmovdqa 112(%rdi), %xmm56223; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]6224; AVX-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]6225; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]6226; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[0,1,0,1]6227; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm3, %ymm36228; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm3[0,1],ymm2[2],ymm3[3,4],ymm2[5],ymm3[6,7]6229; AVX-NEXT: vmovdqa 112(%r8), %xmm26230; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[2,1,3,3,4,5,6,7]6231; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,1,2,1]6232; AVX-NEXT: vblendps {{.*#+}} xmm7 = xmm3[0],xmm6[1,2],xmm3[3]6233; AVX-NEXT: vmovdqa 112(%r9), %xmm36234; AVX-NEXT: vpshuflw {{.*#+}} xmm8 = xmm3[0,2,2,3,4,5,6,7]6235; AVX-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[0,1,2,1]6236; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0],xmm8[1],xmm7[2,3,4,5,6],xmm8[7]6237; AVX-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6238; AVX-NEXT: vextractf128 $1, %ymm6, %xmm66239; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3],xmm2[4,5],xmm6[6,7]6240; AVX-NEXT: vpslld $16, %xmm3, %xmm76241; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm7[5],xmm6[6,7]6242; AVX-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6243; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]6244; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm56245; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[1,1,2,2]6246; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]6247; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm46248; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]6249; AVX-NEXT: vpsrldq {{.*#+}} xmm5 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero6250; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm4[0],xmm5[1],xmm4[2,3]6251; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[2,2,3,3]6252; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1,2],xmm6[3],xmm5[4,5,6,7]6253; AVX-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6254; AVX-NEXT: vextractf128 $1, %ymm4, %xmm46255; AVX-NEXT: vpshufhw {{.*#+}} xmm5 = xmm2[0,1,2,3,6,5,7,7]6256; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[2,1,2,3]6257; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3,4,5],xmm5[6,7]6258; AVX-NEXT: vpshufhw {{.*#+}} xmm5 = xmm3[0,1,2,3,4,6,6,7]6259; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[2,1,2,3]6260; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm5[1],xmm4[2,3,4,5,6],xmm5[7]6261; AVX-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6262; AVX-NEXT: vmovdqa 96(%rcx), %xmm86263; AVX-NEXT: vmovdqa 96(%rdx), %xmm96264; AVX-NEXT: vpunpckhwd {{.*#+}} xmm6 = xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]6265; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm6[1,1,2,2]6266; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm6[2,2,3,3]6267; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm4, %ymm46268; AVX-NEXT: vmovdqa 96(%rsi), %xmm106269; AVX-NEXT: vmovdqa 96(%rdi), %xmm116270; AVX-NEXT: vpunpckhwd {{.*#+}} xmm7 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]6271; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm7[2,3,2,3]6272; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm7, %ymm56273; AVX-NEXT: vblendps {{.*#+}} ymm12 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]6274; AVX-NEXT: vmovdqa 96(%r8), %xmm46275; AVX-NEXT: vpsrldq {{.*#+}} xmm5 = xmm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero6276; AVX-NEXT: vblendps {{.*#+}} xmm13 = xmm12[0],xmm5[1],xmm12[2,3]6277; AVX-NEXT: vmovdqa 96(%r9), %xmm56278; AVX-NEXT: vpshufd {{.*#+}} xmm14 = xmm5[2,2,3,3]6279; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0,1,2],xmm14[3],xmm13[4,5,6,7]6280; AVX-NEXT: vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6281; AVX-NEXT: vextractf128 $1, %ymm12, %xmm126282; AVX-NEXT: vpshufhw {{.*#+}} xmm13 = xmm4[0,1,2,3,6,5,7,7]6283; AVX-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[2,1,2,3]6284; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm13[0,1],xmm12[2,3,4,5],xmm13[6,7]6285; AVX-NEXT: vpshufhw {{.*#+}} xmm13 = xmm5[0,1,2,3,4,6,6,7]6286; AVX-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[2,1,2,3]6287; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0],xmm13[1],xmm12[2,3,4,5,6],xmm13[7]6288; AVX-NEXT: vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6289; AVX-NEXT: vpshufd {{.*#+}} xmm12 = xmm0[0,0,1,1]6290; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,2]6291; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm12, %ymm06292; AVX-NEXT: vpshufd {{.*#+}} xmm12 = xmm1[0,1,0,1]6293; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm12, %ymm16294; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7]6295; AVX-NEXT: vinsertps {{.*#+}} xmm1 = xmm0[0,1],xmm2[0],xmm0[3]6296; AVX-NEXT: vpslldq {{.*#+}} xmm12 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1,2,3,4,5]6297; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm12[5],xmm1[6,7]6298; AVX-NEXT: vmovdqa %xmm1, (%rsp) # 16-byte Spill6299; AVX-NEXT: vextractf128 $1, %ymm0, %xmm06300; AVX-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero6301; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]6302; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[0,0,1,1]6303; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3],xmm0[4,5,6,7]6304; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6305; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]6306; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,0,1,1]6307; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,2]6308; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm16309; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]6310; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,1,0,1]6311; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm36312; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm3[0],ymm1[1],ymm3[2,3],ymm1[4],ymm3[5,6],ymm1[7]6313; AVX-NEXT: vinsertps {{.*#+}} xmm3 = xmm1[0,1],xmm4[0],xmm1[3]6314; AVX-NEXT: vpslldq {{.*#+}} xmm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm5[0,1,2,3,4,5]6315; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm8[5],xmm3[6,7]6316; AVX-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6317; AVX-NEXT: vextractf128 $1, %ymm1, %xmm16318; AVX-NEXT: vpmovzxwd {{.*#+}} xmm3 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero6319; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5,6,7]6320; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm5[0,0,1,1]6321; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm3[3],xmm1[4,5,6,7]6322; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6323; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]6324; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm6[0,0,1,1]6325; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm06326; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]6327; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm7[0,1,0,1]6328; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm16329; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]6330; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm4[2,1,3,3,4,5,6,7]6331; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]6332; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0],xmm0[1,2],xmm1[3]6333; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm5[0,2,2,3,4,5,6,7]6334; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,2,1]6335; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2,3,4,5,6],xmm2[7]6336; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6337; AVX-NEXT: vextractf128 $1, %ymm0, %xmm06338; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm4[4,5],xmm0[6,7]6339; AVX-NEXT: vpslld $16, %xmm5, %xmm16340; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm1[5],xmm0[6,7]6341; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6342; AVX-NEXT: vmovdqa 16(%rcx), %xmm06343; AVX-NEXT: vmovdqa 16(%rdx), %xmm16344; AVX-NEXT: vpunpcklwd {{.*#+}} xmm11 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]6345; AVX-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]6346; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm11[2,2,3,3]6347; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,0,1,1]6348; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm16349; AVX-NEXT: vmovdqa 16(%rsi), %xmm26350; AVX-NEXT: vmovdqa 16(%rdi), %xmm36351; AVX-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]6352; AVX-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]6353; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm10[2,3,2,3]6354; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[0,1,0,1]6355; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm26356; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]6357; AVX-NEXT: vmovdqa 16(%r8), %xmm36358; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm3[2,1,3,3,4,5,6,7]6359; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,2,1]6360; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm2[0],xmm1[1,2],xmm2[3]6361; AVX-NEXT: vmovdqa 16(%r9), %xmm26362; AVX-NEXT: vpshuflw {{.*#+}} xmm6 = xmm2[0,2,2,3,4,5,6,7]6363; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[0,1,2,1]6364; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0],xmm6[1],xmm5[2,3,4,5,6],xmm6[7]6365; AVX-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6366; AVX-NEXT: vextractf128 $1, %ymm1, %xmm16367; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5],xmm1[6,7]6368; AVX-NEXT: vpslld $16, %xmm2, %xmm56369; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm5[5],xmm1[6,7]6370; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6371; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm4[2,3,2,3]6372; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm4, %ymm16373; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,2,2]6374; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]6375; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm4, %ymm06376; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]6377; AVX-NEXT: vpsrldq {{.*#+}} xmm1 = xmm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero6378; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2,3]6379; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,2,3,3]6380; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3],xmm1[4,5,6,7]6381; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6382; AVX-NEXT: vextractf128 $1, %ymm0, %xmm06383; AVX-NEXT: vpshufhw {{.*#+}} xmm1 = xmm3[0,1,2,3,6,5,7,7]6384; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]6385; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3,4,5],xmm1[6,7]6386; AVX-NEXT: vpshufhw {{.*#+}} xmm1 = xmm2[0,1,2,3,4,6,6,7]6387; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]6388; AVX-NEXT: vpblendw {{.*#+}} xmm14 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6],xmm1[7]6389; AVX-NEXT: vmovdqa (%rcx), %xmm96390; AVX-NEXT: vmovdqa (%rdx), %xmm86391; AVX-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm8[4],xmm9[4],xmm8[5],xmm9[5],xmm8[6],xmm9[6],xmm8[7],xmm9[7]6392; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm5[1,1,2,2]6393; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm5[2,2,3,3]6394; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm06395; AVX-NEXT: vmovdqa (%rsi), %xmm76396; AVX-NEXT: vmovdqa (%rdi), %xmm66397; AVX-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7]6398; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm4[2,3,2,3]6399; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm4, %ymm16400; AVX-NEXT: vblendps {{.*#+}} ymm12 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]6401; AVX-NEXT: vmovdqa (%r8), %xmm16402; AVX-NEXT: vpsrldq {{.*#+}} xmm0 = xmm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero6403; AVX-NEXT: vblendps {{.*#+}} xmm13 = xmm12[0],xmm0[1],xmm12[2,3]6404; AVX-NEXT: vmovdqa (%r9), %xmm06405; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm0[2,2,3,3]6406; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0,1,2],xmm15[3],xmm13[4,5,6,7]6407; AVX-NEXT: vextractf128 $1, %ymm12, %xmm126408; AVX-NEXT: vpshufhw {{.*#+}} xmm15 = xmm1[0,1,2,3,6,5,7,7]6409; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm15[2,1,2,3]6410; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm15[0,1],xmm12[2,3,4,5],xmm15[6,7]6411; AVX-NEXT: vpshufhw {{.*#+}} xmm15 = xmm0[0,1,2,3,4,6,6,7]6412; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm15[2,1,2,3]6413; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0],xmm15[1],xmm12[2,3,4,5,6],xmm15[7]6414; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm11[0,0,1,1]6415; AVX-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[1,1,2,2]6416; AVX-NEXT: vinsertf128 $1, %xmm11, %ymm15, %ymm116417; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm10[0,1,0,1]6418; AVX-NEXT: vinsertf128 $1, %xmm10, %ymm15, %ymm106419; AVX-NEXT: vblendps {{.*#+}} ymm11 = ymm10[0],ymm11[1],ymm10[2,3],ymm11[4],ymm10[5,6],ymm11[7]6420; AVX-NEXT: vinsertps {{.*#+}} xmm10 = xmm11[0,1],xmm3[0],xmm11[3]6421; AVX-NEXT: vpslldq {{.*#+}} xmm15 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2,3,4,5]6422; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1,2,3,4],xmm15[5],xmm10[6,7]6423; AVX-NEXT: vextractf128 $1, %ymm11, %xmm116424; AVX-NEXT: vpmovzxwd {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero6425; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm11[0,1],xmm3[2,3],xmm11[4,5,6,7]6426; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]6427; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3],xmm3[4,5,6,7]6428; AVX-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]6429; AVX-NEXT: vpshufd {{.*#+}} xmm8 = xmm3[0,0,1,1]6430; AVX-NEXT: vpshufd {{.*#+}} xmm9 = xmm3[1,1,2,2]6431; AVX-NEXT: vinsertf128 $1, %xmm9, %ymm8, %ymm86432; AVX-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]6433; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[0,1,0,1]6434; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm7, %ymm76435; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm7[0],ymm8[1],ymm7[2,3],ymm8[4],ymm7[5,6],ymm8[7]6436; AVX-NEXT: vinsertps {{.*#+}} xmm8 = xmm7[0,1],xmm1[0],xmm7[3]6437; AVX-NEXT: vpslldq {{.*#+}} xmm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]6438; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1,2,3,4],xmm9[5],xmm8[6,7]6439; AVX-NEXT: vextractf128 $1, %ymm7, %xmm76440; AVX-NEXT: vpmovzxwd {{.*#+}} xmm9 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero6441; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1],xmm9[2,3],xmm7[4,5,6,7]6442; AVX-NEXT: vpshufd {{.*#+}} xmm9 = xmm0[0,0,1,1]6443; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2],xmm9[3],xmm7[4,5,6,7]6444; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,2,3,3]6445; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[0,0,1,1]6446; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm3, %ymm36447; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm6[2,3,2,3]6448; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]6449; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm5, %ymm46450; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7]6451; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm1[2,1,3,3,4,5,6,7]6452; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,1,2,1]6453; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm4[0],xmm3[1,2],xmm4[3]6454; AVX-NEXT: vpshuflw {{.*#+}} xmm5 = xmm0[0,2,2,3,4,5,6,7]6455; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[0,1,2,1]6456; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm5[1],xmm4[2,3,4,5,6],xmm5[7]6457; AVX-NEXT: vextractf128 $1, %ymm3, %xmm36458; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5],xmm3[6,7]6459; AVX-NEXT: vpslld $16, %xmm0, %xmm06460; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4],xmm0[5],xmm1[6,7]6461; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax6462; AVX-NEXT: vmovdqa %xmm0, 48(%rax)6463; AVX-NEXT: vmovdqa %xmm4, 32(%rax)6464; AVX-NEXT: vmovdqa %xmm7, 16(%rax)6465; AVX-NEXT: vmovdqa %xmm8, (%rax)6466; AVX-NEXT: vmovdqa %xmm2, 112(%rax)6467; AVX-NEXT: vmovdqa %xmm10, 96(%rax)6468; AVX-NEXT: vmovdqa %xmm12, 80(%rax)6469; AVX-NEXT: vmovdqa %xmm13, 64(%rax)6470; AVX-NEXT: vmovdqa %xmm14, 176(%rax)6471; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6472; AVX-NEXT: vmovaps %xmm0, 160(%rax)6473; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6474; AVX-NEXT: vmovaps %xmm0, 144(%rax)6475; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6476; AVX-NEXT: vmovaps %xmm0, 128(%rax)6477; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6478; AVX-NEXT: vmovaps %xmm0, 624(%rax)6479; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6480; AVX-NEXT: vmovaps %xmm0, 608(%rax)6481; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6482; AVX-NEXT: vmovaps %xmm0, 592(%rax)6483; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6484; AVX-NEXT: vmovaps %xmm0, 576(%rax)6485; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6486; AVX-NEXT: vmovaps %xmm0, 688(%rax)6487; AVX-NEXT: vmovaps (%rsp), %xmm0 # 16-byte Reload6488; AVX-NEXT: vmovaps %xmm0, 672(%rax)6489; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6490; AVX-NEXT: vmovaps %xmm0, 656(%rax)6491; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6492; AVX-NEXT: vmovaps %xmm0, 640(%rax)6493; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6494; AVX-NEXT: vmovaps %xmm0, 752(%rax)6495; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6496; AVX-NEXT: vmovaps %xmm0, 736(%rax)6497; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6498; AVX-NEXT: vmovaps %xmm0, 720(%rax)6499; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6500; AVX-NEXT: vmovaps %xmm0, 704(%rax)6501; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6502; AVX-NEXT: vmovaps %xmm0, 240(%rax)6503; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6504; AVX-NEXT: vmovaps %xmm0, 224(%rax)6505; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6506; AVX-NEXT: vmovaps %xmm0, 208(%rax)6507; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6508; AVX-NEXT: vmovaps %xmm0, 192(%rax)6509; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6510; AVX-NEXT: vmovaps %xmm0, 304(%rax)6511; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6512; AVX-NEXT: vmovaps %xmm0, 288(%rax)6513; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6514; AVX-NEXT: vmovaps %xmm0, 272(%rax)6515; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6516; AVX-NEXT: vmovaps %xmm0, 256(%rax)6517; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6518; AVX-NEXT: vmovaps %xmm0, 368(%rax)6519; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6520; AVX-NEXT: vmovaps %xmm0, 352(%rax)6521; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6522; AVX-NEXT: vmovaps %xmm0, 336(%rax)6523; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6524; AVX-NEXT: vmovaps %xmm0, 320(%rax)6525; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6526; AVX-NEXT: vmovaps %xmm0, 432(%rax)6527; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6528; AVX-NEXT: vmovaps %xmm0, 416(%rax)6529; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6530; AVX-NEXT: vmovaps %xmm0, 400(%rax)6531; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6532; AVX-NEXT: vmovaps %xmm0, 384(%rax)6533; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6534; AVX-NEXT: vmovaps %xmm0, 496(%rax)6535; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6536; AVX-NEXT: vmovaps %xmm0, 480(%rax)6537; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6538; AVX-NEXT: vmovaps %xmm0, 464(%rax)6539; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6540; AVX-NEXT: vmovaps %xmm0, 448(%rax)6541; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6542; AVX-NEXT: vmovaps %xmm0, 560(%rax)6543; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6544; AVX-NEXT: vmovaps %xmm0, 544(%rax)6545; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6546; AVX-NEXT: vmovaps %xmm0, 528(%rax)6547; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6548; AVX-NEXT: vmovaps %xmm0, 512(%rax)6549; AVX-NEXT: addq $504, %rsp # imm = 0x1F86550; AVX-NEXT: vzeroupper6551; AVX-NEXT: retq6552;6553; AVX2-LABEL: store_i16_stride6_vf64:6554; AVX2: # %bb.0:6555; AVX2-NEXT: subq $1544, %rsp # imm = 0x6086556; AVX2-NEXT: vmovdqa (%rcx), %xmm126557; AVX2-NEXT: vmovdqa 32(%rcx), %xmm56558; AVX2-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6559; AVX2-NEXT: vpsrldq {{.*#+}} xmm0 = xmm12[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero6560; AVX2-NEXT: vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6561; AVX2-NEXT: vmovdqa (%rdx), %xmm116562; AVX2-NEXT: vmovdqa 32(%rdx), %xmm66563; AVX2-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6564; AVX2-NEXT: vpsrldq {{.*#+}} xmm1 = xmm11[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero6565; AVX2-NEXT: vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6566; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]6567; AVX2-NEXT: vpbroadcastq %xmm0, %ymm06568; AVX2-NEXT: vmovdqa (%rsi), %xmm146569; AVX2-NEXT: vmovdqa 32(%rsi), %xmm136570; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm14[0,1,2,1]6571; AVX2-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,5]6572; AVX2-NEXT: vmovdqa (%rdi), %xmm106573; AVX2-NEXT: vmovdqa 32(%rdi), %xmm86574; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm10[0,1,2,1]6575; AVX2-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,5]6576; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]6577; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]6578; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]6579; AVX2-NEXT: vmovdqa (%r8), %xmm16580; AVX2-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6581; AVX2-NEXT: vmovdqa 32(%r8), %xmm46582; AVX2-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6583; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,1,3,3,4,5,6,7]6584; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]6585; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7]6586; AVX2-NEXT: vmovdqa (%r9), %xmm06587; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6588; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]6589; AVX2-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]6590; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm0[0,0,2,1]6591; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm0 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]6592; AVX2-NEXT: vpblendvb %ymm0, %ymm1, %ymm2, %ymm16593; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6594; AVX2-NEXT: vpsrldq {{.*#+}} xmm1 = xmm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero6595; AVX2-NEXT: vpsrldq {{.*#+}} xmm2 = xmm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero6596; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]6597; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm13[0,1,2,1]6598; AVX2-NEXT: vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6599; AVX2-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,5]6600; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm8[0,1,2,1]6601; AVX2-NEXT: vmovdqa %xmm8, %xmm56602; AVX2-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6603; AVX2-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,5]6604; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]6605; AVX2-NEXT: vmovdqa 32(%r9), %xmm36606; AVX2-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6607; AVX2-NEXT: vpbroadcastq %xmm1, %ymm16608; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]6609; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]6610; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm4[2,1,3,3,4,5,6,7]6611; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]6612; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]6613; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm3[0,2,2,3,4,5,6,7]6614; AVX2-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,4,4]6615; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]6616; AVX2-NEXT: vpblendvb %ymm0, %ymm1, %ymm2, %ymm16617; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6618; AVX2-NEXT: vmovdqa 64(%rcx), %xmm16619; AVX2-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6620; AVX2-NEXT: vmovdqa 64(%rdx), %xmm26621; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6622; AVX2-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero6623; AVX2-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero6624; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]6625; AVX2-NEXT: vmovdqa 64(%rsi), %xmm96626; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm9[0,1,2,1]6627; AVX2-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,5]6628; AVX2-NEXT: vmovdqa 64(%rdi), %xmm36629; AVX2-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6630; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,1,2,1]6631; AVX2-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,5]6632; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]6633; AVX2-NEXT: vpbroadcastq %xmm1, %ymm16634; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]6635; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]6636; AVX2-NEXT: vmovdqa 64(%r8), %xmm26637; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6638; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,1,3,3,4,5,6,7]6639; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]6640; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]6641; AVX2-NEXT: vmovdqa 64(%r9), %xmm26642; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6643; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]6644; AVX2-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,4,4]6645; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]6646; AVX2-NEXT: vpblendvb %ymm0, %ymm1, %ymm2, %ymm16647; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6648; AVX2-NEXT: vmovdqa 96(%rcx), %xmm16649; AVX2-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6650; AVX2-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero6651; AVX2-NEXT: vmovdqa 96(%rdx), %xmm26652; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6653; AVX2-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero6654; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]6655; AVX2-NEXT: vmovdqa 96(%rsi), %xmm26656; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6657; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,2,1]6658; AVX2-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,5]6659; AVX2-NEXT: vmovdqa 96(%rdi), %xmm36660; AVX2-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6661; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,1,2,1]6662; AVX2-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,5]6663; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]6664; AVX2-NEXT: vpbroadcastq %xmm1, %ymm16665; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]6666; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]6667; AVX2-NEXT: vmovdqa 96(%r8), %xmm26668; AVX2-NEXT: vmovdqa %xmm2, (%rsp) # 16-byte Spill6669; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,1,3,3,4,5,6,7]6670; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]6671; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]6672; AVX2-NEXT: vmovdqa 96(%r9), %xmm26673; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6674; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]6675; AVX2-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,4,4]6676; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]6677; AVX2-NEXT: vpblendvb %ymm0, %ymm1, %ymm2, %ymm16678; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6679; AVX2-NEXT: vmovdqa (%rdx), %ymm26680; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6681; AVX2-NEXT: vmovdqa (%rcx), %ymm16682; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6683; AVX2-NEXT: vpsrldq {{.*#+}} ymm1 = ymm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm1[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero6684; AVX2-NEXT: vpsrldq {{.*#+}} ymm2 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero6685; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]6686; AVX2-NEXT: vmovdqa (%rsi), %ymm26687; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6688; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[2,1,2,3,6,5,6,7]6689; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]6690; AVX2-NEXT: vmovdqa (%rdi), %ymm36691; AVX2-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6692; AVX2-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[2,1,2,3,6,5,6,7]6693; AVX2-NEXT: vpshuflw {{.*#+}} ymm3 = ymm3[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]6694; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]6695; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]6696; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]6697; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]6698; AVX2-NEXT: vmovdqa (%r8), %ymm26699; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6700; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]6701; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]6702; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]6703; AVX2-NEXT: vmovdqa (%r9), %ymm26704; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6705; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]6706; AVX2-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]6707; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]6708; AVX2-NEXT: vpblendvb %ymm0, %ymm1, %ymm2, %ymm16709; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6710; AVX2-NEXT: vmovdqa 32(%rdx), %ymm26711; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6712; AVX2-NEXT: vmovdqa 32(%rcx), %ymm16713; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6714; AVX2-NEXT: vpsrldq {{.*#+}} ymm1 = ymm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm1[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero6715; AVX2-NEXT: vpsrldq {{.*#+}} ymm2 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero6716; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]6717; AVX2-NEXT: vmovdqa 32(%rsi), %ymm76718; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm7[2,1,2,3,6,5,6,7]6719; AVX2-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6720; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]6721; AVX2-NEXT: vmovdqa 32(%rdi), %ymm156722; AVX2-NEXT: vpshufd {{.*#+}} ymm3 = ymm15[2,1,2,3,6,5,6,7]6723; AVX2-NEXT: vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6724; AVX2-NEXT: vpshuflw {{.*#+}} ymm3 = ymm3[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]6725; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]6726; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]6727; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]6728; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]6729; AVX2-NEXT: vmovdqa 32(%r8), %ymm26730; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6731; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]6732; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]6733; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]6734; AVX2-NEXT: vmovdqa 32(%r9), %ymm26735; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6736; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]6737; AVX2-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]6738; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]6739; AVX2-NEXT: vpblendvb %ymm0, %ymm1, %ymm2, %ymm16740; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6741; AVX2-NEXT: vmovdqa 64(%rdx), %ymm26742; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6743; AVX2-NEXT: vmovdqa 64(%rcx), %ymm16744; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6745; AVX2-NEXT: vpsrldq {{.*#+}} ymm1 = ymm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm1[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero6746; AVX2-NEXT: vpsrldq {{.*#+}} ymm2 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero6747; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]6748; AVX2-NEXT: vmovdqa 64(%rsi), %ymm26749; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6750; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[2,1,2,3,6,5,6,7]6751; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]6752; AVX2-NEXT: vmovdqa 64(%rdi), %ymm36753; AVX2-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6754; AVX2-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[2,1,2,3,6,5,6,7]6755; AVX2-NEXT: vpshuflw {{.*#+}} ymm3 = ymm3[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]6756; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]6757; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]6758; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]6759; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]6760; AVX2-NEXT: vmovdqa 64(%r8), %ymm26761; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6762; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]6763; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]6764; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]6765; AVX2-NEXT: vmovdqa 64(%r9), %ymm26766; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6767; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]6768; AVX2-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]6769; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]6770; AVX2-NEXT: vpblendvb %ymm0, %ymm1, %ymm2, %ymm16771; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6772; AVX2-NEXT: vmovdqa 96(%rdx), %ymm36773; AVX2-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6774; AVX2-NEXT: vmovdqa 96(%rcx), %ymm16775; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6776; AVX2-NEXT: vpsrldq {{.*#+}} ymm1 = ymm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm1[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero6777; AVX2-NEXT: vpsrldq {{.*#+}} ymm2 = ymm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm3[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero6778; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]6779; AVX2-NEXT: vmovdqa 96(%rsi), %ymm66780; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm6[2,1,2,3,6,5,6,7]6781; AVX2-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6782; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]6783; AVX2-NEXT: vmovdqa 96(%rdi), %ymm36784; AVX2-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6785; AVX2-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[2,1,2,3,6,5,6,7]6786; AVX2-NEXT: vpshuflw {{.*#+}} ymm3 = ymm3[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]6787; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]6788; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]6789; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]6790; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]6791; AVX2-NEXT: vmovdqa 96(%r8), %ymm86792; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm8[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]6793; AVX2-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6794; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]6795; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]6796; AVX2-NEXT: vmovdqa 96(%r9), %ymm26797; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6798; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]6799; AVX2-NEXT: vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]6800; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]6801; AVX2-NEXT: vpblendvb %ymm0, %ymm1, %ymm2, %ymm06802; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6803; AVX2-NEXT: vmovdqa %xmm14, %xmm46804; AVX2-NEXT: vmovdqa %xmm10, %xmm36805; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]6806; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]6807; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]6808; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,0,2,2]6809; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]6810; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]6811; AVX2-NEXT: vpmovzxwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload6812; AVX2-NEXT: # xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero6813; AVX2-NEXT: vpbroadcastq %xmm1, %ymm16814; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]6815; AVX2-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload6816; AVX2-NEXT: # xmm0 = mem[0,0,2,1,4,5,6,7]6817; AVX2-NEXT: vpbroadcastq %xmm0, %ymm26818; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm0 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]6819; AVX2-NEXT: vpblendvb %ymm0, %ymm1, %ymm2, %ymm16820; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6821; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm13[0],xmm5[1],xmm13[1],xmm5[2],xmm13[2],xmm5[3],xmm13[3]6822; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]6823; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload6824; AVX2-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm2 # 16-byte Folded Reload6825; AVX2-NEXT: # xmm2 = xmm5[0],mem[0],xmm5[1],mem[1],xmm5[2],mem[2],xmm5[3],mem[3]6826; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,0,2,2]6827; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]6828; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7]6829; AVX2-NEXT: vpmovzxwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload6830; AVX2-NEXT: # xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero6831; AVX2-NEXT: vpbroadcastq %xmm2, %ymm26832; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]6833; AVX2-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload6834; AVX2-NEXT: # xmm2 = mem[0,0,2,1,4,5,6,7]6835; AVX2-NEXT: vpbroadcastq %xmm2, %ymm26836; AVX2-NEXT: vpblendvb %ymm0, %ymm1, %ymm2, %ymm16837; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6838; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload6839; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm9[0],xmm1[1],xmm9[1],xmm1[2],xmm9[2],xmm1[3],xmm9[3]6840; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]6841; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload6842; AVX2-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload6843; AVX2-NEXT: # xmm2 = xmm2[0],mem[0],xmm2[1],mem[1],xmm2[2],mem[2],xmm2[3],mem[3]6844; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,0,2,2]6845; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]6846; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7]6847; AVX2-NEXT: vpmovzxwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload6848; AVX2-NEXT: # xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero6849; AVX2-NEXT: vpbroadcastq %xmm2, %ymm26850; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]6851; AVX2-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload6852; AVX2-NEXT: # xmm2 = mem[0,0,2,1,4,5,6,7]6853; AVX2-NEXT: vpbroadcastq %xmm2, %ymm26854; AVX2-NEXT: vpblendvb %ymm0, %ymm1, %ymm2, %ymm16855; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6856; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload6857; AVX2-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload6858; AVX2-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]6859; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]6860; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload6861; AVX2-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload6862; AVX2-NEXT: # xmm2 = xmm2[0],mem[0],xmm2[1],mem[1],xmm2[2],mem[2],xmm2[3],mem[3]6863; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,0,2,2]6864; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]6865; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7]6866; AVX2-NEXT: vpmovzxwd (%rsp), %xmm2 # 16-byte Folded Reload6867; AVX2-NEXT: # xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero6868; AVX2-NEXT: vpbroadcastq %xmm2, %ymm26869; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]6870; AVX2-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload6871; AVX2-NEXT: # xmm2 = mem[0,0,2,1,4,5,6,7]6872; AVX2-NEXT: vpbroadcastq %xmm2, %ymm26873; AVX2-NEXT: vpblendvb %ymm0, %ymm1, %ymm2, %ymm16874; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6875; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload6876; AVX2-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload6877; AVX2-NEXT: # ymm1 = ymm1[0],mem[0],ymm1[1],mem[1],ymm1[2],mem[2],ymm1[3],mem[3],ymm1[8],mem[8],ymm1[9],mem[9],ymm1[10],mem[10],ymm1[11],mem[11]6878; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]6879; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload6880; AVX2-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload6881; AVX2-NEXT: # ymm2 = ymm2[0],mem[0],ymm2[1],mem[1],ymm2[2],mem[2],ymm2[3],mem[3],ymm2[8],mem[8],ymm2[9],mem[9],ymm2[10],mem[10],ymm2[11],mem[11]6882; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[1,0,2,2,5,4,6,6]6883; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]6884; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7]6885; AVX2-NEXT: vpshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload6886; AVX2-NEXT: # ymm2 = mem[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]6887; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]6888; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]6889; AVX2-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload6890; AVX2-NEXT: # ymm2 = mem[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]6891; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]6892; AVX2-NEXT: vpblendvb %ymm0, %ymm1, %ymm2, %ymm16893; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6894; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm15[0],ymm7[0],ymm15[1],ymm7[1],ymm15[2],ymm7[2],ymm15[3],ymm7[3],ymm15[8],ymm7[8],ymm15[9],ymm7[9],ymm15[10],ymm7[10],ymm15[11],ymm7[11]6895; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]6896; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload6897; AVX2-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload6898; AVX2-NEXT: # ymm2 = ymm2[0],mem[0],ymm2[1],mem[1],ymm2[2],mem[2],ymm2[3],mem[3],ymm2[8],mem[8],ymm2[9],mem[9],ymm2[10],mem[10],ymm2[11],mem[11]6899; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[1,0,2,2,5,4,6,6]6900; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]6901; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7]6902; AVX2-NEXT: vpshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload6903; AVX2-NEXT: # ymm2 = mem[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]6904; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]6905; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]6906; AVX2-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload6907; AVX2-NEXT: # ymm2 = mem[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]6908; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]6909; AVX2-NEXT: vpblendvb %ymm0, %ymm1, %ymm2, %ymm16910; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6911; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload6912; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload6913; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm12[0],ymm13[0],ymm12[1],ymm13[1],ymm12[2],ymm13[2],ymm12[3],ymm13[3],ymm12[8],ymm13[8],ymm12[9],ymm13[9],ymm12[10],ymm13[10],ymm12[11],ymm13[11]6914; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]6915; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload6916; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload6917; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm10[0],ymm11[0],ymm10[1],ymm11[1],ymm10[2],ymm11[2],ymm10[3],ymm11[3],ymm10[8],ymm11[8],ymm10[9],ymm11[9],ymm10[10],ymm11[10],ymm10[11],ymm11[11]6918; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[1,0,2,2,5,4,6,6]6919; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]6920; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7]6921; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload6922; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm14[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]6923; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]6924; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]6925; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload6926; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm15[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]6927; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]6928; AVX2-NEXT: vpblendvb %ymm0, %ymm1, %ymm2, %ymm16929; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6930; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload6931; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm7[0],ymm6[0],ymm7[1],ymm6[1],ymm7[2],ymm6[2],ymm7[3],ymm6[3],ymm7[8],ymm6[8],ymm7[9],ymm6[9],ymm7[10],ymm6[10],ymm7[11],ymm6[11]6932; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]6933; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload6934; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload6935; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm2[0],ymm6[0],ymm2[1],ymm6[1],ymm2[2],ymm6[2],ymm2[3],ymm6[3],ymm2[8],ymm6[8],ymm2[9],ymm6[9],ymm2[10],ymm6[10],ymm2[11],ymm6[11]6936; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[1,0,2,2,5,4,6,6]6937; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]6938; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7]6939; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm8[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]6940; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]6941; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]6942; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload6943; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm8[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]6944; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]6945; AVX2-NEXT: vpblendvb %ymm0, %ymm1, %ymm2, %ymm06946; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6947; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]6948; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload6949; AVX2-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload6950; AVX2-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]6951; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,1,1,1]6952; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,2,3,3]6953; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]6954; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]6955; AVX2-NEXT: vmovdqa {{.*#+}} xmm1 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]6956; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload6957; AVX2-NEXT: vpshufb %xmm1, %xmm2, %xmm26958; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]6959; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6],ymm2[7]6960; AVX2-NEXT: vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload6961; AVX2-NEXT: # xmm2 = mem[2,3,2,3]6962; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,2,2,1,4,5,6,7]6963; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm2[0,1,0,1]6964; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm4 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]6965; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm3, %ymm06966; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6967; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6968; AVX2-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload6969; AVX2-NEXT: # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]6970; AVX2-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm3 # 16-byte Folded Reload6971; AVX2-NEXT: # xmm3 = xmm5[4],mem[4],xmm5[5],mem[5],xmm5[6],mem[6],xmm5[7],mem[7]6972; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,1,1,1]6973; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[1,2,3,3]6974; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]6975; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1],ymm0[2],ymm3[3,4],ymm0[5],ymm3[6,7]6976; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload6977; AVX2-NEXT: vpshufb %xmm1, %xmm2, %xmm36978; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,1,0,1]6979; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm3[1],ymm0[2,3],ymm3[4],ymm0[5,6],ymm3[7]6980; AVX2-NEXT: vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload6981; AVX2-NEXT: # xmm3 = mem[2,3,2,3]6982; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,2,2,1,4,5,6,7]6983; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,1,0,1]6984; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm3, %ymm06985; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6986; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload6987; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7]6988; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload6989; AVX2-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm3 # 16-byte Folded Reload6990; AVX2-NEXT: # xmm3 = xmm2[4],mem[4],xmm2[5],mem[5],xmm2[6],mem[6],xmm2[7],mem[7]6991; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,1,1,1]6992; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[1,2,3,3]6993; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]6994; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1],ymm0[2],ymm3[3,4],ymm0[5],ymm3[6,7]6995; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload6996; AVX2-NEXT: vpshufb %xmm1, %xmm2, %xmm36997; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,1,0,1]6998; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm3[1],ymm0[2,3],ymm3[4],ymm0[5,6],ymm3[7]6999; AVX2-NEXT: vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload7000; AVX2-NEXT: # xmm3 = mem[2,3,2,3]7001; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,2,2,1,4,5,6,7]7002; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,1,0,1]7003; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm3, %ymm07004; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7005; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload7006; AVX2-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload7007; AVX2-NEXT: # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]7008; AVX2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload7009; AVX2-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm3 # 16-byte Folded Reload7010; AVX2-NEXT: # xmm3 = xmm2[4],mem[4],xmm2[5],mem[5],xmm2[6],mem[6],xmm2[7],mem[7]7011; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,1,1,1]7012; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[1,2,3,3]7013; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]7014; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1],ymm0[2],ymm3[3,4],ymm0[5],ymm3[6,7]7015; AVX2-NEXT: vmovdqa (%rsp), %xmm2 # 16-byte Reload7016; AVX2-NEXT: vpshufb %xmm1, %xmm2, %xmm17017; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]7018; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]7019; AVX2-NEXT: vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload7020; AVX2-NEXT: # xmm1 = mem[2,3,2,3]7021; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,1,4,5,6,7]7022; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]7023; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm1, %ymm97024; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7025; AVX2-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload7026; AVX2-NEXT: # ymm0 = ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15]7027; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload7028; AVX2-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload7029; AVX2-NEXT: # ymm1 = ymm1[4],mem[4],ymm1[5],mem[5],ymm1[6],mem[6],ymm1[7],mem[7],ymm1[12],mem[12],ymm1[13],mem[13],ymm1[14],mem[14],ymm1[15],mem[15]7030; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,3,3,3]7031; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[1,2,3,3,5,6,7,7]7032; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]7033; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]7034; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]7035; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7036; AVX2-NEXT: vpshufb %ymm2, %ymm0, %ymm37037; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,1,2,3]7038; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5,6],ymm3[7]7039; AVX2-NEXT: vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload7040; AVX2-NEXT: # ymm3 = mem[2,3,2,3,6,7,6,7]7041; AVX2-NEXT: vpshuflw {{.*#+}} ymm3 = ymm3[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]7042; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,1,2,3]7043; AVX2-NEXT: vpblendvb %ymm4, %ymm1, %ymm3, %ymm57044; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7045; AVX2-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload7046; AVX2-NEXT: # ymm1 = ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15]7047; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7048; AVX2-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload7049; AVX2-NEXT: # ymm0 = ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15]7050; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[3,3,3,3]7051; AVX2-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[1,2,3,3,5,6,7,7]7052; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]7053; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]7054; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload7055; AVX2-NEXT: vpshufb %ymm2, %ymm1, %ymm17056; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]7057; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]7058; AVX2-NEXT: vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload7059; AVX2-NEXT: # ymm1 = mem[2,3,2,3,6,7,6,7]7060; AVX2-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]7061; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]7062; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm1, %ymm37063; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm12[4],ymm13[4],ymm12[5],ymm13[5],ymm12[6],ymm13[6],ymm12[7],ymm13[7],ymm12[12],ymm13[12],ymm12[13],ymm13[13],ymm12[14],ymm13[14],ymm12[15],ymm13[15]7064; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm10[4],ymm11[4],ymm10[5],ymm11[5],ymm10[6],ymm11[6],ymm10[7],ymm11[7],ymm10[12],ymm11[12],ymm10[13],ymm11[13],ymm10[14],ymm11[14],ymm10[15],ymm11[15]7065; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,3,3,3]7066; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[1,2,3,3,5,6,7,7]7067; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]7068; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]7069; AVX2-NEXT: vpshufb %ymm2, %ymm14, %ymm17070; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]7071; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]7072; AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm15[2,3,2,3,6,7,6,7]7073; AVX2-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]7074; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]7075; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm1, %ymm07076; AVX2-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm1 # 32-byte Folded Reload7077; AVX2-NEXT: # ymm1 = ymm7[4],mem[4],ymm7[5],mem[5],ymm7[6],mem[6],ymm7[7],mem[7],ymm7[12],mem[12],ymm7[13],mem[13],ymm7[14],mem[14],ymm7[15],mem[15]7078; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload7079; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm11 = ymm7[4],ymm6[4],ymm7[5],ymm6[5],ymm7[6],ymm6[6],ymm7[7],ymm6[7],ymm7[12],ymm6[12],ymm7[13],ymm6[13],ymm7[14],ymm6[14],ymm7[15],ymm6[15]7080; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[3,3,3,3]7081; AVX2-NEXT: vpshufd {{.*#+}} ymm11 = ymm11[1,2,3,3,5,6,7,7]7082; AVX2-NEXT: vpermq {{.*#+}} ymm11 = ymm11[2,2,2,3]7083; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1],ymm1[2],ymm11[3,4],ymm1[5],ymm11[6,7]7084; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload7085; AVX2-NEXT: vpshufb %ymm2, %ymm6, %ymm27086; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]7087; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7]7088; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm8[2,3,2,3,6,7,6,7]7089; AVX2-NEXT: vpshuflw {{.*#+}} ymm2 = ymm2[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]7090; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]7091; AVX2-NEXT: vpblendvb %ymm4, %ymm1, %ymm2, %ymm17092; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax7093; AVX2-NEXT: vmovdqa %ymm1, 736(%rax)7094; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload7095; AVX2-NEXT: vmovaps %ymm1, 672(%rax)7096; AVX2-NEXT: vmovdqa %ymm0, 544(%rax)7097; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7098; AVX2-NEXT: vmovaps %ymm0, 480(%rax)7099; AVX2-NEXT: vmovdqa %ymm3, 352(%rax)7100; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7101; AVX2-NEXT: vmovaps %ymm0, 288(%rax)7102; AVX2-NEXT: vmovdqa %ymm5, 160(%rax)7103; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7104; AVX2-NEXT: vmovaps %ymm0, 96(%rax)7105; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7106; AVX2-NEXT: vmovaps %ymm0, 704(%rax)7107; AVX2-NEXT: vmovdqa %ymm9, 640(%rax)7108; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7109; AVX2-NEXT: vmovaps %ymm0, 576(%rax)7110; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7111; AVX2-NEXT: vmovaps %ymm0, 512(%rax)7112; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7113; AVX2-NEXT: vmovaps %ymm0, 448(%rax)7114; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7115; AVX2-NEXT: vmovaps %ymm0, 384(%rax)7116; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7117; AVX2-NEXT: vmovaps %ymm0, 320(%rax)7118; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7119; AVX2-NEXT: vmovaps %ymm0, 256(%rax)7120; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7121; AVX2-NEXT: vmovaps %ymm0, 192(%rax)7122; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7123; AVX2-NEXT: vmovaps %ymm0, 128(%rax)7124; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7125; AVX2-NEXT: vmovaps %ymm0, 64(%rax)7126; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7127; AVX2-NEXT: vmovaps %ymm0, (%rax)7128; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7129; AVX2-NEXT: vmovaps %ymm0, 608(%rax)7130; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7131; AVX2-NEXT: vmovaps %ymm0, 416(%rax)7132; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7133; AVX2-NEXT: vmovaps %ymm0, 224(%rax)7134; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7135; AVX2-NEXT: vmovaps %ymm0, 32(%rax)7136; AVX2-NEXT: addq $1544, %rsp # imm = 0x6087137; AVX2-NEXT: vzeroupper7138; AVX2-NEXT: retq7139;7140; AVX2-FP-LABEL: store_i16_stride6_vf64:7141; AVX2-FP: # %bb.0:7142; AVX2-FP-NEXT: subq $1544, %rsp # imm = 0x6087143; AVX2-FP-NEXT: vmovdqa (%rsi), %xmm07144; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7145; AVX2-FP-NEXT: vmovdqa 32(%rsi), %xmm57146; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm17147; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7148; AVX2-FP-NEXT: vmovdqa 32(%rdi), %xmm47149; AVX2-FP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7150; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]7151; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]7152; AVX2-FP-NEXT: vmovdqa (%rcx), %xmm17153; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7154; AVX2-FP-NEXT: vmovdqa 32(%rcx), %xmm117155; AVX2-FP-NEXT: vmovdqa (%rdx), %xmm27156; AVX2-FP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7157; AVX2-FP-NEXT: vmovdqa 32(%rdx), %xmm67158; AVX2-FP-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7159; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]7160; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,0,2,2]7161; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]7162; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]7163; AVX2-FP-NEXT: vmovdqa (%r8), %xmm17164; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7165; AVX2-FP-NEXT: vmovdqa 32(%r8), %xmm27166; AVX2-FP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7167; AVX2-FP-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero7168; AVX2-FP-NEXT: vpbroadcastq %xmm1, %ymm17169; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]7170; AVX2-FP-NEXT: vmovdqa (%r9), %xmm17171; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7172; AVX2-FP-NEXT: vmovdqa 32(%r9), %xmm37173; AVX2-FP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7174; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,1,4,5,6,7]7175; AVX2-FP-NEXT: vpbroadcastq %xmm1, %ymm17176; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm10 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]7177; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm0, %ymm1, %ymm07178; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7179; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]7180; AVX2-FP-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7181; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]7182; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm6[0],xmm11[0],xmm6[1],xmm11[1],xmm6[2],xmm11[2],xmm6[3],xmm11[3]7183; AVX2-FP-NEXT: vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7184; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,0,2,2]7185; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]7186; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]7187; AVX2-FP-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero7188; AVX2-FP-NEXT: vpbroadcastq %xmm1, %ymm17189; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]7190; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm3[0,0,2,1,4,5,6,7]7191; AVX2-FP-NEXT: vpbroadcastq %xmm1, %ymm17192; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm0, %ymm1, %ymm07193; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7194; AVX2-FP-NEXT: vmovdqa 64(%rsi), %xmm127195; AVX2-FP-NEXT: vmovdqa 64(%rdi), %xmm07196; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7197; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm12[0],xmm0[1],xmm12[1],xmm0[2],xmm12[2],xmm0[3],xmm12[3]7198; AVX2-FP-NEXT: vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7199; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]7200; AVX2-FP-NEXT: vmovdqa 64(%rcx), %xmm17201; AVX2-FP-NEXT: vmovdqa %xmm1, (%rsp) # 16-byte Spill7202; AVX2-FP-NEXT: vmovdqa 64(%rdx), %xmm27203; AVX2-FP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7204; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]7205; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,0,2,2]7206; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]7207; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]7208; AVX2-FP-NEXT: vmovdqa 64(%r8), %xmm17209; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7210; AVX2-FP-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero7211; AVX2-FP-NEXT: vpbroadcastq %xmm1, %ymm17212; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]7213; AVX2-FP-NEXT: vmovdqa 64(%r9), %xmm17214; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7215; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,1,4,5,6,7]7216; AVX2-FP-NEXT: vpbroadcastq %xmm1, %ymm17217; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm0, %ymm1, %ymm07218; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7219; AVX2-FP-NEXT: vmovdqa 96(%rsi), %xmm67220; AVX2-FP-NEXT: vmovdqa 96(%rdi), %xmm07221; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7222; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3]7223; AVX2-FP-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7224; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]7225; AVX2-FP-NEXT: vmovdqa 96(%rcx), %xmm17226; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7227; AVX2-FP-NEXT: vmovdqa 96(%rdx), %xmm27228; AVX2-FP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7229; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]7230; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,0,2,2]7231; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]7232; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]7233; AVX2-FP-NEXT: vmovdqa 96(%r8), %xmm17234; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7235; AVX2-FP-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero7236; AVX2-FP-NEXT: vpbroadcastq %xmm1, %ymm17237; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]7238; AVX2-FP-NEXT: vmovdqa 96(%r9), %xmm17239; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7240; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,1,4,5,6,7]7241; AVX2-FP-NEXT: vpbroadcastq %xmm1, %ymm17242; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm0, %ymm1, %ymm07243; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7244; AVX2-FP-NEXT: vmovdqa (%rdi), %ymm147245; AVX2-FP-NEXT: vmovdqa (%rsi), %ymm07246; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7247; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm14[0],ymm0[0],ymm14[1],ymm0[1],ymm14[2],ymm0[2],ymm14[3],ymm0[3],ymm14[8],ymm0[8],ymm14[9],ymm0[9],ymm14[10],ymm0[10],ymm14[11],ymm0[11]7248; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]7249; AVX2-FP-NEXT: vmovdqa (%rdx), %ymm137250; AVX2-FP-NEXT: vmovdqa (%rcx), %ymm97251; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm13[0],ymm9[0],ymm13[1],ymm9[1],ymm13[2],ymm9[2],ymm13[3],ymm9[3],ymm13[8],ymm9[8],ymm13[9],ymm9[9],ymm13[10],ymm9[10],ymm13[11],ymm9[11]7252; AVX2-FP-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7253; AVX2-FP-NEXT: vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7254; AVX2-FP-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[1,0,2,2,5,4,6,6]7255; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]7256; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]7257; AVX2-FP-NEXT: vmovdqa (%r8), %ymm17258; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7259; AVX2-FP-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]7260; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]7261; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]7262; AVX2-FP-NEXT: vmovdqa (%r9), %ymm17263; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7264; AVX2-FP-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]7265; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]7266; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm0, %ymm1, %ymm07267; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7268; AVX2-FP-NEXT: vmovdqa 32(%rdi), %ymm17269; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7270; AVX2-FP-NEXT: vmovdqa 32(%rsi), %ymm07271; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7272; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]7273; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]7274; AVX2-FP-NEXT: vmovdqa 32(%rdx), %ymm157275; AVX2-FP-NEXT: vmovdqa 32(%rcx), %ymm17276; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7277; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm15[0],ymm1[0],ymm15[1],ymm1[1],ymm15[2],ymm1[2],ymm15[3],ymm1[3],ymm15[8],ymm1[8],ymm15[9],ymm1[9],ymm15[10],ymm1[10],ymm15[11],ymm1[11]7278; AVX2-FP-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[1,0,2,2,5,4,6,6]7279; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]7280; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]7281; AVX2-FP-NEXT: vmovdqa 32(%r8), %ymm17282; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7283; AVX2-FP-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]7284; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]7285; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]7286; AVX2-FP-NEXT: vmovdqa 32(%r9), %ymm17287; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7288; AVX2-FP-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]7289; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]7290; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm0, %ymm1, %ymm07291; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7292; AVX2-FP-NEXT: vmovdqa 64(%rdi), %ymm17293; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7294; AVX2-FP-NEXT: vmovdqa 64(%rsi), %ymm07295; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7296; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]7297; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]7298; AVX2-FP-NEXT: vmovdqa 64(%rdx), %ymm87299; AVX2-FP-NEXT: vmovdqa 64(%rcx), %ymm17300; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7301; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm8[0],ymm1[0],ymm8[1],ymm1[1],ymm8[2],ymm1[2],ymm8[3],ymm1[3],ymm8[8],ymm1[8],ymm8[9],ymm1[9],ymm8[10],ymm1[10],ymm8[11],ymm1[11]7302; AVX2-FP-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7303; AVX2-FP-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[1,0,2,2,5,4,6,6]7304; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]7305; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]7306; AVX2-FP-NEXT: vmovdqa 64(%r8), %ymm17307; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7308; AVX2-FP-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]7309; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]7310; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]7311; AVX2-FP-NEXT: vmovdqa 64(%r9), %ymm17312; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7313; AVX2-FP-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]7314; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]7315; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm0, %ymm1, %ymm07316; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7317; AVX2-FP-NEXT: vmovdqa 96(%rdi), %ymm07318; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7319; AVX2-FP-NEXT: vmovdqa 96(%rsi), %ymm77320; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm7[0],ymm0[1],ymm7[1],ymm0[2],ymm7[2],ymm0[3],ymm7[3],ymm0[8],ymm7[8],ymm0[9],ymm7[9],ymm0[10],ymm7[10],ymm0[11],ymm7[11]7321; AVX2-FP-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7322; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]7323; AVX2-FP-NEXT: vmovdqa 96(%rdx), %ymm27324; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7325; AVX2-FP-NEXT: vmovdqa 96(%rcx), %ymm17326; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7327; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]7328; AVX2-FP-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[1,0,2,2,5,4,6,6]7329; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]7330; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]7331; AVX2-FP-NEXT: vmovdqa 96(%r8), %ymm17332; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7333; AVX2-FP-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]7334; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]7335; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]7336; AVX2-FP-NEXT: vmovdqa 96(%r9), %ymm17337; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7338; AVX2-FP-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]7339; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]7340; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm0, %ymm1, %ymm07341; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7342; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7,8,9,6,7,4,5,10,11]7343; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload7344; AVX2-FP-NEXT: vpshufb %xmm0, %xmm1, %xmm17345; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload7346; AVX2-FP-NEXT: vpshufb %xmm0, %xmm2, %xmm27347; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]7348; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload7349; AVX2-FP-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero7350; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload7351; AVX2-FP-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero7352; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]7353; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]7354; AVX2-FP-NEXT: vpbroadcastq %xmm2, %ymm27355; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]7356; AVX2-FP-NEXT: vpshuflw $246, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload7357; AVX2-FP-NEXT: # xmm2 = mem[2,1,3,3,4,5,6,7]7358; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]7359; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]7360; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]7361; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload7362; AVX2-FP-NEXT: vpshufb %xmm1, %xmm3, %xmm37363; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]7364; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm4 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]7365; AVX2-FP-NEXT: vpblendvb %ymm4, %ymm2, %ymm3, %ymm27366; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7367; AVX2-FP-NEXT: vpshufb %xmm0, %xmm5, %xmm27368; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload7369; AVX2-FP-NEXT: vpshufb %xmm0, %xmm3, %xmm37370; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]7371; AVX2-FP-NEXT: vpsrldq {{.*#+}} xmm3 = xmm11[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero7372; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload7373; AVX2-FP-NEXT: vpsrldq {{.*#+}} xmm5 = xmm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero7374; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]7375; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]7376; AVX2-FP-NEXT: vpbroadcastq %xmm3, %ymm37377; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]7378; AVX2-FP-NEXT: vpshuflw $246, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload7379; AVX2-FP-NEXT: # xmm3 = mem[2,1,3,3,4,5,6,7]7380; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]7381; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1,2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7]7382; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload7383; AVX2-FP-NEXT: vpshufb %xmm1, %xmm3, %xmm37384; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]7385; AVX2-FP-NEXT: vpblendvb %ymm4, %ymm2, %ymm3, %ymm27386; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7387; AVX2-FP-NEXT: vpshufb %xmm0, %xmm12, %xmm27388; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload7389; AVX2-FP-NEXT: vpshufb %xmm0, %xmm3, %xmm37390; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]7391; AVX2-FP-NEXT: vmovdqa (%rsp), %xmm3 # 16-byte Reload7392; AVX2-FP-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero7393; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload7394; AVX2-FP-NEXT: vpsrldq {{.*#+}} xmm5 = xmm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero7395; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]7396; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]7397; AVX2-FP-NEXT: vpbroadcastq %xmm3, %ymm37398; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]7399; AVX2-FP-NEXT: vpshuflw $246, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload7400; AVX2-FP-NEXT: # xmm3 = mem[2,1,3,3,4,5,6,7]7401; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]7402; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1,2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7]7403; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload7404; AVX2-FP-NEXT: vpshufb %xmm1, %xmm3, %xmm37405; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]7406; AVX2-FP-NEXT: vpblendvb %ymm4, %ymm2, %ymm3, %ymm27407; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7408; AVX2-FP-NEXT: vpshufb %xmm0, %xmm6, %xmm27409; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload7410; AVX2-FP-NEXT: vpshufb %xmm0, %xmm3, %xmm07411; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]7412; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload7413; AVX2-FP-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero7414; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload7415; AVX2-FP-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero7416; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]7417; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]7418; AVX2-FP-NEXT: vpbroadcastq %xmm2, %ymm27419; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2],ymm0[3,4],ymm2[5],ymm0[6,7]7420; AVX2-FP-NEXT: vpshuflw $246, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload7421; AVX2-FP-NEXT: # xmm2 = mem[2,1,3,3,4,5,6,7]7422; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]7423; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7]7424; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload7425; AVX2-FP-NEXT: vpshufb %xmm1, %xmm2, %xmm17426; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]7427; AVX2-FP-NEXT: vpblendvb %ymm4, %ymm0, %ymm1, %ymm07428; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7429; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm0 = [u,u,u,u,4,5,10,11,u,u,u,u,u,u,u,u,24,25,22,23,20,21,26,27,u,u,u,u,u,u,u,u]7430; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload7431; AVX2-FP-NEXT: vpshufb %ymm0, %ymm1, %ymm17432; AVX2-FP-NEXT: vmovdqa %ymm14, %ymm67433; AVX2-FP-NEXT: vpshufb %ymm0, %ymm14, %ymm27434; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]7435; AVX2-FP-NEXT: vpsrldq {{.*#+}} ymm2 = ymm9[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm9[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero7436; AVX2-FP-NEXT: vpsrldq {{.*#+}} ymm3 = ymm13[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm13[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero7437; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]7438; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]7439; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]7440; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]7441; AVX2-FP-NEXT: vpshuflw $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload7442; AVX2-FP-NEXT: # ymm2 = mem[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]7443; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]7444; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]7445; AVX2-FP-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25,16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25]7446; AVX2-FP-NEXT: # ymm1 = mem[0,1,0,1]7447; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload7448; AVX2-FP-NEXT: vpshufb %ymm1, %ymm3, %ymm37449; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]7450; AVX2-FP-NEXT: vpblendvb %ymm4, %ymm2, %ymm3, %ymm27451; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7452; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload7453; AVX2-FP-NEXT: vpshufb %ymm0, %ymm2, %ymm27454; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload7455; AVX2-FP-NEXT: vpshufb %ymm0, %ymm3, %ymm37456; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]7457; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload7458; AVX2-FP-NEXT: vpsrldq {{.*#+}} ymm3 = ymm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm3[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero7459; AVX2-FP-NEXT: vmovdqa %ymm15, %ymm137460; AVX2-FP-NEXT: vpsrldq {{.*#+}} ymm5 = ymm15[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm15[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero7461; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm3 = ymm5[0],ymm3[0],ymm5[1],ymm3[1],ymm5[2],ymm3[2],ymm5[3],ymm3[3],ymm5[8],ymm3[8],ymm5[9],ymm3[9],ymm5[10],ymm3[10],ymm5[11],ymm3[11]7462; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]7463; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,2]7464; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]7465; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload7466; AVX2-FP-NEXT: vpshuflw {{.*#+}} ymm3 = ymm14[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]7467; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]7468; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1,2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7]7469; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload7470; AVX2-FP-NEXT: vpshufb %ymm1, %ymm15, %ymm37471; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]7472; AVX2-FP-NEXT: vpblendvb %ymm4, %ymm2, %ymm3, %ymm27473; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7474; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload7475; AVX2-FP-NEXT: vpshufb %ymm0, %ymm2, %ymm27476; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload7477; AVX2-FP-NEXT: vpshufb %ymm0, %ymm3, %ymm37478; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]7479; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload7480; AVX2-FP-NEXT: vpsrldq {{.*#+}} ymm3 = ymm12[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm12[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero7481; AVX2-FP-NEXT: vpsrldq {{.*#+}} ymm5 = ymm8[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm8[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero7482; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm3 = ymm5[0],ymm3[0],ymm5[1],ymm3[1],ymm5[2],ymm3[2],ymm5[3],ymm3[3],ymm5[8],ymm3[8],ymm5[9],ymm3[9],ymm5[10],ymm3[10],ymm5[11],ymm3[11]7483; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]7484; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,2]7485; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]7486; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload7487; AVX2-FP-NEXT: vpshuflw {{.*#+}} ymm3 = ymm11[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]7488; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]7489; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1,2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7]7490; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload7491; AVX2-FP-NEXT: vpshufb %ymm1, %ymm3, %ymm37492; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]7493; AVX2-FP-NEXT: vpblendvb %ymm4, %ymm2, %ymm3, %ymm27494; AVX2-FP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7495; AVX2-FP-NEXT: vpshufb %ymm0, %ymm7, %ymm27496; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload7497; AVX2-FP-NEXT: vpshufb %ymm0, %ymm10, %ymm07498; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11]7499; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload7500; AVX2-FP-NEXT: vpsrldq {{.*#+}} ymm2 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero7501; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload7502; AVX2-FP-NEXT: vpsrldq {{.*#+}} ymm3 = ymm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm7[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero7503; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]7504; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,1,2,3]7505; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]7506; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2],ymm0[3,4],ymm2[5],ymm0[6,7]7507; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload7508; AVX2-FP-NEXT: vpshuflw {{.*#+}} ymm2 = ymm8[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]7509; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]7510; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7]7511; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload7512; AVX2-FP-NEXT: vpshufb %ymm1, %ymm9, %ymm17513; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]7514; AVX2-FP-NEXT: vpblendvb %ymm4, %ymm0, %ymm1, %ymm07515; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7516; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload7517; AVX2-FP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload7518; AVX2-FP-NEXT: # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]7519; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload7520; AVX2-FP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload7521; AVX2-FP-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]7522; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,1,1,1]7523; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,2,3,3]7524; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]7525; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]7526; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm1 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]7527; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload7528; AVX2-FP-NEXT: vpshufb %xmm1, %xmm2, %xmm27529; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]7530; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6],ymm2[7]7531; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm2 = [8,9,12,13,12,13,10,11,8,9,10,11,12,13,14,15]7532; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload7533; AVX2-FP-NEXT: vpshufb %xmm2, %xmm0, %xmm07534; AVX2-FP-NEXT: vpermq {{.*#+}} ymm4 = ymm0[0,1,0,1]7535; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm5 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]7536; AVX2-FP-NEXT: vpblendvb %ymm5, %ymm3, %ymm4, %ymm07537; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7538; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload7539; AVX2-FP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm3 # 16-byte Folded Reload7540; AVX2-FP-NEXT: # xmm3 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]7541; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload7542; AVX2-FP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm4 # 16-byte Folded Reload7543; AVX2-FP-NEXT: # xmm4 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]7544; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[1,1,1,1]7545; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[1,2,3,3]7546; AVX2-FP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,0,2,1]7547; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7]7548; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload7549; AVX2-FP-NEXT: vpshufb %xmm1, %xmm0, %xmm47550; AVX2-FP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,1,0,1]7551; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7]7552; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload7553; AVX2-FP-NEXT: vpshufb %xmm2, %xmm0, %xmm47554; AVX2-FP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,1,0,1]7555; AVX2-FP-NEXT: vpblendvb %ymm5, %ymm3, %ymm4, %ymm07556; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7557; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload7558; AVX2-FP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm3 # 16-byte Folded Reload7559; AVX2-FP-NEXT: # xmm3 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]7560; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload7561; AVX2-FP-NEXT: vpunpckhwd (%rsp), %xmm0, %xmm4 # 16-byte Folded Reload7562; AVX2-FP-NEXT: # xmm4 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]7563; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[1,1,1,1]7564; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[1,2,3,3]7565; AVX2-FP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,0,2,1]7566; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7]7567; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload7568; AVX2-FP-NEXT: vpshufb %xmm1, %xmm0, %xmm47569; AVX2-FP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,1,0,1]7570; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7]7571; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload7572; AVX2-FP-NEXT: vpshufb %xmm2, %xmm0, %xmm47573; AVX2-FP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,1,0,1]7574; AVX2-FP-NEXT: vpblendvb %ymm5, %ymm3, %ymm4, %ymm07575; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7576; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload7577; AVX2-FP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm3 # 16-byte Folded Reload7578; AVX2-FP-NEXT: # xmm3 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]7579; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload7580; AVX2-FP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm4 # 16-byte Folded Reload7581; AVX2-FP-NEXT: # xmm4 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]7582; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[1,1,1,1]7583; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[1,2,3,3]7584; AVX2-FP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,0,2,1]7585; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7]7586; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload7587; AVX2-FP-NEXT: vpshufb %xmm1, %xmm0, %xmm17588; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]7589; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0],ymm1[1],ymm3[2,3],ymm1[4],ymm3[5,6],ymm1[7]7590; AVX2-FP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload7591; AVX2-FP-NEXT: vpshufb %xmm2, %xmm0, %xmm27592; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]7593; AVX2-FP-NEXT: vpblendvb %ymm5, %ymm1, %ymm2, %ymm07594; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7595; AVX2-FP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm1 # 32-byte Folded Reload7596; AVX2-FP-NEXT: # ymm1 = ymm6[4],mem[4],ymm6[5],mem[5],ymm6[6],mem[6],ymm6[7],mem[7],ymm6[12],mem[12],ymm6[13],mem[13],ymm6[14],mem[14],ymm6[15],mem[15]7597; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7598; AVX2-FP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload7599; AVX2-FP-NEXT: # ymm2 = ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15]7600; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[3,3,3,3]7601; AVX2-FP-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[1,2,3,3,5,6,7,7]7602; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]7603; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7]7604; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm6 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]7605; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7606; AVX2-FP-NEXT: vpshufb %ymm6, %ymm0, %ymm37607; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,1,2,3]7608; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5,6],ymm3[7]7609; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm2 = [u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,24,25,28,29,28,29,26,27,24,25,26,27,28,29,30,31]7610; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7611; AVX2-FP-NEXT: vpshufb %ymm2, %ymm0, %ymm47612; AVX2-FP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,1,2,3]7613; AVX2-FP-NEXT: vpblendvb %ymm5, %ymm3, %ymm4, %ymm47614; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7615; AVX2-FP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload7616; AVX2-FP-NEXT: # ymm3 = ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15]7617; AVX2-FP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm0 # 32-byte Folded Reload7618; AVX2-FP-NEXT: # ymm0 = ymm13[4],mem[4],ymm13[5],mem[5],ymm13[6],mem[6],ymm13[7],mem[7],ymm13[12],mem[12],ymm13[13],mem[13],ymm13[14],mem[14],ymm13[15],mem[15]7619; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[3,3,3,3]7620; AVX2-FP-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[1,2,3,3,5,6,7,7]7621; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]7622; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm3[2],ymm0[3,4],ymm3[5],ymm0[6,7]7623; AVX2-FP-NEXT: vpshufb %ymm6, %ymm14, %ymm37624; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,1,2,3]7625; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm3[1],ymm0[2,3],ymm3[4],ymm0[5,6],ymm3[7]7626; AVX2-FP-NEXT: vpshufb %ymm2, %ymm15, %ymm37627; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,1,2,3]7628; AVX2-FP-NEXT: vpblendvb %ymm5, %ymm0, %ymm3, %ymm37629; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7630; AVX2-FP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload7631; AVX2-FP-NEXT: # ymm0 = ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15]7632; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload7633; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm12[4],ymm1[5],ymm12[5],ymm1[6],ymm12[6],ymm1[7],ymm12[7],ymm1[12],ymm12[12],ymm1[13],ymm12[13],ymm1[14],ymm12[14],ymm1[15],ymm12[15]7634; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,3,3,3]7635; AVX2-FP-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[1,2,3,3,5,6,7,7]7636; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]7637; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7]7638; AVX2-FP-NEXT: vpshufb %ymm6, %ymm11, %ymm17639; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]7640; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]7641; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload7642; AVX2-FP-NEXT: vpshufb %ymm2, %ymm1, %ymm17643; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]7644; AVX2-FP-NEXT: vpblendvb %ymm5, %ymm0, %ymm1, %ymm07645; AVX2-FP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm1 # 32-byte Folded Reload7646; AVX2-FP-NEXT: # ymm1 = ymm10[4],mem[4],ymm10[5],mem[5],ymm10[6],mem[6],ymm10[7],mem[7],ymm10[12],mem[12],ymm10[13],mem[13],ymm10[14],mem[14],ymm10[15],mem[15]7647; AVX2-FP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm11 # 32-byte Folded Reload7648; AVX2-FP-NEXT: # ymm11 = ymm7[4],mem[4],ymm7[5],mem[5],ymm7[6],mem[6],ymm7[7],mem[7],ymm7[12],mem[12],ymm7[13],mem[13],ymm7[14],mem[14],ymm7[15],mem[15]7649; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[3,3,3,3]7650; AVX2-FP-NEXT: vpshufd {{.*#+}} ymm11 = ymm11[1,2,3,3,5,6,7,7]7651; AVX2-FP-NEXT: vpermq {{.*#+}} ymm11 = ymm11[2,2,2,3]7652; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1],ymm1[2],ymm11[3,4],ymm1[5],ymm11[6,7]7653; AVX2-FP-NEXT: vpshufb %ymm6, %ymm8, %ymm67654; AVX2-FP-NEXT: vpermq {{.*#+}} ymm6 = ymm6[2,1,2,3]7655; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0],ymm6[1],ymm1[2,3],ymm6[4],ymm1[5,6],ymm6[7]7656; AVX2-FP-NEXT: vpshufb %ymm2, %ymm9, %ymm27657; AVX2-FP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]7658; AVX2-FP-NEXT: vpblendvb %ymm5, %ymm1, %ymm2, %ymm17659; AVX2-FP-NEXT: movq {{[0-9]+}}(%rsp), %rax7660; AVX2-FP-NEXT: vmovdqa %ymm1, 736(%rax)7661; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload7662; AVX2-FP-NEXT: vmovaps %ymm1, 704(%rax)7663; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload7664; AVX2-FP-NEXT: vmovaps %ymm1, 672(%rax)7665; AVX2-FP-NEXT: vmovdqa %ymm0, 544(%rax)7666; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7667; AVX2-FP-NEXT: vmovaps %ymm0, 512(%rax)7668; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7669; AVX2-FP-NEXT: vmovaps %ymm0, 480(%rax)7670; AVX2-FP-NEXT: vmovdqa %ymm3, 352(%rax)7671; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7672; AVX2-FP-NEXT: vmovaps %ymm0, 320(%rax)7673; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7674; AVX2-FP-NEXT: vmovaps %ymm0, 288(%rax)7675; AVX2-FP-NEXT: vmovdqa %ymm4, 160(%rax)7676; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7677; AVX2-FP-NEXT: vmovaps %ymm0, 128(%rax)7678; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7679; AVX2-FP-NEXT: vmovaps %ymm0, 96(%rax)7680; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7681; AVX2-FP-NEXT: vmovaps %ymm0, 640(%rax)7682; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7683; AVX2-FP-NEXT: vmovaps %ymm0, 608(%rax)7684; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7685; AVX2-FP-NEXT: vmovaps %ymm0, 576(%rax)7686; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7687; AVX2-FP-NEXT: vmovaps %ymm0, 448(%rax)7688; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7689; AVX2-FP-NEXT: vmovaps %ymm0, 416(%rax)7690; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7691; AVX2-FP-NEXT: vmovaps %ymm0, 384(%rax)7692; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7693; AVX2-FP-NEXT: vmovaps %ymm0, 256(%rax)7694; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7695; AVX2-FP-NEXT: vmovaps %ymm0, 224(%rax)7696; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7697; AVX2-FP-NEXT: vmovaps %ymm0, 192(%rax)7698; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7699; AVX2-FP-NEXT: vmovaps %ymm0, 64(%rax)7700; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7701; AVX2-FP-NEXT: vmovaps %ymm0, 32(%rax)7702; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7703; AVX2-FP-NEXT: vmovaps %ymm0, (%rax)7704; AVX2-FP-NEXT: addq $1544, %rsp # imm = 0x6087705; AVX2-FP-NEXT: vzeroupper7706; AVX2-FP-NEXT: retq7707;7708; AVX2-FCP-LABEL: store_i16_stride6_vf64:7709; AVX2-FCP: # %bb.0:7710; AVX2-FCP-NEXT: subq $1560, %rsp # imm = 0x6187711; AVX2-FCP-NEXT: vmovdqa (%rsi), %xmm17712; AVX2-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7713; AVX2-FCP-NEXT: vmovdqa 32(%rsi), %xmm57714; AVX2-FCP-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7715; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7,8,9,6,7,4,5,10,11]7716; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm1, %xmm17717; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm27718; AVX2-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7719; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %xmm77720; AVX2-FCP-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7721; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm2, %xmm27722; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]7723; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]7724; AVX2-FCP-NEXT: vmovdqa (%rcx), %xmm27725; AVX2-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7726; AVX2-FCP-NEXT: vmovdqa 32(%rcx), %xmm87727; AVX2-FCP-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7728; AVX2-FCP-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero7729; AVX2-FCP-NEXT: vmovdqa (%rdx), %xmm37730; AVX2-FCP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7731; AVX2-FCP-NEXT: vmovdqa 32(%rdx), %xmm147732; AVX2-FCP-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero7733; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]7734; AVX2-FCP-NEXT: vpbroadcastq %xmm2, %ymm27735; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]7736; AVX2-FCP-NEXT: vmovdqa (%r8), %xmm27737; AVX2-FCP-NEXT: vmovdqa %xmm2, (%rsp) # 16-byte Spill7738; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,1,3,3,4,5,6,7]7739; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]7740; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]7741; AVX2-FCP-NEXT: vmovdqa (%r9), %xmm17742; AVX2-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7743; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]7744; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm1, %xmm17745; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm1[0,0,2,1]7746; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm1 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]7747; AVX2-FCP-NEXT: vpblendvb %ymm1, %ymm3, %ymm4, %ymm37748; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7749; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm5, %xmm37750; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm7, %xmm47751; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]7752; AVX2-FCP-NEXT: vpsrldq {{.*#+}} xmm4 = xmm8[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero7753; AVX2-FCP-NEXT: vpsrldq {{.*#+}} xmm5 = xmm14[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero7754; AVX2-FCP-NEXT: vmovdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7755; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]7756; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,1,0,1]7757; AVX2-FCP-NEXT: vpbroadcastq %xmm4, %ymm47758; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]7759; AVX2-FCP-NEXT: vmovdqa 32(%r8), %xmm47760; AVX2-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7761; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[2,1,3,3,4,5,6,7]7762; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,0,2,1]7763; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0],ymm3[1,2],ymm4[3],ymm3[4,5],ymm4[6],ymm3[7]7764; AVX2-FCP-NEXT: vmovdqa 32(%r9), %xmm47765; AVX2-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7766; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm4, %xmm47767; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,0,2,1]7768; AVX2-FCP-NEXT: vpblendvb %ymm1, %ymm3, %ymm4, %ymm37769; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7770; AVX2-FCP-NEXT: vmovdqa 64(%rsi), %xmm37771; AVX2-FCP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7772; AVX2-FCP-NEXT: vmovdqa 64(%rdi), %xmm47773; AVX2-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7774; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm37775; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm4, %xmm47776; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]7777; AVX2-FCP-NEXT: vmovdqa 64(%rcx), %xmm47778; AVX2-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7779; AVX2-FCP-NEXT: vmovdqa 64(%rdx), %xmm57780; AVX2-FCP-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7781; AVX2-FCP-NEXT: vpsrldq {{.*#+}} xmm4 = xmm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero7782; AVX2-FCP-NEXT: vpsrldq {{.*#+}} xmm5 = xmm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero7783; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]7784; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,1,0,1]7785; AVX2-FCP-NEXT: vpbroadcastq %xmm4, %ymm47786; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]7787; AVX2-FCP-NEXT: vmovdqa 64(%r8), %xmm47788; AVX2-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7789; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[2,1,3,3,4,5,6,7]7790; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,0,2,1]7791; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0],ymm3[1,2],ymm4[3],ymm3[4,5],ymm4[6],ymm3[7]7792; AVX2-FCP-NEXT: vmovdqa 64(%r9), %xmm47793; AVX2-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7794; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm4, %xmm47795; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,0,2,1]7796; AVX2-FCP-NEXT: vpblendvb %ymm1, %ymm3, %ymm4, %ymm37797; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7798; AVX2-FCP-NEXT: vmovdqa 96(%rsi), %xmm37799; AVX2-FCP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7800; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm37801; AVX2-FCP-NEXT: vmovdqa 96(%rdi), %xmm47802; AVX2-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7803; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm4, %xmm07804; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]7805; AVX2-FCP-NEXT: vmovdqa 96(%rcx), %xmm137806; AVX2-FCP-NEXT: vpsrldq {{.*#+}} xmm3 = xmm13[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero7807; AVX2-FCP-NEXT: vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7808; AVX2-FCP-NEXT: vmovdqa 96(%rdx), %xmm127809; AVX2-FCP-NEXT: vpsrldq {{.*#+}} xmm4 = xmm12[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero7810; AVX2-FCP-NEXT: vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7811; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]7812; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]7813; AVX2-FCP-NEXT: vpbroadcastq %xmm3, %ymm37814; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm3[2],ymm0[3,4],ymm3[5],ymm0[6,7]7815; AVX2-FCP-NEXT: vmovdqa 96(%r8), %xmm37816; AVX2-FCP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7817; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,1,3,3,4,5,6,7]7818; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]7819; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0],ymm0[1,2],ymm3[3],ymm0[4,5],ymm3[6],ymm0[7]7820; AVX2-FCP-NEXT: vmovdqa 96(%r9), %xmm37821; AVX2-FCP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill7822; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm3, %xmm27823; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]7824; AVX2-FCP-NEXT: vpblendvb %ymm1, %ymm0, %ymm2, %ymm07825; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7826; AVX2-FCP-NEXT: vmovdqa (%rdi), %ymm37827; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7828; AVX2-FCP-NEXT: vmovdqa (%rsi), %ymm27829; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7830; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm0 = [u,u,u,u,4,5,10,11,u,u,u,u,u,u,u,u,24,25,22,23,20,21,26,27,u,u,u,u,u,u,u,u]7831; AVX2-FCP-NEXT: vpshufb %ymm0, %ymm2, %ymm27832; AVX2-FCP-NEXT: vpshufb %ymm0, %ymm3, %ymm37833; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]7834; AVX2-FCP-NEXT: vmovdqa (%rdx), %ymm47835; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7836; AVX2-FCP-NEXT: vmovdqa (%rcx), %ymm37837; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7838; AVX2-FCP-NEXT: vpsrldq {{.*#+}} ymm3 = ymm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm3[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero7839; AVX2-FCP-NEXT: vpsrldq {{.*#+}} ymm4 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero7840; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]7841; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]7842; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,2]7843; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]7844; AVX2-FCP-NEXT: vmovdqa (%r8), %ymm37845; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7846; AVX2-FCP-NEXT: vpshuflw {{.*#+}} ymm3 = ymm3[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]7847; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]7848; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0],ymm2[1,2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7]7849; AVX2-FCP-NEXT: vmovdqa (%r9), %ymm47850; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7851; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25,16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25]7852; AVX2-FCP-NEXT: # ymm2 = mem[0,1,0,1]7853; AVX2-FCP-NEXT: vpshufb %ymm2, %ymm4, %ymm47854; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]7855; AVX2-FCP-NEXT: vpblendvb %ymm1, %ymm3, %ymm4, %ymm37856; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7857; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %ymm47858; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7859; AVX2-FCP-NEXT: vmovdqa 32(%rsi), %ymm37860; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7861; AVX2-FCP-NEXT: vpshufb %ymm0, %ymm3, %ymm37862; AVX2-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm47863; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]7864; AVX2-FCP-NEXT: vmovdqa 32(%rdx), %ymm107865; AVX2-FCP-NEXT: vmovdqa 32(%rcx), %ymm117866; AVX2-FCP-NEXT: vpsrldq {{.*#+}} ymm4 = ymm11[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm11[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero7867; AVX2-FCP-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7868; AVX2-FCP-NEXT: vpsrldq {{.*#+}} ymm5 = ymm10[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm10[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero7869; AVX2-FCP-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7870; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm4 = ymm5[0],ymm4[0],ymm5[1],ymm4[1],ymm5[2],ymm4[2],ymm5[3],ymm4[3],ymm5[8],ymm4[8],ymm5[9],ymm4[9],ymm5[10],ymm4[10],ymm5[11],ymm4[11]7871; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,1,2,3]7872; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,2,2,2]7873; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]7874; AVX2-FCP-NEXT: vmovdqa 32(%r8), %ymm47875; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7876; AVX2-FCP-NEXT: vpshuflw {{.*#+}} ymm4 = ymm4[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]7877; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]7878; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0],ymm3[1,2],ymm4[3],ymm3[4,5],ymm4[6],ymm3[7]7879; AVX2-FCP-NEXT: vmovdqa 32(%r9), %ymm47880; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7881; AVX2-FCP-NEXT: vpshufb %ymm2, %ymm4, %ymm47882; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]7883; AVX2-FCP-NEXT: vpblendvb %ymm1, %ymm3, %ymm4, %ymm37884; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7885; AVX2-FCP-NEXT: vmovdqa 64(%rdi), %ymm57886; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7887; AVX2-FCP-NEXT: vmovdqa 64(%rsi), %ymm97888; AVX2-FCP-NEXT: vpshufb %ymm0, %ymm9, %ymm37889; AVX2-FCP-NEXT: vpshufb %ymm0, %ymm5, %ymm47890; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]7891; AVX2-FCP-NEXT: vmovdqa 64(%rdx), %ymm157892; AVX2-FCP-NEXT: vmovdqa 64(%rcx), %ymm47893; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7894; AVX2-FCP-NEXT: vpsrldq {{.*#+}} ymm4 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero7895; AVX2-FCP-NEXT: vpsrldq {{.*#+}} ymm5 = ymm15[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm15[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero7896; AVX2-FCP-NEXT: vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7897; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm4 = ymm5[0],ymm4[0],ymm5[1],ymm4[1],ymm5[2],ymm4[2],ymm5[3],ymm4[3],ymm5[8],ymm4[8],ymm5[9],ymm4[9],ymm5[10],ymm4[10],ymm5[11],ymm4[11]7898; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,1,2,3]7899; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,2,2,2]7900; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]7901; AVX2-FCP-NEXT: vmovdqa 64(%r8), %ymm47902; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7903; AVX2-FCP-NEXT: vpshuflw {{.*#+}} ymm4 = ymm4[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]7904; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]7905; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0],ymm3[1,2],ymm4[3],ymm3[4,5],ymm4[6],ymm3[7]7906; AVX2-FCP-NEXT: vmovdqa 64(%r9), %ymm47907; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7908; AVX2-FCP-NEXT: vpshufb %ymm2, %ymm4, %ymm47909; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]7910; AVX2-FCP-NEXT: vpblendvb %ymm1, %ymm3, %ymm4, %ymm37911; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7912; AVX2-FCP-NEXT: vmovdqa 96(%rdi), %ymm47913; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7914; AVX2-FCP-NEXT: vmovdqa 96(%rsi), %ymm37915; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7916; AVX2-FCP-NEXT: vpshufb %ymm0, %ymm3, %ymm37917; AVX2-FCP-NEXT: vpshufb %ymm0, %ymm4, %ymm07918; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[2],ymm3[2],ymm0[3],ymm3[3],ymm0[8],ymm3[8],ymm0[9],ymm3[9],ymm0[10],ymm3[10],ymm0[11],ymm3[11]7919; AVX2-FCP-NEXT: vmovdqa 96(%rdx), %ymm87920; AVX2-FCP-NEXT: vmovdqa 96(%rcx), %ymm77921; AVX2-FCP-NEXT: vpsrldq {{.*#+}} ymm3 = ymm7[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm7[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero7922; AVX2-FCP-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7923; AVX2-FCP-NEXT: vpsrldq {{.*#+}} ymm4 = ymm8[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm8[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero7924; AVX2-FCP-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7925; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]7926; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,1,2,3]7927; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,2]7928; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm3[2],ymm0[3,4],ymm3[5],ymm0[6,7]7929; AVX2-FCP-NEXT: vmovdqa 96(%r8), %ymm67930; AVX2-FCP-NEXT: vpshuflw {{.*#+}} ymm3 = ymm6[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]7931; AVX2-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7932; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]7933; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0],ymm0[1,2],ymm3[3],ymm0[4,5],ymm3[6],ymm0[7]7934; AVX2-FCP-NEXT: vmovdqa 96(%r9), %ymm37935; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7936; AVX2-FCP-NEXT: vpshufb %ymm2, %ymm3, %ymm27937; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]7938; AVX2-FCP-NEXT: vpblendvb %ymm1, %ymm0, %ymm2, %ymm07939; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7940; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload7941; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload7942; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]7943; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [1,0,2,2,1,0,2,2]7944; AVX2-FCP-NEXT: # ymm1 = mem[0,1,0,1]7945; AVX2-FCP-NEXT: vpermd %ymm0, %ymm1, %ymm07946; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload7947; AVX2-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload7948; AVX2-FCP-NEXT: # xmm2 = xmm2[0],mem[0],xmm2[1],mem[1],xmm2[2],mem[2],xmm2[3],mem[3]7949; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]7950; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7]7951; AVX2-FCP-NEXT: vpmovzxwd (%rsp), %xmm2 # 16-byte Folded Reload7952; AVX2-FCP-NEXT: # xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero7953; AVX2-FCP-NEXT: vpbroadcastq %xmm2, %ymm27954; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm0[0,1],ymm2[2],ymm0[3,4],ymm2[5],ymm0[6,7]7955; AVX2-FCP-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload7956; AVX2-FCP-NEXT: # xmm0 = mem[0,0,2,1,4,5,6,7]7957; AVX2-FCP-NEXT: vpbroadcastq %xmm0, %ymm37958; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm0 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]7959; AVX2-FCP-NEXT: vpblendvb %ymm0, %ymm2, %ymm3, %ymm27960; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7961; AVX2-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm14, %xmm2 # 16-byte Folded Reload7962; AVX2-FCP-NEXT: # xmm2 = xmm14[0],mem[0],xmm14[1],mem[1],xmm14[2],mem[2],xmm14[3],mem[3]7963; AVX2-FCP-NEXT: vpermd %ymm2, %ymm1, %ymm27964; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload7965; AVX2-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm14, %xmm3 # 16-byte Folded Reload7966; AVX2-FCP-NEXT: # xmm3 = xmm14[0],mem[0],xmm14[1],mem[1],xmm14[2],mem[2],xmm14[3],mem[3]7967; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]7968; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6],ymm2[7]7969; AVX2-FCP-NEXT: vpmovzxwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload7970; AVX2-FCP-NEXT: # xmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero7971; AVX2-FCP-NEXT: vpbroadcastq %xmm3, %ymm37972; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]7973; AVX2-FCP-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload7974; AVX2-FCP-NEXT: # xmm3 = mem[0,0,2,1,4,5,6,7]7975; AVX2-FCP-NEXT: vpbroadcastq %xmm3, %ymm37976; AVX2-FCP-NEXT: vpblendvb %ymm0, %ymm2, %ymm3, %ymm27977; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7978; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload7979; AVX2-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload7980; AVX2-FCP-NEXT: # xmm2 = xmm2[0],mem[0],xmm2[1],mem[1],xmm2[2],mem[2],xmm2[3],mem[3]7981; AVX2-FCP-NEXT: vpermd %ymm2, %ymm1, %ymm27982; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload7983; AVX2-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload7984; AVX2-FCP-NEXT: # xmm3 = xmm3[0],mem[0],xmm3[1],mem[1],xmm3[2],mem[2],xmm3[3],mem[3]7985; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]7986; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6],ymm2[7]7987; AVX2-FCP-NEXT: vpmovzxwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload7988; AVX2-FCP-NEXT: # xmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero7989; AVX2-FCP-NEXT: vpbroadcastq %xmm3, %ymm37990; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]7991; AVX2-FCP-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload7992; AVX2-FCP-NEXT: # xmm3 = mem[0,0,2,1,4,5,6,7]7993; AVX2-FCP-NEXT: vpbroadcastq %xmm3, %ymm37994; AVX2-FCP-NEXT: vpblendvb %ymm0, %ymm2, %ymm3, %ymm27995; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7996; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]7997; AVX2-FCP-NEXT: vpermd %ymm2, %ymm1, %ymm17998; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload7999; AVX2-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload8000; AVX2-FCP-NEXT: # xmm2 = xmm2[0],mem[0],xmm2[1],mem[1],xmm2[2],mem[2],xmm2[3],mem[3]8001; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]8002; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6],ymm1[7]8003; AVX2-FCP-NEXT: vpmovzxwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload8004; AVX2-FCP-NEXT: # xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero8005; AVX2-FCP-NEXT: vpbroadcastq %xmm2, %ymm28006; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]8007; AVX2-FCP-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload8008; AVX2-FCP-NEXT: # xmm2 = mem[0,0,2,1,4,5,6,7]8009; AVX2-FCP-NEXT: vpbroadcastq %xmm2, %ymm28010; AVX2-FCP-NEXT: vpblendvb %ymm0, %ymm1, %ymm2, %ymm18011; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8012; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload8013; AVX2-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm2 # 32-byte Folded Reload8014; AVX2-FCP-NEXT: # ymm2 = ymm1[0],mem[0],ymm1[1],mem[1],ymm1[2],mem[2],ymm1[3],mem[3],ymm1[8],mem[8],ymm1[9],mem[9],ymm1[10],mem[10],ymm1[11],mem[11]8015; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [5,4,2,2,5,4,6,6]8016; AVX2-FCP-NEXT: vpermd %ymm2, %ymm1, %ymm28017; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload8018; AVX2-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload8019; AVX2-FCP-NEXT: # ymm3 = ymm3[0],mem[0],ymm3[1],mem[1],ymm3[2],mem[2],ymm3[3],mem[3],ymm3[8],mem[8],ymm3[9],mem[9],ymm3[10],mem[10],ymm3[11],mem[11]8020; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]8021; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6],ymm2[7]8022; AVX2-FCP-NEXT: vpshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload8023; AVX2-FCP-NEXT: # ymm3 = mem[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]8024; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,2]8025; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]8026; AVX2-FCP-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload8027; AVX2-FCP-NEXT: # ymm3 = mem[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]8028; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,2]8029; AVX2-FCP-NEXT: vpblendvb %ymm0, %ymm2, %ymm3, %ymm28030; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8031; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm10[0],ymm11[0],ymm10[1],ymm11[1],ymm10[2],ymm11[2],ymm10[3],ymm11[3],ymm10[8],ymm11[8],ymm10[9],ymm11[9],ymm10[10],ymm11[10],ymm10[11],ymm11[11]8032; AVX2-FCP-NEXT: vpermd %ymm2, %ymm1, %ymm28033; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload8034; AVX2-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload8035; AVX2-FCP-NEXT: # ymm3 = ymm3[0],mem[0],ymm3[1],mem[1],ymm3[2],mem[2],ymm3[3],mem[3],ymm3[8],mem[8],ymm3[9],mem[9],ymm3[10],mem[10],ymm3[11],mem[11]8036; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]8037; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6],ymm2[7]8038; AVX2-FCP-NEXT: vpshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload8039; AVX2-FCP-NEXT: # ymm3 = mem[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]8040; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,2]8041; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]8042; AVX2-FCP-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload8043; AVX2-FCP-NEXT: # ymm3 = mem[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]8044; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,2]8045; AVX2-FCP-NEXT: vpblendvb %ymm0, %ymm2, %ymm3, %ymm28046; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8047; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload8048; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm15[0],ymm13[0],ymm15[1],ymm13[1],ymm15[2],ymm13[2],ymm15[3],ymm13[3],ymm15[8],ymm13[8],ymm15[9],ymm13[9],ymm15[10],ymm13[10],ymm15[11],ymm13[11]8049; AVX2-FCP-NEXT: vpermd %ymm2, %ymm1, %ymm28050; AVX2-FCP-NEXT: vmovdqa %ymm9, %ymm128051; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload8052; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm3 = ymm11[0],ymm9[0],ymm11[1],ymm9[1],ymm11[2],ymm9[2],ymm11[3],ymm9[3],ymm11[8],ymm9[8],ymm11[9],ymm9[9],ymm11[10],ymm9[10],ymm11[11],ymm9[11]8053; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]8054; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6],ymm2[7]8055; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload8056; AVX2-FCP-NEXT: vpshuflw {{.*#+}} ymm3 = ymm15[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]8057; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,2]8058; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7]8059; AVX2-FCP-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload8060; AVX2-FCP-NEXT: # ymm3 = mem[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]8061; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,2]8062; AVX2-FCP-NEXT: vpblendvb %ymm0, %ymm2, %ymm3, %ymm28063; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8064; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm8[0],ymm7[0],ymm8[1],ymm7[1],ymm8[2],ymm7[2],ymm8[3],ymm7[3],ymm8[8],ymm7[8],ymm8[9],ymm7[9],ymm8[10],ymm7[10],ymm8[11],ymm7[11]8065; AVX2-FCP-NEXT: vpermd %ymm2, %ymm1, %ymm18066; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload8067; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload8068; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11]8069; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]8070; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6],ymm1[7]8071; AVX2-FCP-NEXT: vpshuflw {{.*#+}} ymm2 = ymm6[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]8072; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]8073; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]8074; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload8075; AVX2-FCP-NEXT: vpshuflw {{.*#+}} ymm2 = ymm9[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]8076; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]8077; AVX2-FCP-NEXT: vpblendvb %ymm0, %ymm1, %ymm2, %ymm08078; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8079; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]8080; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload8081; AVX2-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload8082; AVX2-FCP-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]8083; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [1,2,1,2,0,0,3,3]8084; AVX2-FCP-NEXT: vpermd %ymm0, %ymm3, %ymm08085; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[1,1,1,1]8086; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6,7]8087; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm2 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]8088; AVX2-FCP-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload8089; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm1, %xmm18090; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]8091; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7]8092; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm1 = [8,9,12,13,12,13,10,11,8,9,10,11,12,13,14,15]8093; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload8094; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm0, %xmm08095; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm5 = ymm0[0,1,0,1]8096; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm6 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]8097; AVX2-FCP-NEXT: vpblendvb %ymm6, %ymm4, %ymm5, %ymm08098; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8099; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload8100; AVX2-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm4 # 16-byte Folded Reload8101; AVX2-FCP-NEXT: # xmm4 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]8102; AVX2-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm14, %xmm5 # 16-byte Folded Reload8103; AVX2-FCP-NEXT: # xmm5 = xmm14[4],mem[4],xmm14[5],mem[5],xmm14[6],mem[6],xmm14[7],mem[7]8104; AVX2-FCP-NEXT: vpermd %ymm4, %ymm3, %ymm48105; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm5 = ymm5[1,1,1,1]8106; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]8107; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload8108; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm0, %xmm58109; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm5 = ymm5[0,1,0,1]8110; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0],ymm5[1],ymm4[2,3],ymm5[4],ymm4[5,6],ymm5[7]8111; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload8112; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm0, %xmm58113; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm5 = ymm5[0,1,0,1]8114; AVX2-FCP-NEXT: vpblendvb %ymm6, %ymm4, %ymm5, %ymm08115; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8116; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload8117; AVX2-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm4 # 16-byte Folded Reload8118; AVX2-FCP-NEXT: # xmm4 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]8119; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload8120; AVX2-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm5 # 16-byte Folded Reload8121; AVX2-FCP-NEXT: # xmm5 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]8122; AVX2-FCP-NEXT: vpermd %ymm4, %ymm3, %ymm48123; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm5 = ymm5[1,1,1,1]8124; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7]8125; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload8126; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm0, %xmm58127; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm5 = ymm5[0,1,0,1]8128; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0],ymm5[1],ymm4[2,3],ymm5[4],ymm4[5,6],ymm5[7]8129; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload8130; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm0, %xmm58131; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm5 = ymm5[0,1,0,1]8132; AVX2-FCP-NEXT: vpblendvb %ymm6, %ymm4, %ymm5, %ymm08133; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8134; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload8135; AVX2-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm4 # 16-byte Folded Reload8136; AVX2-FCP-NEXT: # xmm4 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]8137; AVX2-FCP-NEXT: vpermd %ymm4, %ymm3, %ymm38138; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload8139; AVX2-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm4 # 16-byte Folded Reload8140; AVX2-FCP-NEXT: # xmm4 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]8141; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[1,1,1,1]8142; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7]8143; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload8144; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm0, %xmm28145; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1]8146; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6],ymm2[7]8147; AVX2-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload8148; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm0, %xmm18149; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]8150; AVX2-FCP-NEXT: vpblendvb %ymm6, %ymm2, %ymm1, %ymm108151; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8152; AVX2-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload8153; AVX2-FCP-NEXT: # ymm1 = ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15]8154; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8155; AVX2-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload8156; AVX2-FCP-NEXT: # ymm2 = ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15]8157; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [5,6,5,6,5,6,7,7]8158; AVX2-FCP-NEXT: vpermd %ymm1, %ymm3, %ymm18159; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[3,3,3,3]8160; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7]8161; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm2 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]8162; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8163; AVX2-FCP-NEXT: vpshufb %ymm2, %ymm0, %ymm48164; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,1,2,3]8165; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm1[0],ymm4[1],ymm1[2,3],ymm4[4],ymm1[5,6],ymm4[7]8166; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,24,25,28,29,28,29,26,27,24,25,26,27,28,29,30,31]8167; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8168; AVX2-FCP-NEXT: vpshufb %ymm1, %ymm0, %ymm148169; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm14 = ymm14[2,1,2,3]8170; AVX2-FCP-NEXT: vpblendvb %ymm6, %ymm4, %ymm14, %ymm58171; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8172; AVX2-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload8173; AVX2-FCP-NEXT: # ymm14 = ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15]8174; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8175; AVX2-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload8176; AVX2-FCP-NEXT: # ymm0 = ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15]8177; AVX2-FCP-NEXT: vpermd %ymm14, %ymm3, %ymm148178; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,3,3,3]8179; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm14[0,1],ymm0[2],ymm14[3,4],ymm0[5],ymm14[6,7]8180; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload8181; AVX2-FCP-NEXT: vpshufb %ymm2, %ymm4, %ymm148182; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm14 = ymm14[2,1,2,3]8183; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm14[1],ymm0[2,3],ymm14[4],ymm0[5,6],ymm14[7]8184; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload8185; AVX2-FCP-NEXT: vpshufb %ymm1, %ymm4, %ymm148186; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm14 = ymm14[2,1,2,3]8187; AVX2-FCP-NEXT: vpblendvb %ymm6, %ymm0, %ymm14, %ymm48188; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8189; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} ymm14 = ymm0[4],ymm13[4],ymm0[5],ymm13[5],ymm0[6],ymm13[6],ymm0[7],ymm13[7],ymm0[12],ymm13[12],ymm0[13],ymm13[13],ymm0[14],ymm13[14],ymm0[15],ymm13[15]8190; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm11[4],ymm12[4],ymm11[5],ymm12[5],ymm11[6],ymm12[6],ymm11[7],ymm12[7],ymm11[12],ymm12[12],ymm11[13],ymm12[13],ymm11[14],ymm12[14],ymm11[15],ymm12[15]8191; AVX2-FCP-NEXT: vpermd %ymm14, %ymm3, %ymm148192; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,3,3,3]8193; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm14[0,1],ymm0[2],ymm14[3,4],ymm0[5],ymm14[6,7]8194; AVX2-FCP-NEXT: vpshufb %ymm2, %ymm15, %ymm148195; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm14 = ymm14[2,1,2,3]8196; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm14[1],ymm0[2,3],ymm14[4],ymm0[5,6],ymm14[7]8197; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload8198; AVX2-FCP-NEXT: vpshufb %ymm1, %ymm11, %ymm148199; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm14 = ymm14[2,1,2,3]8200; AVX2-FCP-NEXT: vpblendvb %ymm6, %ymm0, %ymm14, %ymm08201; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload8202; AVX2-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm14 # 32-byte Folded Reload8203; AVX2-FCP-NEXT: # ymm14 = ymm11[4],mem[4],ymm11[5],mem[5],ymm11[6],mem[6],ymm11[7],mem[7],ymm11[12],mem[12],ymm11[13],mem[13],ymm11[14],mem[14],ymm11[15],mem[15]8204; AVX2-FCP-NEXT: vpermd %ymm14, %ymm3, %ymm38205; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} ymm14 = ymm7[4],ymm8[4],ymm7[5],ymm8[5],ymm7[6],ymm8[6],ymm7[7],ymm8[7],ymm7[12],ymm8[12],ymm7[13],ymm8[13],ymm7[14],ymm8[14],ymm7[15],ymm8[15]8206; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm14 = ymm14[3,3,3,3]8207; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm14[2],ymm3[3,4],ymm14[5],ymm3[6,7]8208; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload8209; AVX2-FCP-NEXT: vpshufb %ymm2, %ymm7, %ymm28210; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,1,2,3]8211; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6],ymm2[7]8212; AVX2-FCP-NEXT: vpshufb %ymm1, %ymm9, %ymm18213; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]8214; AVX2-FCP-NEXT: vpblendvb %ymm6, %ymm2, %ymm1, %ymm18215; AVX2-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax8216; AVX2-FCP-NEXT: vmovdqa %ymm1, 736(%rax)8217; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload8218; AVX2-FCP-NEXT: vmovaps %ymm1, 672(%rax)8219; AVX2-FCP-NEXT: vmovdqa %ymm0, 544(%rax)8220; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8221; AVX2-FCP-NEXT: vmovaps %ymm0, 480(%rax)8222; AVX2-FCP-NEXT: vmovdqa %ymm4, 352(%rax)8223; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8224; AVX2-FCP-NEXT: vmovaps %ymm0, 288(%rax)8225; AVX2-FCP-NEXT: vmovdqa %ymm5, 160(%rax)8226; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8227; AVX2-FCP-NEXT: vmovaps %ymm0, 96(%rax)8228; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8229; AVX2-FCP-NEXT: vmovaps %ymm0, 704(%rax)8230; AVX2-FCP-NEXT: vmovdqa %ymm10, 640(%rax)8231; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8232; AVX2-FCP-NEXT: vmovaps %ymm0, 576(%rax)8233; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8234; AVX2-FCP-NEXT: vmovaps %ymm0, 512(%rax)8235; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8236; AVX2-FCP-NEXT: vmovaps %ymm0, 448(%rax)8237; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8238; AVX2-FCP-NEXT: vmovaps %ymm0, 384(%rax)8239; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8240; AVX2-FCP-NEXT: vmovaps %ymm0, 320(%rax)8241; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8242; AVX2-FCP-NEXT: vmovaps %ymm0, 256(%rax)8243; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8244; AVX2-FCP-NEXT: vmovaps %ymm0, 192(%rax)8245; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8246; AVX2-FCP-NEXT: vmovaps %ymm0, 128(%rax)8247; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8248; AVX2-FCP-NEXT: vmovaps %ymm0, 64(%rax)8249; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8250; AVX2-FCP-NEXT: vmovaps %ymm0, (%rax)8251; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8252; AVX2-FCP-NEXT: vmovaps %ymm0, 608(%rax)8253; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8254; AVX2-FCP-NEXT: vmovaps %ymm0, 416(%rax)8255; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8256; AVX2-FCP-NEXT: vmovaps %ymm0, 224(%rax)8257; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload8258; AVX2-FCP-NEXT: vmovaps %ymm0, 32(%rax)8259; AVX2-FCP-NEXT: addq $1560, %rsp # imm = 0x6188260; AVX2-FCP-NEXT: vzeroupper8261; AVX2-FCP-NEXT: retq8262;8263; AVX512-LABEL: store_i16_stride6_vf64:8264; AVX512: # %bb.0:8265; AVX512-NEXT: subq $744, %rsp # imm = 0x2E88266; AVX512-NEXT: vmovdqa 96(%rcx), %ymm98267; AVX512-NEXT: vpsrldq {{.*#+}} ymm1 = ymm9[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm9[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero8268; AVX512-NEXT: vmovdqa 96(%rdx), %ymm08269; AVX512-NEXT: vpsrldq {{.*#+}} ymm2 = ymm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm0[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero8270; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]8271; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8272; AVX512-NEXT: vmovdqa 96(%rsi), %ymm38273; AVX512-NEXT: vpshufd {{.*#+}} ymm1 = ymm3[2,1,2,3,6,5,6,7]8274; AVX512-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]8275; AVX512-NEXT: vmovdqa 96(%rdi), %ymm28276; AVX512-NEXT: vpshufd {{.*#+}} ymm4 = ymm2[2,1,2,3,6,5,6,7]8277; AVX512-NEXT: vpshuflw {{.*#+}} ymm4 = ymm4[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]8278; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm4[0],ymm1[0],ymm4[1],ymm1[1],ymm4[2],ymm1[2],ymm4[3],ymm1[3],ymm4[8],ymm1[8],ymm4[9],ymm1[9],ymm4[10],ymm1[10],ymm4[11],ymm1[11]8279; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8280; AVX512-NEXT: vmovdqa 64(%rcx), %ymm48281; AVX512-NEXT: vpsrldq {{.*#+}} ymm1 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero8282; AVX512-NEXT: vmovdqa64 %ymm4, %ymm168283; AVX512-NEXT: vmovdqa 64(%rdx), %ymm58284; AVX512-NEXT: vpsrldq {{.*#+}} ymm4 = ymm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm5[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero8285; AVX512-NEXT: vmovdqa64 %ymm5, %ymm278286; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm4[0],ymm1[0],ymm4[1],ymm1[1],ymm4[2],ymm1[2],ymm4[3],ymm1[3],ymm4[8],ymm1[8],ymm4[9],ymm1[9],ymm4[10],ymm1[10],ymm4[11],ymm1[11]8287; AVX512-NEXT: vmovdqa64 %ymm1, %ymm308288; AVX512-NEXT: vmovdqa 64(%rsi), %ymm158289; AVX512-NEXT: vpshufd {{.*#+}} ymm1 = ymm15[2,1,2,3,6,5,6,7]8290; AVX512-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]8291; AVX512-NEXT: vmovdqa 64(%rdi), %ymm88292; AVX512-NEXT: vpshufd {{.*#+}} ymm5 = ymm8[2,1,2,3,6,5,6,7]8293; AVX512-NEXT: vpshuflw {{.*#+}} ymm5 = ymm5[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]8294; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm5[0],ymm1[0],ymm5[1],ymm1[1],ymm5[2],ymm1[2],ymm5[3],ymm1[3],ymm5[8],ymm1[8],ymm5[9],ymm1[9],ymm5[10],ymm1[10],ymm5[11],ymm1[11]8295; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8296; AVX512-NEXT: vmovdqa 32(%rcx), %ymm58297; AVX512-NEXT: vpsrldq {{.*#+}} ymm1 = ymm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm5[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero8298; AVX512-NEXT: vmovdqa64 %ymm5, %ymm248299; AVX512-NEXT: vmovdqa 32(%rdx), %ymm118300; AVX512-NEXT: vpsrldq {{.*#+}} ymm5 = ymm11[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm11[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero8301; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm5[0],ymm1[0],ymm5[1],ymm1[1],ymm5[2],ymm1[2],ymm5[3],ymm1[3],ymm5[8],ymm1[8],ymm5[9],ymm1[9],ymm5[10],ymm1[10],ymm5[11],ymm1[11]8302; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8303; AVX512-NEXT: vmovdqa64 32(%rsi), %ymm298304; AVX512-NEXT: vpshufd {{.*#+}} ymm1 = ymm29[2,1,2,3,6,5,6,7]8305; AVX512-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]8306; AVX512-NEXT: vmovdqa64 32(%rdi), %ymm258307; AVX512-NEXT: vpshufd {{.*#+}} ymm5 = ymm25[2,1,2,3,6,5,6,7]8308; AVX512-NEXT: vpshuflw {{.*#+}} ymm5 = ymm5[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]8309; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm5[0],ymm1[0],ymm5[1],ymm1[1],ymm5[2],ymm1[2],ymm5[3],ymm1[3],ymm5[8],ymm1[8],ymm5[9],ymm1[9],ymm5[10],ymm1[10],ymm5[11],ymm1[11]8310; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8311; AVX512-NEXT: vmovdqa (%rcx), %ymm58312; AVX512-NEXT: vpsrldq {{.*#+}} ymm1 = ymm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm5[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero8313; AVX512-NEXT: vmovdqa (%rdx), %ymm128314; AVX512-NEXT: vpsrldq {{.*#+}} ymm6 = ymm12[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm12[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero8315; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm6[0],ymm1[0],ymm6[1],ymm1[1],ymm6[2],ymm1[2],ymm6[3],ymm1[3],ymm6[8],ymm1[8],ymm6[9],ymm1[9],ymm6[10],ymm1[10],ymm6[11],ymm1[11]8316; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8317; AVX512-NEXT: vmovdqa (%rsi), %ymm138318; AVX512-NEXT: vpshufd {{.*#+}} ymm1 = ymm13[2,1,2,3,6,5,6,7]8319; AVX512-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]8320; AVX512-NEXT: vmovdqa (%rdi), %ymm148321; AVX512-NEXT: vpshufd {{.*#+}} ymm6 = ymm14[2,1,2,3,6,5,6,7]8322; AVX512-NEXT: vpshuflw {{.*#+}} ymm6 = ymm6[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]8323; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm6[0],ymm1[0],ymm6[1],ymm1[1],ymm6[2],ymm1[2],ymm6[3],ymm1[3],ymm6[8],ymm1[8],ymm6[9],ymm1[9],ymm6[10],ymm1[10],ymm6[11],ymm1[11]8324; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8325; AVX512-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm12[4],ymm5[4],ymm12[5],ymm5[5],ymm12[6],ymm5[6],ymm12[7],ymm5[7],ymm12[12],ymm5[12],ymm12[13],ymm5[13],ymm12[14],ymm5[14],ymm12[15],ymm5[15]8326; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8327; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm12 = ymm12[0],ymm5[0],ymm12[1],ymm5[1],ymm12[2],ymm5[2],ymm12[3],ymm5[3],ymm12[8],ymm5[8],ymm12[9],ymm5[9],ymm12[10],ymm5[10],ymm12[11],ymm5[11]8328; AVX512-NEXT: vpmovsxbd {{.*#+}} zmm20 = [17,18,17,18,0,0,19,19,5,4,2,2,5,4,6,6]8329; AVX512-NEXT: vmovdqa (%rcx), %xmm58330; AVX512-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill8331; AVX512-NEXT: vmovdqa (%rdx), %xmm18332; AVX512-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill8333; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7]8334; AVX512-NEXT: vpermt2d %zmm1, %zmm20, %zmm128335; AVX512-NEXT: vmovdqa (%rsi), %xmm58336; AVX512-NEXT: vmovdqa %xmm5, (%rsp) # 16-byte Spill8337; AVX512-NEXT: vmovdqa (%rdi), %xmm18338; AVX512-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill8339; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7]8340; AVX512-NEXT: vpermq {{.*#+}} ymm1 = ymm1[1,1,1,1]8341; AVX512-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm14[4],ymm13[4],ymm14[5],ymm13[5],ymm14[6],ymm13[6],ymm14[7],ymm13[7],ymm14[12],ymm13[12],ymm14[13],ymm13[13],ymm14[14],ymm13[14],ymm14[15],ymm13[15]8342; AVX512-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8343; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm5 = ymm14[0],ymm13[0],ymm14[1],ymm13[1],ymm14[2],ymm13[2],ymm14[3],ymm13[3],ymm14[8],ymm13[8],ymm14[9],ymm13[9],ymm14[10],ymm13[10],ymm14[11],ymm13[11]8344; AVX512-NEXT: vpermq {{.*#+}} ymm5 = ymm5[2,2,2,3]8345; AVX512-NEXT: vmovdqa (%r8), %xmm68346; AVX512-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill8347; AVX512-NEXT: vmovdqa {{.*#+}} xmm4 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]8348; AVX512-NEXT: vpshufb %xmm4, %xmm6, %xmm68349; AVX512-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,1,0,1]8350; AVX512-NEXT: vmovdqa (%r8), %ymm78351; AVX512-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8352; AVX512-NEXT: vpshuflw {{.*#+}} ymm7 = ymm7[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]8353; AVX512-NEXT: vpermq {{.*#+}} ymm7 = ymm7[2,2,2,2]8354; AVX512-NEXT: vmovdqa (%r9), %xmm108355; AVX512-NEXT: vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill8356; AVX512-NEXT: vpshufd {{.*#+}} xmm13 = xmm10[2,3,2,3]8357; AVX512-NEXT: vpshuflw {{.*#+}} xmm13 = xmm13[0,2,2,1,4,5,6,7]8358; AVX512-NEXT: vpermq {{.*#+}} ymm13 = ymm13[0,1,0,1]8359; AVX512-NEXT: vmovdqa (%r9), %ymm108360; AVX512-NEXT: vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8361; AVX512-NEXT: vpshuflw {{.*#+}} ymm14 = ymm10[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]8362; AVX512-NEXT: vpermq {{.*#+}} ymm14 = ymm14[2,2,2,2]8363; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm1, %zmm18364; AVX512-NEXT: movw $18724, %ax # imm = 0x49248365; AVX512-NEXT: kmovw %eax, %k18366; AVX512-NEXT: vmovdqa32 %zmm1, %zmm12 {%k1}8367; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm6, %zmm18368; AVX512-NEXT: movw $9362, %ax # imm = 0x24928369; AVX512-NEXT: kmovw %eax, %k28370; AVX512-NEXT: vmovdqa32 %zmm1, %zmm12 {%k2}8371; AVX512-NEXT: vinserti64x4 $1, %ymm14, %zmm13, %zmm18372; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm21 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]8373; AVX512-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm21 & (zmm1 ^ zmm12))8374; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8375; AVX512-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm0[4],ymm9[4],ymm0[5],ymm9[5],ymm0[6],ymm9[6],ymm0[7],ymm9[7],ymm0[12],ymm9[12],ymm0[13],ymm9[13],ymm0[14],ymm9[14],ymm0[15],ymm9[15]8376; AVX512-NEXT: vmovdqa64 %ymm1, %ymm318377; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm9[0],ymm0[1],ymm9[1],ymm0[2],ymm9[2],ymm0[3],ymm9[3],ymm0[8],ymm9[8],ymm0[9],ymm9[9],ymm0[10],ymm9[10],ymm0[11],ymm9[11]8378; AVX512-NEXT: vmovdqa 96(%rcx), %xmm58379; AVX512-NEXT: vmovdqa 96(%rdx), %xmm128380; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm12[4],xmm5[4],xmm12[5],xmm5[5],xmm12[6],xmm5[6],xmm12[7],xmm5[7]8381; AVX512-NEXT: vpermt2d %zmm1, %zmm20, %zmm08382; AVX512-NEXT: vmovdqa 96(%rsi), %xmm18383; AVX512-NEXT: vmovdqa 96(%rdi), %xmm98384; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm6 = xmm9[4],xmm1[4],xmm9[5],xmm1[5],xmm9[6],xmm1[6],xmm9[7],xmm1[7]8385; AVX512-NEXT: vpermq {{.*#+}} ymm6 = ymm6[1,1,1,1]8386; AVX512-NEXT: vpunpckhwd {{.*#+}} ymm7 = ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15]8387; AVX512-NEXT: vmovdqa64 %ymm7, %ymm188388; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11]8389; AVX512-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]8390; AVX512-NEXT: vmovdqa 96(%r8), %xmm38391; AVX512-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill8392; AVX512-NEXT: vpshufb %xmm4, %xmm3, %xmm38393; AVX512-NEXT: vmovdqa64 %xmm4, %xmm268394; AVX512-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,1,0,1]8395; AVX512-NEXT: vmovdqa 96(%r8), %ymm48396; AVX512-NEXT: vpshuflw {{.*#+}} ymm7 = ymm4[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]8397; AVX512-NEXT: vmovdqa64 %ymm4, %ymm288398; AVX512-NEXT: vpermq {{.*#+}} ymm7 = ymm7[2,2,2,2]8399; AVX512-NEXT: vmovdqa 96(%r9), %xmm108400; AVX512-NEXT: vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill8401; AVX512-NEXT: vpshufd {{.*#+}} xmm13 = xmm10[2,3,2,3]8402; AVX512-NEXT: vpshuflw {{.*#+}} xmm13 = xmm13[0,2,2,1,4,5,6,7]8403; AVX512-NEXT: vpermq {{.*#+}} ymm17 = ymm13[0,1,0,1]8404; AVX512-NEXT: vmovdqa 96(%r9), %ymm148405; AVX512-NEXT: vpshuflw {{.*#+}} ymm10 = ymm14[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]8406; AVX512-NEXT: vpermq {{.*#+}} ymm10 = ymm10[2,2,2,2]8407; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm6, %zmm28408; AVX512-NEXT: vmovdqa32 %zmm2, %zmm0 {%k1}8409; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm3, %zmm28410; AVX512-NEXT: vmovdqa32 %zmm2, %zmm0 {%k2}8411; AVX512-NEXT: vinserti64x4 $1, %ymm10, %zmm17, %zmm28412; AVX512-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm21 & (zmm2 ^ zmm0))8413; AVX512-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8414; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm12[0],xmm5[0],xmm12[1],xmm5[1],xmm12[2],xmm5[2],xmm12[3],xmm5[3]8415; AVX512-NEXT: vpsrldq {{.*#+}} xmm2 = xmm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero8416; AVX512-NEXT: vpsrldq {{.*#+}} xmm3 = xmm12[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero8417; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]8418; AVX512-NEXT: vpmovsxbd {{.*#+}} zmm17 = [1,0,2,2,1,0,2,2,16,17,16,17,16,17,16,17]8419; AVX512-NEXT: vpermt2d %zmm2, %zmm17, %zmm08420; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[0,1,2,1]8421; AVX512-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,5]8422; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm9[0,1,2,1]8423; AVX512-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,5]8424; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]8425; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm9[0],xmm1[0],xmm9[1],xmm1[1],xmm9[2],xmm1[2],xmm9[3],xmm1[3]8426; AVX512-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]8427; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm1[0,1,2,3],zmm2[0,1,0,1]8428; AVX512-NEXT: vmovdqa32 %zmm0, %zmm1 {%k2}8429; AVX512-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8430; AVX512-NEXT: vmovdqa 64(%rcx), %xmm98431; AVX512-NEXT: vmovdqa 64(%rdx), %xmm128432; AVX512-NEXT: vpsrldq {{.*#+}} xmm0 = xmm9[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero8433; AVX512-NEXT: vpsrldq {{.*#+}} xmm1 = xmm12[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero8434; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]8435; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm12[0],xmm9[0],xmm12[1],xmm9[1],xmm12[2],xmm9[2],xmm12[3],xmm9[3]8436; AVX512-NEXT: vpermt2d %zmm0, %zmm17, %zmm68437; AVX512-NEXT: vmovdqa 64(%rsi), %xmm08438; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,1,2,1]8439; AVX512-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,5]8440; AVX512-NEXT: vmovdqa 64(%rdi), %xmm28441; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,1,2,1]8442; AVX512-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,5]8443; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]8444; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]8445; AVX512-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]8446; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm19 = zmm3[0,1,2,3],zmm1[0,1,0,1]8447; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm12[4],xmm9[4],xmm12[5],xmm9[5],xmm12[6],xmm9[6],xmm12[7],xmm9[7]8448; AVX512-NEXT: vmovdqa64 %ymm16, %ymm18449; AVX512-NEXT: vmovdqa64 %ymm27, %ymm48450; AVX512-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm4[4],ymm1[4],ymm4[5],ymm1[5],ymm4[6],ymm1[6],ymm4[7],ymm1[7],ymm4[12],ymm1[12],ymm4[13],ymm1[13],ymm4[14],ymm1[14],ymm4[15],ymm1[15]8451; AVX512-NEXT: vmovdqa64 %ymm5, %ymm168452; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm4[0],ymm1[0],ymm4[1],ymm1[1],ymm4[2],ymm1[2],ymm4[3],ymm1[3],ymm4[8],ymm1[8],ymm4[9],ymm1[9],ymm4[10],ymm1[10],ymm4[11],ymm1[11]8453; AVX512-NEXT: vpermt2d %zmm3, %zmm20, %zmm18454; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]8455; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,1,1,1]8456; AVX512-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm8[4],ymm15[4],ymm8[5],ymm15[5],ymm8[6],ymm15[6],ymm8[7],ymm15[7],ymm8[12],ymm15[12],ymm8[13],ymm15[13],ymm8[14],ymm15[14],ymm8[15],ymm15[15]8457; AVX512-NEXT: vmovdqa64 %ymm2, %ymm228458; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm8[0],ymm15[0],ymm8[1],ymm15[1],ymm8[2],ymm15[2],ymm8[3],ymm15[3],ymm8[8],ymm15[8],ymm8[9],ymm15[9],ymm8[10],ymm15[10],ymm8[11],ymm15[11]8459; AVX512-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]8460; AVX512-NEXT: vmovdqa 64(%r8), %xmm38461; AVX512-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill8462; AVX512-NEXT: vmovdqa64 %xmm26, %xmm98463; AVX512-NEXT: vpshufb %xmm9, %xmm3, %xmm38464; AVX512-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,1,0,1]8465; AVX512-NEXT: vmovdqa 64(%r8), %ymm108466; AVX512-NEXT: vpshuflw {{.*#+}} ymm4 = ymm10[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]8467; AVX512-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,2,2,2]8468; AVX512-NEXT: vmovdqa 64(%r9), %xmm58469; AVX512-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill8470; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]8471; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,2,2,1,4,5,6,7]8472; AVX512-NEXT: vpermq {{.*#+}} ymm5 = ymm5[0,1,0,1]8473; AVX512-NEXT: vmovdqa 64(%r9), %ymm128474; AVX512-NEXT: vpshuflw {{.*#+}} ymm8 = ymm12[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]8475; AVX512-NEXT: vpermq {{.*#+}} ymm8 = ymm8[2,2,2,2]8476; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm08477; AVX512-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}8478; AVX512-NEXT: vinserti64x4 $1, %ymm4, %zmm3, %zmm08479; AVX512-NEXT: vmovdqa32 %zmm0, %zmm1 {%k2}8480; AVX512-NEXT: vinserti64x4 $1, %ymm8, %zmm5, %zmm08481; AVX512-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm21 & (zmm0 ^ zmm1))8482; AVX512-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8483; AVX512-NEXT: vmovdqa64 %ymm24, %ymm08484; AVX512-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm11[4],ymm0[4],ymm11[5],ymm0[5],ymm11[6],ymm0[6],ymm11[7],ymm0[7],ymm11[12],ymm0[12],ymm11[13],ymm0[13],ymm11[14],ymm0[14],ymm11[15],ymm0[15]8485; AVX512-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8486; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm11[0],ymm0[0],ymm11[1],ymm0[1],ymm11[2],ymm0[2],ymm11[3],ymm0[3],ymm11[8],ymm0[8],ymm11[9],ymm0[9],ymm11[10],ymm0[10],ymm11[11],ymm0[11]8487; AVX512-NEXT: vmovdqa 32(%rcx), %xmm28488; AVX512-NEXT: vmovdqa 32(%rdx), %xmm18489; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]8490; AVX512-NEXT: vpermt2d %zmm3, %zmm20, %zmm08491; AVX512-NEXT: vmovdqa 32(%rsi), %xmm48492; AVX512-NEXT: vmovdqa 32(%rdi), %xmm38493; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]8494; AVX512-NEXT: vpermq {{.*#+}} ymm20 = ymm5[1,1,1,1]8495; AVX512-NEXT: vmovdqa64 %ymm29, %ymm58496; AVX512-NEXT: vmovdqa64 %ymm25, %ymm78497; AVX512-NEXT: vpunpckhwd {{.*#+}} ymm8 = ymm7[4],ymm5[4],ymm7[5],ymm5[5],ymm7[6],ymm5[6],ymm7[7],ymm5[7],ymm7[12],ymm5[12],ymm7[13],ymm5[13],ymm7[14],ymm5[14],ymm7[15],ymm5[15]8498; AVX512-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8499; AVX512-NEXT: vpunpcklwd {{.*#+}} ymm5 = ymm7[0],ymm5[0],ymm7[1],ymm5[1],ymm7[2],ymm5[2],ymm7[3],ymm5[3],ymm7[8],ymm5[8],ymm7[9],ymm5[9],ymm7[10],ymm5[10],ymm7[11],ymm5[11]8500; AVX512-NEXT: vpermq {{.*#+}} ymm24 = ymm5[2,2,2,3]8501; AVX512-NEXT: vmovdqa 32(%r8), %xmm58502; AVX512-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill8503; AVX512-NEXT: vpshufb %xmm9, %xmm5, %xmm58504; AVX512-NEXT: vpermq {{.*#+}} ymm25 = ymm5[0,1,0,1]8505; AVX512-NEXT: vmovdqa 32(%r8), %ymm98506; AVX512-NEXT: vpshuflw {{.*#+}} ymm8 = ymm9[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]8507; AVX512-NEXT: vpermq {{.*#+}} ymm26 = ymm8[2,2,2,2]8508; AVX512-NEXT: vmovdqa 32(%r9), %xmm58509; AVX512-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill8510; AVX512-NEXT: vpshufd {{.*#+}} xmm11 = xmm5[2,3,2,3]8511; AVX512-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[0,2,2,1,4,5,6,7]8512; AVX512-NEXT: vpermq {{.*#+}} ymm27 = ymm11[0,1,0,1]8513; AVX512-NEXT: vmovdqa 32(%r9), %ymm138514; AVX512-NEXT: vpshuflw {{.*#+}} ymm15 = ymm13[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]8515; AVX512-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,2,2,2]8516; AVX512-NEXT: vinserti64x4 $1, %ymm24, %zmm20, %zmm208517; AVX512-NEXT: vmovdqa32 %zmm20, %zmm0 {%k1}8518; AVX512-NEXT: vinserti64x4 $1, %ymm26, %zmm25, %zmm208519; AVX512-NEXT: vmovdqa32 %zmm20, %zmm0 {%k2}8520; AVX512-NEXT: vinserti64x4 $1, %ymm15, %zmm27, %zmm208521; AVX512-NEXT: vpternlogq {{.*#+}} zmm20 = zmm20 ^ (zmm21 & (zmm20 ^ zmm0))8522; AVX512-NEXT: vpermq $170, {{[-0-9]+}}(%r{{[sb]}}p), %ymm21 # 32-byte Folded Reload8523; AVX512-NEXT: # ymm21 = mem[2,2,2,2]8524; AVX512-NEXT: vpshufd {{.*#+}} ymm25 = ymm31[1,2,3,3,5,6,7,7]8525; AVX512-NEXT: vpermq $230, {{[-0-9]+}}(%r{{[sb]}}p), %ymm24 # 32-byte Folded Reload8526; AVX512-NEXT: # ymm24 = mem[2,1,2,3]8527; AVX512-NEXT: vpermq {{.*#+}} ymm26 = ymm18[3,3,3,3]8528; AVX512-NEXT: vmovdqa32 %zmm6, %zmm19 {%k2}8529; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]8530; AVX512-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero8531; AVX512-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero8532; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]8533; AVX512-NEXT: vmovdqa64 %zmm17, %zmm58534; AVX512-NEXT: vpermt2d %zmm1, %zmm17, %zmm08535; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm4[0,1,2,1]8536; AVX512-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,5]8537; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[0,1,2,1]8538; AVX512-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,5]8539; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]8540; AVX512-NEXT: vmovdqa64 %ymm28, %ymm68541; AVX512-NEXT: vpshuflw {{.*#+}} ymm2 = ymm6[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]8542; AVX512-NEXT: vpermq {{.*#+}} ymm27 = ymm2[2,2,2,3]8543; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]8544; AVX512-NEXT: vmovdqa {{.*#+}} ymm8 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]8545; AVX512-NEXT: vpshufb %ymm8, %ymm6, %ymm48546; AVX512-NEXT: vpermq {{.*#+}} ymm28 = ymm4[2,1,2,3]8547; AVX512-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]8548; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm18 = zmm2[0,1,2,3],zmm1[0,1,0,1]8549; AVX512-NEXT: vpshuflw {{.*#+}} ymm1 = ymm14[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]8550; AVX512-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]8551; AVX512-NEXT: vpermq {{.*#+}} ymm29 = ymm1[2,2,2,3]8552; AVX512-NEXT: vpshufd {{.*#+}} ymm1 = ymm14[2,3,2,3,6,7,6,7]8553; AVX512-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]8554; AVX512-NEXT: vpermq {{.*#+}} ymm14 = ymm1[2,1,2,3]8555; AVX512-NEXT: vpermq {{.*#+}} ymm30 = ymm30[2,2,2,2]8556; AVX512-NEXT: vpshufd {{.*#+}} ymm31 = ymm16[1,2,3,3,5,6,7,7]8557; AVX512-NEXT: vpermq $230, {{[-0-9]+}}(%r{{[sb]}}p), %ymm17 # 32-byte Folded Reload8558; AVX512-NEXT: # ymm17 = mem[2,1,2,3]8559; AVX512-NEXT: vpermq {{.*#+}} ymm16 = ymm22[3,3,3,3]8560; AVX512-NEXT: vmovdqa32 %zmm0, %zmm18 {%k2}8561; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload8562; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload8563; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]8564; AVX512-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero8565; AVX512-NEXT: vpsrldq {{.*#+}} xmm6 = xmm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero8566; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3]8567; AVX512-NEXT: vpshuflw {{.*#+}} ymm6 = ymm10[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]8568; AVX512-NEXT: vpermq {{.*#+}} ymm15 = ymm6[2,2,2,3]8569; AVX512-NEXT: vpshufb %ymm8, %ymm10, %ymm68570; AVX512-NEXT: vpermq {{.*#+}} ymm11 = ymm6[2,1,2,3]8571; AVX512-NEXT: vpermt2d %zmm0, %zmm5, %zmm48572; AVX512-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload8573; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm1[0,1,2,1]8574; AVX512-NEXT: vpshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,7,6,5]8575; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload8576; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm0[0,1,2,1]8577; AVX512-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,7,6,5]8578; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]8579; AVX512-NEXT: vpshuflw {{.*#+}} ymm6 = ymm12[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]8580; AVX512-NEXT: vpshufhw {{.*#+}} ymm6 = ymm6[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]8581; AVX512-NEXT: vpermq {{.*#+}} ymm22 = ymm6[2,2,2,3]8582; AVX512-NEXT: vpshufd {{.*#+}} ymm6 = ymm12[2,3,2,3,6,7,6,7]8583; AVX512-NEXT: vpshuflw {{.*#+}} ymm6 = ymm6[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]8584; AVX512-NEXT: vpermq {{.*#+}} ymm10 = ymm6[2,1,2,3]8585; AVX512-NEXT: vpermq $170, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload8586; AVX512-NEXT: # ymm7 = mem[2,2,2,2]8587; AVX512-NEXT: vpshufd $249, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload8588; AVX512-NEXT: # ymm3 = mem[1,2,3,3,5,6,7,7]8589; AVX512-NEXT: vpermq $230, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload8590; AVX512-NEXT: # ymm6 = mem[2,1,2,3]8591; AVX512-NEXT: vpermq $255, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload8592; AVX512-NEXT: # ymm5 = mem[3,3,3,3]8593; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]8594; AVX512-NEXT: vpshuflw {{.*#+}} ymm12 = ymm9[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]8595; AVX512-NEXT: vpermq {{.*#+}} ymm12 = ymm12[2,2,2,3]8596; AVX512-NEXT: vpshufb %ymm8, %ymm9, %ymm98597; AVX512-NEXT: vmovdqa64 %ymm8, %ymm238598; AVX512-NEXT: vpermq {{.*#+}} ymm9 = ymm9[2,1,2,3]8599; AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]8600; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm2[0,1,0,1]8601; AVX512-NEXT: vpshuflw {{.*#+}} ymm1 = ymm13[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]8602; AVX512-NEXT: vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]8603; AVX512-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]8604; AVX512-NEXT: vmovdqa32 %zmm4, %zmm0 {%k2}8605; AVX512-NEXT: vpermq {{.*#+}} ymm4 = ymm25[2,2,2,3]8606; AVX512-NEXT: vinserti64x4 $1, %ymm4, %zmm21, %zmm48607; AVX512-NEXT: vinserti64x4 $1, %ymm26, %zmm24, %zmm248608; AVX512-NEXT: vmovdqa32 %zmm4, %zmm24 {%k1}8609; AVX512-NEXT: vinserti64x4 $1, %ymm28, %zmm27, %zmm48610; AVX512-NEXT: movw $-28087, %ax # imm = 0x92498611; AVX512-NEXT: kmovw %eax, %k28612; AVX512-NEXT: vmovdqa32 %zmm4, %zmm24 {%k2}8613; AVX512-NEXT: vinserti64x4 $1, %ymm14, %zmm29, %zmm218614; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm26 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]8615; AVX512-NEXT: vpternlogq {{.*#+}} zmm21 = zmm21 ^ (zmm26 & (zmm21 ^ zmm24))8616; AVX512-NEXT: vpermq {{.*#+}} ymm4 = ymm31[2,2,2,3]8617; AVX512-NEXT: vinserti64x4 $1, %ymm4, %zmm30, %zmm48618; AVX512-NEXT: vinserti64x4 $1, %ymm16, %zmm17, %zmm148619; AVX512-NEXT: vmovdqa32 %zmm4, %zmm14 {%k1}8620; AVX512-NEXT: vinserti64x4 $1, %ymm11, %zmm15, %zmm48621; AVX512-NEXT: vmovdqa32 %zmm4, %zmm14 {%k2}8622; AVX512-NEXT: vinserti64x4 $1, %ymm10, %zmm22, %zmm48623; AVX512-NEXT: vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm26 & (zmm4 ^ zmm14))8624; AVX512-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]8625; AVX512-NEXT: vinserti64x4 $1, %ymm3, %zmm7, %zmm38626; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm6, %zmm28627; AVX512-NEXT: vpshufd {{.*#+}} ymm5 = ymm13[2,3,2,3,6,7,6,7]8628; AVX512-NEXT: vpshuflw {{.*#+}} ymm5 = ymm5[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]8629; AVX512-NEXT: vpermq {{.*#+}} ymm5 = ymm5[2,1,2,3]8630; AVX512-NEXT: vpermq $170, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload8631; AVX512-NEXT: # ymm6 = mem[2,2,2,2]8632; AVX512-NEXT: vpshufd $249, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Folded Reload8633; AVX512-NEXT: # ymm8 = mem[1,2,3,3,5,6,7,7]8634; AVX512-NEXT: vpermq $230, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload8635; AVX512-NEXT: # ymm10 = mem[2,1,2,3]8636; AVX512-NEXT: vpermq $255, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Folded Reload8637; AVX512-NEXT: # ymm11 = mem[3,3,3,3]8638; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload8639; AVX512-NEXT: vmovdqa64 %ymm23, %ymm78640; AVX512-NEXT: vpshufb %ymm7, %ymm13, %ymm148641; AVX512-NEXT: vpshuflw {{.*#+}} ymm15 = ymm13[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]8642; AVX512-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,2,2,3]8643; AVX512-NEXT: vmovdqa32 %zmm3, %zmm2 {%k1}8644; AVX512-NEXT: vpermq {{.*#+}} ymm14 = ymm14[2,1,2,3]8645; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload8646; AVX512-NEXT: vpshuflw {{.*#+}} ymm3 = ymm13[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]8647; AVX512-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]8648; AVX512-NEXT: vpermq {{.*#+}} ymm16 = ymm3[2,2,2,3]8649; AVX512-NEXT: vpshufd {{.*#+}} ymm3 = ymm13[2,3,2,3,6,7,6,7]8650; AVX512-NEXT: vpshuflw {{.*#+}} ymm3 = ymm3[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]8651; AVX512-NEXT: vpermq {{.*#+}} ymm22 = ymm3[2,1,2,3]8652; AVX512-NEXT: vinserti64x4 $1, %ymm9, %zmm12, %zmm38653; AVX512-NEXT: vmovdqa32 %zmm3, %zmm2 {%k2}8654; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload8655; AVX512-NEXT: vpmovzxwd {{.*#+}} xmm27 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero8656; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,1,3,3,4,5,6,7]8657; AVX512-NEXT: vpermq {{.*#+}} ymm28 = ymm3[0,0,2,1]8658; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload8659; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm7[0,0,2,1,4,5,6,7]8660; AVX512-NEXT: vpbroadcastq %xmm3, %ymm248661; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm7[0,2,2,3,4,5,6,7]8662; AVX512-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,4,4]8663; AVX512-NEXT: vpermq {{.*#+}} ymm25 = ymm3[0,0,2,1]8664; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm1, %zmm38665; AVX512-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 ^ (zmm26 & (zmm3 ^ zmm2))8666; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload8667; AVX512-NEXT: vpmovzxwd {{.*#+}} xmm29 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero8668; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm1[2,1,3,3,4,5,6,7]8669; AVX512-NEXT: vpermq {{.*#+}} ymm30 = ymm2[0,0,2,1]8670; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload8671; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm1[0,0,2,1,4,5,6,7]8672; AVX512-NEXT: vpbroadcastq %xmm5, %ymm58673; AVX512-NEXT: vpshuflw {{.*#+}} xmm13 = xmm1[0,2,2,3,4,5,6,7]8674; AVX512-NEXT: vpshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,4,4,4,4]8675; AVX512-NEXT: vpermq {{.*#+}} ymm13 = ymm13[0,0,2,1]8676; AVX512-NEXT: vpermq {{.*#+}} ymm8 = ymm8[2,2,2,3]8677; AVX512-NEXT: vinserti64x4 $1, %ymm8, %zmm6, %zmm68678; AVX512-NEXT: vinserti64x4 $1, %ymm11, %zmm10, %zmm88679; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload8680; AVX512-NEXT: vpmovzxwd {{.*#+}} xmm10 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero8681; AVX512-NEXT: vpshuflw {{.*#+}} xmm11 = xmm1[2,1,3,3,4,5,6,7]8682; AVX512-NEXT: vpermq {{.*#+}} ymm11 = ymm11[0,0,2,1]8683; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload8684; AVX512-NEXT: vpshuflw {{.*#+}} xmm9 = xmm1[0,0,2,1,4,5,6,7]8685; AVX512-NEXT: vpbroadcastq %xmm9, %ymm98686; AVX512-NEXT: vpshuflw {{.*#+}} xmm12 = xmm1[0,2,2,3,4,5,6,7]8687; AVX512-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,4,4,4]8688; AVX512-NEXT: vpermq {{.*#+}} ymm12 = ymm12[0,0,2,1]8689; AVX512-NEXT: vmovdqa32 %zmm6, %zmm8 {%k1}8690; AVX512-NEXT: vinserti64x4 $1, %ymm14, %zmm15, %zmm68691; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload8692; AVX512-NEXT: vpmovzxwd {{.*#+}} xmm14 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero8693; AVX512-NEXT: vpshuflw {{.*#+}} xmm15 = xmm1[2,1,3,3,4,5,6,7]8694; AVX512-NEXT: vpermq {{.*#+}} ymm15 = ymm15[0,0,2,1]8695; AVX512-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload8696; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm2[0,0,2,1,4,5,6,7]8697; AVX512-NEXT: vpbroadcastq %xmm1, %ymm18698; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]8699; AVX512-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,4,4]8700; AVX512-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]8701; AVX512-NEXT: vmovdqa32 %zmm6, %zmm8 {%k2}8702; AVX512-NEXT: vinserti64x4 $1, %ymm22, %zmm16, %zmm68703; AVX512-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm26 & (zmm6 ^ zmm8))8704; AVX512-NEXT: vpbroadcastq %xmm27, %ymm88705; AVX512-NEXT: vinserti64x4 $1, %ymm28, %zmm8, %zmm88706; AVX512-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload8707; AVX512-NEXT: vmovdqa32 %zmm8, %zmm7 {%k1}8708; AVX512-NEXT: vinserti64x4 $1, %ymm25, %zmm24, %zmm88709; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]8710; AVX512-NEXT: vpternlogq {{.*#+}} zmm8 = zmm8 ^ (zmm16 & (zmm8 ^ zmm7))8711; AVX512-NEXT: vpbroadcastq %xmm29, %ymm178712; AVX512-NEXT: vinserti64x4 $1, %ymm30, %zmm17, %zmm178713; AVX512-NEXT: vmovdqa32 %zmm17, %zmm19 {%k1}8714; AVX512-NEXT: vinserti64x4 $1, %ymm13, %zmm5, %zmm58715; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm16 & (zmm5 ^ zmm19))8716; AVX512-NEXT: vpbroadcastq %xmm10, %ymm108717; AVX512-NEXT: vinserti64x4 $1, %ymm11, %zmm10, %zmm108718; AVX512-NEXT: vmovdqa32 %zmm10, %zmm18 {%k1}8719; AVX512-NEXT: vinserti64x4 $1, %ymm12, %zmm9, %zmm98720; AVX512-NEXT: vpternlogq {{.*#+}} zmm9 = zmm9 ^ (zmm16 & (zmm9 ^ zmm18))8721; AVX512-NEXT: vpbroadcastq %xmm14, %ymm78722; AVX512-NEXT: vinserti64x4 $1, %ymm15, %zmm7, %zmm78723; AVX512-NEXT: vmovdqa32 %zmm7, %zmm0 {%k1}8724; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm18725; AVX512-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm16 & (zmm1 ^ zmm0))8726; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax8727; AVX512-NEXT: vmovdqa64 %zmm1, (%rax)8728; AVX512-NEXT: vmovdqa64 %zmm9, 192(%rax)8729; AVX512-NEXT: vmovdqa64 %zmm20, 256(%rax)8730; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8731; AVX512-NEXT: vmovaps %zmm0, 448(%rax)8732; AVX512-NEXT: vmovdqa64 %zmm5, 384(%rax)8733; AVX512-NEXT: vmovdqa64 %zmm8, 576(%rax)8734; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8735; AVX512-NEXT: vmovaps %zmm0, 640(%rax)8736; AVX512-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8737; AVX512-NEXT: vmovaps %zmm0, 64(%rax)8738; AVX512-NEXT: vmovdqa64 %zmm6, 128(%rax)8739; AVX512-NEXT: vmovdqa64 %zmm3, 320(%rax)8740; AVX512-NEXT: vmovdqa64 %zmm4, 512(%rax)8741; AVX512-NEXT: vmovdqa64 %zmm21, 704(%rax)8742; AVX512-NEXT: addq $744, %rsp # imm = 0x2E88743; AVX512-NEXT: vzeroupper8744; AVX512-NEXT: retq8745;8746; AVX512-FCP-LABEL: store_i16_stride6_vf64:8747; AVX512-FCP: # %bb.0:8748; AVX512-FCP-NEXT: subq $1224, %rsp # imm = 0x4C88749; AVX512-FCP-NEXT: vmovdqa 96(%rcx), %ymm18750; AVX512-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8751; AVX512-FCP-NEXT: vmovdqa 96(%rdx), %ymm28752; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8753; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15]8754; AVX512-FCP-NEXT: vpsrldq {{.*#+}} ymm1 = ymm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm1[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero8755; AVX512-FCP-NEXT: vpsrldq {{.*#+}} ymm2 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero8756; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]8757; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm22 = [4,5,4,5,4,5,4,5,21,22,21,22,21,22,23,23]8758; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm22, %zmm28759; AVX512-FCP-NEXT: vmovdqa 96(%rsi), %ymm48760; AVX512-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8761; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [u,u,u,u,4,5,10,11,u,u,u,u,u,u,u,u,24,25,22,23,20,21,26,27,u,u,u,u,u,u,u,u]8762; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm4, %ymm08763; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %ymm58764; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8765; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm5, %ymm38766; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11]8767; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm5[4],ymm4[4],ymm5[5],ymm4[5],ymm5[6],ymm4[6],ymm5[7],ymm4[7],ymm5[12],ymm4[12],ymm5[13],ymm4[13],ymm5[14],ymm4[14],ymm5[15],ymm4[15]8768; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm23 = [2,1,2,3,11,11,11,11]8769; AVX512-FCP-NEXT: vpermt2q %zmm3, %zmm23, %zmm08770; AVX512-FCP-NEXT: movw $18724, %ax # imm = 0x49248771; AVX512-FCP-NEXT: kmovw %eax, %k18772; AVX512-FCP-NEXT: vmovdqa32 %zmm2, %zmm0 {%k1}8773; AVX512-FCP-NEXT: vmovdqa 96(%r8), %ymm48774; AVX512-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8775; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm2 = ymm4[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]8776; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm21 = [12,1,2,13,4,5,14,7]8777; AVX512-FCP-NEXT: vmovdqa %ymm0, %ymm38778; AVX512-FCP-NEXT: vpermt2d %ymm2, %ymm21, %ymm38779; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm24 = [8,21,10,11,20,13,14,23]8780; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm8 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]8781; AVX512-FCP-NEXT: vpshufb %ymm8, %ymm4, %ymm28782; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm24, %zmm08783; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm08784; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8785; AVX512-FCP-NEXT: vmovdqa 96(%r9), %ymm28786; AVX512-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8787; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm9 = [u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,24,25,28,29,28,29,26,27,24,25,26,27,28,29,30,31]8788; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm2, %ymm08789; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm15 = [16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25,16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25]8790; AVX512-FCP-NEXT: # ymm15 = mem[0,1,0,1]8791; AVX512-FCP-NEXT: vpshufb %ymm15, %ymm2, %ymm28792; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm20 = [2,2,0,3,10,0,10,11]8793; AVX512-FCP-NEXT: vpermi2q %zmm0, %zmm2, %zmm208794; AVX512-FCP-NEXT: vmovdqa 64(%rcx), %ymm38795; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8796; AVX512-FCP-NEXT: vmovdqa 64(%rdx), %ymm48797; AVX512-FCP-NEXT: vmovdqa 96(%r9), %xmm28798; AVX512-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill8799; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm10 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]8800; AVX512-FCP-NEXT: vpshufb %xmm10, %xmm2, %xmm08801; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,1,4,5,6,7]8802; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm26 = [0,0,0,0,8,8,0,9]8803; AVX512-FCP-NEXT: vpermi2q %zmm0, %zmm2, %zmm268804; AVX512-FCP-NEXT: vpsrldq {{.*#+}} ymm0 = ymm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm3[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero8805; AVX512-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8806; AVX512-FCP-NEXT: vpsrldq {{.*#+}} ymm2 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero8807; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11]8808; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]8809; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm22, %zmm08810; AVX512-FCP-NEXT: vmovdqa 64(%rsi), %ymm48811; AVX512-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8812; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm4, %ymm28813; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm58814; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8815; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm5, %ymm38816; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]8817; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm5[4],ymm4[4],ymm5[5],ymm4[5],ymm5[6],ymm4[6],ymm5[7],ymm4[7],ymm5[12],ymm4[12],ymm5[13],ymm4[13],ymm5[14],ymm4[14],ymm5[15],ymm4[15]8818; AVX512-FCP-NEXT: vpermt2q %zmm3, %zmm23, %zmm28819; AVX512-FCP-NEXT: vmovdqa32 %zmm0, %zmm2 {%k1}8820; AVX512-FCP-NEXT: vmovdqa %ymm2, %ymm08821; AVX512-FCP-NEXT: vmovdqa 64(%r8), %ymm48822; AVX512-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8823; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm3 = ymm4[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]8824; AVX512-FCP-NEXT: vpermt2d %ymm3, %ymm21, %ymm08825; AVX512-FCP-NEXT: vpshufb %ymm8, %ymm4, %ymm38826; AVX512-FCP-NEXT: vpermt2d %zmm3, %zmm24, %zmm28827; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm08828; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8829; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm0 = [2,2,2,3,10,9,10,11]8830; AVX512-FCP-NEXT: vmovdqa 64(%r9), %ymm38831; AVX512-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8832; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm3, %ymm28833; AVX512-FCP-NEXT: vpshufb %ymm15, %ymm3, %ymm38834; AVX512-FCP-NEXT: vpermt2q %zmm2, %zmm0, %zmm38835; AVX512-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8836; AVX512-FCP-NEXT: vmovdqa 96(%rcx), %xmm48837; AVX512-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill8838; AVX512-FCP-NEXT: vmovdqa 96(%rdx), %xmm58839; AVX512-FCP-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill8840; AVX512-FCP-NEXT: vpsrldq {{.*#+}} xmm2 = xmm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero8841; AVX512-FCP-NEXT: vpsrldq {{.*#+}} xmm3 = xmm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero8842; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]8843; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm30 = [1,0,2,2,1,0,2,2,16,17,16,17,16,17,16,17]8844; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]8845; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm30, %zmm38846; AVX512-FCP-NEXT: vmovdqa 96(%rsi), %xmm68847; AVX512-FCP-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill8848; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7,8,9,6,7,4,5,10,11]8849; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm6, %xmm48850; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %xmm78851; AVX512-FCP-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill8852; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm7, %xmm58853; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]8854; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm29 = [8,8,0,9,0,1,0,1]8855; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]8856; AVX512-FCP-NEXT: vpermt2q %zmm5, %zmm29, %zmm48857; AVX512-FCP-NEXT: movw $9362, %ax # imm = 0x24928858; AVX512-FCP-NEXT: kmovw %eax, %k28859; AVX512-FCP-NEXT: vmovdqa32 %zmm3, %zmm4 {%k2}8860; AVX512-FCP-NEXT: vmovdqa %ymm4, %ymm38861; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm28 = [16,9,10,17,12,13,18,15]8862; AVX512-FCP-NEXT: vmovdqa 96(%r8), %xmm68863; AVX512-FCP-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill8864; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm5 = xmm6[2,1,3,3,4,5,6,7]8865; AVX512-FCP-NEXT: vpermt2d %zmm5, %zmm28, %zmm48866; AVX512-FCP-NEXT: vpmovzxwd {{.*#+}} xmm5 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero8867; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm27 = [0,1,8,3,4,9,6,7]8868; AVX512-FCP-NEXT: vpermt2d %ymm5, %ymm27, %ymm38869; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm4, %zmm3, %zmm38870; AVX512-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8871; AVX512-FCP-NEXT: vmovdqa 64(%rcx), %xmm58872; AVX512-FCP-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill8873; AVX512-FCP-NEXT: vmovdqa 64(%rdx), %xmm68874; AVX512-FCP-NEXT: vmovdqa %xmm6, (%rsp) # 16-byte Spill8875; AVX512-FCP-NEXT: vpsrldq {{.*#+}} xmm3 = xmm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero8876; AVX512-FCP-NEXT: vpsrldq {{.*#+}} xmm4 = xmm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero8877; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]8878; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]8879; AVX512-FCP-NEXT: vpermt2d %zmm3, %zmm30, %zmm48880; AVX512-FCP-NEXT: vmovdqa 64(%rsi), %xmm68881; AVX512-FCP-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill8882; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm6, %xmm38883; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %xmm78884; AVX512-FCP-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill8885; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm7, %xmm58886; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]8887; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]8888; AVX512-FCP-NEXT: vpermt2q %zmm5, %zmm29, %zmm38889; AVX512-FCP-NEXT: vmovdqa32 %zmm4, %zmm3 {%k2}8890; AVX512-FCP-NEXT: vmovdqa %ymm3, %ymm48891; AVX512-FCP-NEXT: vmovdqa 64(%r8), %xmm68892; AVX512-FCP-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill8893; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm5 = xmm6[2,1,3,3,4,5,6,7]8894; AVX512-FCP-NEXT: vpermt2d %zmm5, %zmm28, %zmm38895; AVX512-FCP-NEXT: vpmovzxwd {{.*#+}} xmm5 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero8896; AVX512-FCP-NEXT: vpermt2d %ymm5, %ymm27, %ymm48897; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm38898; AVX512-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8899; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm31 = [0,0,0,0,8,8,0,9]8900; AVX512-FCP-NEXT: vmovdqa 64(%r9), %xmm48901; AVX512-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill8902; AVX512-FCP-NEXT: vpshufb %xmm10, %xmm4, %xmm38903; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,0,2,1,4,5,6,7]8904; AVX512-FCP-NEXT: vpermt2q %zmm3, %zmm31, %zmm48905; AVX512-FCP-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8906; AVX512-FCP-NEXT: vmovdqa 32(%rcx), %ymm58907; AVX512-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8908; AVX512-FCP-NEXT: vmovdqa 32(%rdx), %ymm68909; AVX512-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8910; AVX512-FCP-NEXT: vpsrldq {{.*#+}} ymm3 = ymm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm5[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero8911; AVX512-FCP-NEXT: vpsrldq {{.*#+}} ymm4 = ymm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm6[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero8912; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]8913; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm6[4],ymm5[4],ymm6[5],ymm5[5],ymm6[6],ymm5[6],ymm6[7],ymm5[7],ymm6[12],ymm5[12],ymm6[13],ymm5[13],ymm6[14],ymm5[14],ymm6[15],ymm5[15]8914; AVX512-FCP-NEXT: vpermt2d %zmm4, %zmm22, %zmm38915; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %ymm68916; AVX512-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8917; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm6, %ymm48918; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm78919; AVX512-FCP-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8920; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm7, %ymm58921; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm4 = ymm5[0],ymm4[0],ymm5[1],ymm4[1],ymm5[2],ymm4[2],ymm5[3],ymm4[3],ymm5[8],ymm4[8],ymm5[9],ymm4[9],ymm5[10],ymm4[10],ymm5[11],ymm4[11]8922; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm7[4],ymm6[4],ymm7[5],ymm6[5],ymm7[6],ymm6[6],ymm7[7],ymm6[7],ymm7[12],ymm6[12],ymm7[13],ymm6[13],ymm7[14],ymm6[14],ymm7[15],ymm6[15]8923; AVX512-FCP-NEXT: vpermt2q %zmm5, %zmm23, %zmm48924; AVX512-FCP-NEXT: vmovdqa32 %zmm3, %zmm4 {%k1}8925; AVX512-FCP-NEXT: vmovdqa %ymm4, %ymm38926; AVX512-FCP-NEXT: vmovdqa 32(%r8), %ymm68927; AVX512-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8928; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm5 = ymm6[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]8929; AVX512-FCP-NEXT: vpermt2d %ymm5, %ymm21, %ymm38930; AVX512-FCP-NEXT: vpshufb %ymm8, %ymm6, %ymm58931; AVX512-FCP-NEXT: vpermt2d %zmm5, %zmm24, %zmm48932; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm4, %zmm3, %zmm38933; AVX512-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8934; AVX512-FCP-NEXT: vmovdqa 32(%r9), %ymm48935; AVX512-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill8936; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm4, %ymm38937; AVX512-FCP-NEXT: vpshufb %ymm15, %ymm4, %ymm48938; AVX512-FCP-NEXT: vpermt2q %zmm3, %zmm0, %zmm48939; AVX512-FCP-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8940; AVX512-FCP-NEXT: vmovdqa (%rcx), %ymm148941; AVX512-FCP-NEXT: vmovdqa (%rdx), %ymm138942; AVX512-FCP-NEXT: vpsrldq {{.*#+}} ymm3 = ymm14[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm14[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero8943; AVX512-FCP-NEXT: vpsrldq {{.*#+}} ymm4 = ymm13[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm13[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero8944; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]8945; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm13[4],ymm14[4],ymm13[5],ymm14[5],ymm13[6],ymm14[6],ymm13[7],ymm14[7],ymm13[12],ymm14[12],ymm13[13],ymm14[13],ymm13[14],ymm14[14],ymm13[15],ymm14[15]8946; AVX512-FCP-NEXT: vpermt2d %zmm4, %zmm22, %zmm38947; AVX512-FCP-NEXT: vmovdqa (%rsi), %ymm108948; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm10, %ymm48949; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm78950; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm7, %ymm18951; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm4[0],ymm1[1],ymm4[1],ymm1[2],ymm4[2],ymm1[3],ymm4[3],ymm1[8],ymm4[8],ymm1[9],ymm4[9],ymm1[10],ymm4[10],ymm1[11],ymm4[11]8952; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm7[4],ymm10[4],ymm7[5],ymm10[5],ymm7[6],ymm10[6],ymm7[7],ymm10[7],ymm7[12],ymm10[12],ymm7[13],ymm10[13],ymm7[14],ymm10[14],ymm7[15],ymm10[15]8953; AVX512-FCP-NEXT: vpermt2q %zmm4, %zmm23, %zmm18954; AVX512-FCP-NEXT: vmovdqa32 %zmm3, %zmm1 {%k1}8955; AVX512-FCP-NEXT: vmovdqa (%r8), %ymm128956; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm3 = ymm12[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]8957; AVX512-FCP-NEXT: vpermi2d %ymm3, %ymm1, %ymm218958; AVX512-FCP-NEXT: vpshufb %ymm8, %ymm12, %ymm38959; AVX512-FCP-NEXT: vpermt2d %zmm3, %zmm24, %zmm18960; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm21, %zmm258961; AVX512-FCP-NEXT: vmovdqa (%r9), %ymm118962; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm11, %ymm18963; AVX512-FCP-NEXT: vpshufb %ymm15, %ymm11, %ymm38964; AVX512-FCP-NEXT: vpermt2q %zmm1, %zmm0, %zmm38965; AVX512-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8966; AVX512-FCP-NEXT: vmovdqa 32(%rcx), %xmm38967; AVX512-FCP-NEXT: vmovdqa 32(%rdx), %xmm48968; AVX512-FCP-NEXT: vpsrldq {{.*#+}} xmm0 = xmm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero8969; AVX512-FCP-NEXT: vpsrldq {{.*#+}} xmm1 = xmm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero8970; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]8971; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]8972; AVX512-FCP-NEXT: vmovdqa64 %xmm4, %xmm218973; AVX512-FCP-NEXT: vmovdqa64 %xmm3, %xmm198974; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm30, %zmm18975; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %xmm48976; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %xmm58977; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm4, %xmm08978; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm5, %xmm38979; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]8980; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]8981; AVX512-FCP-NEXT: vmovdqa64 %xmm5, %xmm188982; AVX512-FCP-NEXT: vmovdqa64 %xmm4, %xmm178983; AVX512-FCP-NEXT: vpermt2q %zmm3, %zmm29, %zmm08984; AVX512-FCP-NEXT: vmovdqa32 %zmm1, %zmm0 {%k2}8985; AVX512-FCP-NEXT: vmovdqa %ymm0, %ymm18986; AVX512-FCP-NEXT: vmovdqa 32(%r8), %xmm48987; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm4[2,1,3,3,4,5,6,7]8988; AVX512-FCP-NEXT: vpermt2d %zmm3, %zmm28, %zmm08989; AVX512-FCP-NEXT: vpmovzxwd {{.*#+}} xmm3 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero8990; AVX512-FCP-NEXT: vmovdqa64 %xmm4, %xmm168991; AVX512-FCP-NEXT: vpermt2d %ymm3, %ymm27, %ymm18992; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm228993; AVX512-FCP-NEXT: vmovdqa (%rcx), %xmm68994; AVX512-FCP-NEXT: vmovdqa (%rdx), %xmm58995; AVX512-FCP-NEXT: vpsrldq {{.*#+}} xmm0 = xmm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero8996; AVX512-FCP-NEXT: vpsrldq {{.*#+}} xmm1 = xmm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero8997; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]8998; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]8999; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm30, %zmm39000; AVX512-FCP-NEXT: vmovdqa (%rsi), %xmm49001; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm19002; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm4, %xmm89003; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm1, %xmm29004; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm8 = xmm2[4],xmm8[4],xmm2[5],xmm8[5],xmm2[6],xmm8[6],xmm2[7],xmm8[7]9005; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]9006; AVX512-FCP-NEXT: vpermt2q %zmm2, %zmm29, %zmm89007; AVX512-FCP-NEXT: vmovdqa32 %zmm3, %zmm8 {%k2}9008; AVX512-FCP-NEXT: vmovdqa (%r8), %xmm29009; AVX512-FCP-NEXT: vpmovzxwd {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero9010; AVX512-FCP-NEXT: vpermi2d %ymm3, %ymm8, %ymm279011; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[2,1,3,3,4,5,6,7]9012; AVX512-FCP-NEXT: vpermt2d %zmm3, %zmm28, %zmm89013; AVX512-FCP-NEXT: vmovdqa 32(%r9), %xmm159014; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]9015; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm15, %xmm39016; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm9 = xmm15[0,0,2,1,4,5,6,7]9017; AVX512-FCP-NEXT: vpermt2q %zmm3, %zmm31, %zmm99018; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm8, %zmm27, %zmm239019; AVX512-FCP-NEXT: vmovdqa (%r9), %xmm39020; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm09021; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm8 = xmm3[0,0,2,1,4,5,6,7]9022; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm31, %zmm89023; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm7[0],ymm10[0],ymm7[1],ymm10[1],ymm7[2],ymm10[2],ymm7[3],ymm10[3],ymm7[8],ymm10[8],ymm7[9],ymm10[9],ymm7[10],ymm10[10],ymm7[11],ymm10[11]9024; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]9025; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm24 = [1,1,1,1,10,10,10,11]9026; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm24, %zmm19027; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]9028; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm4 = ymm13[0],ymm14[0],ymm13[1],ymm14[1],ymm13[2],ymm14[2],ymm13[3],ymm14[3],ymm13[8],ymm14[8],ymm13[9],ymm14[9],ymm13[10],ymm14[10],ymm13[11],ymm14[11]9029; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm30 = [17,18,17,18,0,0,19,19,5,4,2,2,5,4,6,6]9030; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm30, %zmm49031; AVX512-FCP-NEXT: vmovdqa32 %zmm1, %zmm4 {%k1}9032; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm5 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]9033; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm2, %xmm09034; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm6 = [0,9,2,3,8,5,6,11]9035; AVX512-FCP-NEXT: vmovdqa %ymm4, %ymm19036; AVX512-FCP-NEXT: vpermt2d %ymm0, %ymm6, %ymm19037; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm0 = ymm12[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]9038; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm27 = [8,9,20,11,12,21,14,15]9039; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm27, %zmm49040; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm4, %zmm1, %zmm289041; AVX512-FCP-NEXT: vpshuflw {{.*#+}} ymm0 = ymm11[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]9042; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm4 = [8,9,12,13,12,13,10,11,8,9,10,11,12,13,14,15]9043; AVX512-FCP-NEXT: vpshufb %xmm4, %xmm3, %xmm29044; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm1 = [0,0,0,1,0,10,10,0]9045; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm29046; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload9047; AVX512-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload9048; AVX512-FCP-NEXT: # ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11]9049; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload9050; AVX512-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload9051; AVX512-FCP-NEXT: # xmm3 = xmm3[4],mem[4],xmm3[5],mem[5],xmm3[6],mem[6],xmm3[7],mem[7]9052; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm24, %zmm39053; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload9054; AVX512-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload9055; AVX512-FCP-NEXT: # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]9056; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload9057; AVX512-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload9058; AVX512-FCP-NEXT: # ymm7 = ymm7[0],mem[0],ymm7[1],mem[1],ymm7[2],mem[2],ymm7[3],mem[3],ymm7[8],mem[8],ymm7[9],mem[9],ymm7[10],mem[10],ymm7[11],mem[11]9059; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm30, %zmm79060; AVX512-FCP-NEXT: vmovdqa32 %zmm3, %zmm7 {%k1}9061; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload9062; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm0, %xmm09063; AVX512-FCP-NEXT: vmovdqa %ymm7, %ymm39064; AVX512-FCP-NEXT: vpermt2d %ymm0, %ymm6, %ymm39065; AVX512-FCP-NEXT: vpshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload9066; AVX512-FCP-NEXT: # ymm0 = mem[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]9067; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm27, %zmm79068; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm3, %zmm299069; AVX512-FCP-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload9070; AVX512-FCP-NEXT: # ymm0 = mem[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]9071; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload9072; AVX512-FCP-NEXT: vpshufb %xmm4, %xmm3, %xmm79073; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm79074; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload9075; AVX512-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload9076; AVX512-FCP-NEXT: # ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11]9077; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload9078; AVX512-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload9079; AVX512-FCP-NEXT: # xmm3 = xmm3[4],mem[4],xmm3[5],mem[5],xmm3[6],mem[6],xmm3[7],mem[7]9080; AVX512-FCP-NEXT: vpermt2q %zmm0, %zmm24, %zmm39081; AVX512-FCP-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload9082; AVX512-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload9083; AVX512-FCP-NEXT: # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]9084; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload9085; AVX512-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload9086; AVX512-FCP-NEXT: # ymm10 = ymm10[0],mem[0],ymm10[1],mem[1],ymm10[2],mem[2],ymm10[3],mem[3],ymm10[8],mem[8],ymm10[9],mem[9],ymm10[10],mem[10],ymm10[11],mem[11]9087; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm30, %zmm109088; AVX512-FCP-NEXT: vmovdqa32 %zmm3, %zmm10 {%k1}9089; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload9090; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm0, %xmm09091; AVX512-FCP-NEXT: vmovdqa %ymm10, %ymm39092; AVX512-FCP-NEXT: vpermt2d %ymm0, %ymm6, %ymm39093; AVX512-FCP-NEXT: vpshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload9094; AVX512-FCP-NEXT: # ymm0 = mem[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]9095; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm27, %zmm109096; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm10, %zmm3, %zmm39097; AVX512-FCP-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload9098; AVX512-FCP-NEXT: # ymm10 = mem[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]9099; AVX512-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload9100; AVX512-FCP-NEXT: vpshufb %xmm4, %xmm0, %xmm09101; AVX512-FCP-NEXT: vpermt2q %zmm10, %zmm1, %zmm09102; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload9103; AVX512-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload9104; AVX512-FCP-NEXT: # ymm10 = ymm10[0],mem[0],ymm10[1],mem[1],ymm10[2],mem[2],ymm10[3],mem[3],ymm10[8],mem[8],ymm10[9],mem[9],ymm10[10],mem[10],ymm10[11],mem[11]9105; AVX512-FCP-NEXT: vmovdqa64 %xmm18, %xmm119106; AVX512-FCP-NEXT: vmovdqa64 %xmm17, %xmm129107; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm11 = xmm11[4],xmm12[4],xmm11[5],xmm12[5],xmm11[6],xmm12[6],xmm11[7],xmm12[7]9108; AVX512-FCP-NEXT: vpermt2q %zmm10, %zmm24, %zmm119109; AVX512-FCP-NEXT: vmovdqa64 %xmm21, %xmm109110; AVX512-FCP-NEXT: vmovdqa64 %xmm19, %xmm129111; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm10 = xmm10[4],xmm12[4],xmm10[5],xmm12[5],xmm10[6],xmm12[6],xmm10[7],xmm12[7]9112; AVX512-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload9113; AVX512-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm12 # 32-byte Folded Reload9114; AVX512-FCP-NEXT: # ymm12 = ymm12[0],mem[0],ymm12[1],mem[1],ymm12[2],mem[2],ymm12[3],mem[3],ymm12[8],mem[8],ymm12[9],mem[9],ymm12[10],mem[10],ymm12[11],mem[11]9115; AVX512-FCP-NEXT: vpermt2d %zmm10, %zmm30, %zmm129116; AVX512-FCP-NEXT: vmovdqa32 %zmm11, %zmm12 {%k1}9117; AVX512-FCP-NEXT: vmovdqa64 %xmm16, %xmm109118; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm10, %xmm59119; AVX512-FCP-NEXT: vpermi2d %ymm5, %ymm12, %ymm69120; AVX512-FCP-NEXT: vpshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload9121; AVX512-FCP-NEXT: # ymm5 = mem[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]9122; AVX512-FCP-NEXT: vpermt2d %zmm5, %zmm27, %zmm129123; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm12, %zmm6, %zmm59124; AVX512-FCP-NEXT: vpshufb %xmm4, %xmm15, %xmm49125; AVX512-FCP-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload9126; AVX512-FCP-NEXT: # ymm6 = mem[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]9127; AVX512-FCP-NEXT: vpermt2q %zmm6, %zmm1, %zmm49128; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]9129; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm1 & (zmm4 ^ zmm5))9130; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax9131; AVX512-FCP-NEXT: vmovdqa64 %zmm4, 256(%rax)9132; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm1 & (zmm0 ^ zmm3))9133; AVX512-FCP-NEXT: vmovdqa64 %zmm0, 448(%rax)9134; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm1 & (zmm7 ^ zmm29))9135; AVX512-FCP-NEXT: vmovdqa64 %zmm7, 640(%rax)9136; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm1 & (zmm2 ^ zmm28))9137; AVX512-FCP-NEXT: vmovdqa64 %zmm2, 64(%rax)9138; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]9139; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm8 = zmm8 ^ (zmm0 & (zmm8 ^ zmm23))9140; AVX512-FCP-NEXT: vmovdqa64 %zmm8, (%rax)9141; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm9 = zmm9 ^ (zmm0 & (zmm9 ^ zmm22))9142; AVX512-FCP-NEXT: vmovdqa64 %zmm9, 192(%rax)9143; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm1 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]9144; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload9145; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm1 & (zmm2 ^ zmm25))9146; AVX512-FCP-NEXT: vmovdqa64 %zmm2, 128(%rax)9147; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload9148; AVX512-FCP-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm2 # 64-byte Folded Reload9149; AVX512-FCP-NEXT: # zmm2 = zmm2 ^ (zmm1 & (zmm2 ^ mem))9150; AVX512-FCP-NEXT: vmovdqa64 %zmm2, 320(%rax)9151; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload9152; AVX512-FCP-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload9153; AVX512-FCP-NEXT: # zmm2 = zmm2 ^ (zmm0 & (zmm2 ^ mem))9154; AVX512-FCP-NEXT: vmovdqa64 %zmm2, 384(%rax)9155; AVX512-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload9156; AVX512-FCP-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm2 # 64-byte Folded Reload9157; AVX512-FCP-NEXT: # zmm2 = zmm2 ^ (zmm1 & (zmm2 ^ mem))9158; AVX512-FCP-NEXT: vmovdqa64 %zmm2, 512(%rax)9159; AVX512-FCP-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm20 # 64-byte Folded Reload9160; AVX512-FCP-NEXT: # zmm20 = zmm20 ^ (zmm1 & (zmm20 ^ mem))9161; AVX512-FCP-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm26 # 64-byte Folded Reload9162; AVX512-FCP-NEXT: # zmm26 = zmm26 ^ (zmm0 & (zmm26 ^ mem))9163; AVX512-FCP-NEXT: vmovdqa64 %zmm26, 576(%rax)9164; AVX512-FCP-NEXT: vmovdqa64 %zmm20, 704(%rax)9165; AVX512-FCP-NEXT: addq $1224, %rsp # imm = 0x4C89166; AVX512-FCP-NEXT: vzeroupper9167; AVX512-FCP-NEXT: retq9168;9169; AVX512DQ-LABEL: store_i16_stride6_vf64:9170; AVX512DQ: # %bb.0:9171; AVX512DQ-NEXT: subq $1400, %rsp # imm = 0x5789172; AVX512DQ-NEXT: vmovdqa 96(%rcx), %xmm09173; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9174; AVX512DQ-NEXT: vmovdqa 96(%rdx), %xmm39175; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]9176; AVX512DQ-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero9177; AVX512DQ-NEXT: vpsrldq {{.*#+}} xmm2 = xmm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero9178; AVX512DQ-NEXT: vmovdqa64 %xmm3, %xmm279179; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]9180; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm0 = [1,0,2,2,1,0,2,2,16,17,16,17,16,17,16,17]9181; AVX512DQ-NEXT: vpermt2d %zmm2, %zmm0, %zmm19182; AVX512DQ-NEXT: vmovdqa 96(%rsi), %xmm49183; AVX512DQ-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9184; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm4[0,1,2,1]9185; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,5]9186; AVX512DQ-NEXT: vmovdqa 96(%rdi), %xmm59187; AVX512DQ-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9188; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm3 = xmm5[0,1,2,1]9189; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,5]9190; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]9191; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]9192; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]9193; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm3[0,1,2,3],zmm2[0,1,0,1]9194; AVX512DQ-NEXT: movw $9362, %ax # imm = 0x24929195; AVX512DQ-NEXT: kmovw %eax, %k19196; AVX512DQ-NEXT: vmovdqa32 %zmm1, %zmm2 {%k1}9197; AVX512DQ-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9198; AVX512DQ-NEXT: vmovdqa 96(%r8), %xmm19199; AVX512DQ-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9200; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,1,3,3,4,5,6,7]9201; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]9202; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9203; AVX512DQ-NEXT: vmovdqa 96(%r9), %xmm29204; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm2[0,0,2,1,4,5,6,7]9205; AVX512DQ-NEXT: vpbroadcastq %xmm1, %ymm19206; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9207; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm2[0,2,2,3,4,5,6,7]9208; AVX512DQ-NEXT: vmovdqa64 %xmm2, %xmm299209; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,4,4]9210; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]9211; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9212; AVX512DQ-NEXT: vmovdqa 32(%rcx), %xmm49213; AVX512DQ-NEXT: vmovdqa 64(%rcx), %xmm29214; AVX512DQ-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9215; AVX512DQ-NEXT: vmovdqa 32(%rdx), %xmm59216; AVX512DQ-NEXT: vmovdqa 64(%rdx), %xmm39217; AVX512DQ-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9218; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]9219; AVX512DQ-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero9220; AVX512DQ-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero9221; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]9222; AVX512DQ-NEXT: vpermt2d %zmm2, %zmm0, %zmm19223; AVX512DQ-NEXT: vmovdqa 64(%rsi), %xmm69224; AVX512DQ-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9225; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm6[0,1,2,1]9226; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,5]9227; AVX512DQ-NEXT: vmovdqa 64(%rdi), %xmm79228; AVX512DQ-NEXT: vmovdqa %xmm7, (%rsp) # 16-byte Spill9229; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm3 = xmm7[0,1,2,1]9230; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,5]9231; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]9232; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]9233; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]9234; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm18 = zmm3[0,1,2,3],zmm2[0,1,0,1]9235; AVX512DQ-NEXT: vmovdqa32 %zmm1, %zmm18 {%k1}9236; AVX512DQ-NEXT: vmovdqa 64(%r8), %xmm19237; AVX512DQ-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9238; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,1,3,3,4,5,6,7]9239; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]9240; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9241; AVX512DQ-NEXT: vmovdqa 64(%r9), %xmm29242; AVX512DQ-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9243; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm2[0,0,2,1,4,5,6,7]9244; AVX512DQ-NEXT: vpbroadcastq %xmm1, %ymm19245; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9246; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm2[0,2,2,3,4,5,6,7]9247; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,4,4]9248; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]9249; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9250; AVX512DQ-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9251; AVX512DQ-NEXT: vpsrldq {{.*#+}} xmm1 = xmm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero9252; AVX512DQ-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9253; AVX512DQ-NEXT: vpsrldq {{.*#+}} xmm2 = xmm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero9254; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]9255; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]9256; AVX512DQ-NEXT: vpermt2d %zmm1, %zmm0, %zmm29257; AVX512DQ-NEXT: vmovdqa 32(%rsi), %xmm49258; AVX512DQ-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9259; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm4[0,1,2,1]9260; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,5]9261; AVX512DQ-NEXT: vmovdqa 32(%rdi), %xmm59262; AVX512DQ-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9263; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm3 = xmm5[0,1,2,1]9264; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,5]9265; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]9266; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]9267; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]9268; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm21 = zmm3[0,1,2,3],zmm1[0,1,0,1]9269; AVX512DQ-NEXT: vmovdqa32 %zmm2, %zmm21 {%k1}9270; AVX512DQ-NEXT: vmovdqa 32(%r8), %xmm19271; AVX512DQ-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9272; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,1,3,3,4,5,6,7]9273; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]9274; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9275; AVX512DQ-NEXT: vmovdqa 32(%r9), %xmm29276; AVX512DQ-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9277; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm2[0,0,2,1,4,5,6,7]9278; AVX512DQ-NEXT: vpbroadcastq %xmm1, %ymm19279; AVX512DQ-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9280; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm2[0,2,2,3,4,5,6,7]9281; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,4,4]9282; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]9283; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9284; AVX512DQ-NEXT: vmovdqa (%rcx), %xmm139285; AVX512DQ-NEXT: vmovdqa (%rdx), %xmm119286; AVX512DQ-NEXT: vpsrldq {{.*#+}} xmm1 = xmm13[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero9287; AVX512DQ-NEXT: vpsrldq {{.*#+}} xmm2 = xmm11[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero9288; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]9289; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm11[0],xmm13[0],xmm11[1],xmm13[1],xmm11[2],xmm13[2],xmm11[3],xmm13[3]9290; AVX512DQ-NEXT: vpermt2d %zmm1, %zmm0, %zmm29291; AVX512DQ-NEXT: vmovdqa (%rsi), %xmm89292; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm8[0,1,2,1]9293; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5]9294; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm79295; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm7[0,1,2,1]9296; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,5]9297; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]9298; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]9299; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]9300; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm25 = zmm1[0,1,2,3],zmm0[0,1,0,1]9301; AVX512DQ-NEXT: vmovdqa32 %zmm2, %zmm25 {%k1}9302; AVX512DQ-NEXT: vmovdqa (%r8), %xmm69303; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm6[2,1,3,3,4,5,6,7]9304; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]9305; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9306; AVX512DQ-NEXT: vmovdqa (%r9), %xmm99307; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm9[0,0,2,1,4,5,6,7]9308; AVX512DQ-NEXT: vpbroadcastq %xmm0, %ymm09309; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9310; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm9[0,2,2,3,4,5,6,7]9311; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]9312; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]9313; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9314; AVX512DQ-NEXT: vmovdqa 96(%rsi), %ymm29315; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm0 = ymm2[2,1,2,3,6,5,6,7]9316; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm0 = ymm0[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]9317; AVX512DQ-NEXT: vmovdqa 96(%rdi), %ymm39318; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm1 = ymm3[2,1,2,3,6,5,6,7]9319; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]9320; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]9321; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,1,2,3]9322; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9323; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm3[4],ymm2[4],ymm3[5],ymm2[5],ymm3[6],ymm2[6],ymm3[7],ymm2[7],ymm3[12],ymm2[12],ymm3[13],ymm2[13],ymm3[14],ymm2[14],ymm3[15],ymm2[15]9324; AVX512DQ-NEXT: vmovdqa64 %ymm3, %ymm169325; AVX512DQ-NEXT: vmovdqa64 %ymm2, %ymm209326; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,3,3,3]9327; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9328; AVX512DQ-NEXT: vmovdqa 96(%rcx), %ymm59329; AVX512DQ-NEXT: vpsrldq {{.*#+}} ymm0 = ymm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm5[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero9330; AVX512DQ-NEXT: vmovdqa 96(%rdx), %ymm39331; AVX512DQ-NEXT: vpsrldq {{.*#+}} ymm1 = ymm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm3[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero9332; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]9333; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]9334; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9335; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm3[4],ymm5[4],ymm3[5],ymm5[5],ymm3[6],ymm5[6],ymm3[7],ymm5[7],ymm3[12],ymm5[12],ymm3[13],ymm5[13],ymm3[14],ymm5[14],ymm3[15],ymm5[15]9336; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[1,2,3,3,5,6,7,7]9337; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9338; AVX512DQ-NEXT: vmovdqa64 64(%rsi), %ymm269339; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm0 = ymm26[2,1,2,3,6,5,6,7]9340; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm0 = ymm0[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]9341; AVX512DQ-NEXT: vmovdqa64 64(%rdi), %ymm249342; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm1 = ymm24[2,1,2,3,6,5,6,7]9343; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]9344; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]9345; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm199346; AVX512DQ-NEXT: vmovdqa 64(%rcx), %ymm49347; AVX512DQ-NEXT: vpsrldq {{.*#+}} ymm0 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero9348; AVX512DQ-NEXT: vmovdqa 64(%rdx), %ymm29349; AVX512DQ-NEXT: vpsrldq {{.*#+}} ymm10 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero9350; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm10[0],ymm0[0],ymm10[1],ymm0[1],ymm10[2],ymm0[2],ymm10[3],ymm0[3],ymm10[8],ymm0[8],ymm10[9],ymm0[9],ymm10[10],ymm0[10],ymm10[11],ymm0[11]9351; AVX512DQ-NEXT: vmovdqa64 %ymm0, %ymm179352; AVX512DQ-NEXT: vmovdqa 32(%rsi), %ymm09353; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9354; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[2,1,2,3,6,5,6,7]9355; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm0 = ymm0[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]9356; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm19357; AVX512DQ-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9358; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm10 = ymm1[2,1,2,3,6,5,6,7]9359; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm10 = ymm10[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]9360; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm10[0],ymm0[0],ymm10[1],ymm0[1],ymm10[2],ymm0[2],ymm10[3],ymm0[3],ymm10[8],ymm0[8],ymm10[9],ymm0[9],ymm10[10],ymm0[10],ymm10[11],ymm0[11]9361; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9362; AVX512DQ-NEXT: vmovdqa 32(%rcx), %ymm129363; AVX512DQ-NEXT: vpsrldq {{.*#+}} ymm0 = ymm12[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm12[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero9364; AVX512DQ-NEXT: vmovdqa 32(%rdx), %ymm149365; AVX512DQ-NEXT: vpsrldq {{.*#+}} ymm10 = ymm14[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm14[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero9366; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm10[0],ymm0[0],ymm10[1],ymm0[1],ymm10[2],ymm0[2],ymm10[3],ymm0[3],ymm10[8],ymm0[8],ymm10[9],ymm0[9],ymm10[10],ymm0[10],ymm10[11],ymm0[11]9367; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9368; AVX512DQ-NEXT: vmovdqa (%rsi), %ymm109369; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm0 = ymm10[2,1,2,3,6,5,6,7]9370; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm0 = ymm0[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]9371; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm19372; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm15 = ymm1[2,1,2,3,6,5,6,7]9373; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm15 = ymm15[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15]9374; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm15[0],ymm0[0],ymm15[1],ymm0[1],ymm15[2],ymm0[2],ymm15[3],ymm0[3],ymm15[8],ymm0[8],ymm15[9],ymm0[9],ymm15[10],ymm0[10],ymm15[11],ymm0[11]9375; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9376; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm11[4],xmm13[4],xmm11[5],xmm13[5],xmm11[6],xmm13[6],xmm11[7],xmm13[7]9377; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm22 = [17,18,17,18,0,0,19,19,5,4,2,2,5,4,6,6]9378; AVX512DQ-NEXT: vmovdqa (%rcx), %ymm139379; AVX512DQ-NEXT: vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9380; AVX512DQ-NEXT: vmovdqa (%rdx), %ymm119381; AVX512DQ-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9382; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm11 = ymm11[0],ymm13[0],ymm11[1],ymm13[1],ymm11[2],ymm13[2],ymm11[3],ymm13[3],ymm11[8],ymm13[8],ymm11[9],ymm13[9],ymm11[10],ymm13[10],ymm11[11],ymm13[11]9383; AVX512DQ-NEXT: vpermt2d %zmm0, %zmm22, %zmm119384; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm7[4],xmm8[4],xmm7[5],xmm8[5],xmm7[6],xmm8[6],xmm7[7],xmm8[7]9385; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[1,1,1,1]9386; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm7 = ymm1[4],ymm10[4],ymm1[5],ymm10[5],ymm1[6],ymm10[6],ymm1[7],ymm10[7],ymm1[12],ymm10[12],ymm1[13],ymm10[13],ymm1[14],ymm10[14],ymm1[15],ymm10[15]9387; AVX512DQ-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9388; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm10[0],ymm1[1],ymm10[1],ymm1[2],ymm10[2],ymm1[3],ymm10[3],ymm1[8],ymm10[8],ymm1[9],ymm10[9],ymm1[10],ymm10[10],ymm1[11],ymm10[11]9389; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,3]9390; AVX512DQ-NEXT: vpmovzxwd {{.*#+}} xmm30 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero9391; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm10 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]9392; AVX512DQ-NEXT: vpshufb %xmm10, %xmm6, %xmm69393; AVX512DQ-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,1,0,1]9394; AVX512DQ-NEXT: vmovdqa (%r8), %ymm79395; AVX512DQ-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9396; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm7 = ymm7[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]9397; AVX512DQ-NEXT: vpermq {{.*#+}} ymm7 = ymm7[2,2,2,2]9398; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm8 = xmm9[2,3,2,3]9399; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,2,2,1,4,5,6,7]9400; AVX512DQ-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,1,0,1]9401; AVX512DQ-NEXT: vmovdqa (%r9), %ymm99402; AVX512DQ-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9403; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm13 = ymm9[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]9404; AVX512DQ-NEXT: vpermq {{.*#+}} ymm13 = ymm13[2,2,2,2]9405; AVX512DQ-NEXT: movw $18724, %ax # imm = 0x49249406; AVX512DQ-NEXT: kmovw %eax, %k29407; AVX512DQ-NEXT: vinserti32x8 $1, %ymm1, %zmm0, %zmm11 {%k2}9408; AVX512DQ-NEXT: vinserti32x8 $1, %ymm7, %zmm6, %zmm11 {%k1}9409; AVX512DQ-NEXT: vinserti64x4 $1, %ymm13, %zmm8, %zmm239410; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm31 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]9411; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm23 = zmm23 ^ (zmm31 & (zmm23 ^ zmm11))9412; AVX512DQ-NEXT: vmovdqa 96(%r8), %ymm19413; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm0 = ymm1[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]9414; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]9415; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9416; AVX512DQ-NEXT: vmovdqa64 %xmm27, %xmm69417; AVX512DQ-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm6 # 16-byte Folded Reload9418; AVX512DQ-NEXT: # xmm6 = xmm6[4],mem[4],xmm6[5],mem[5],xmm6[6],mem[6],xmm6[7],mem[7]9419; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[2],ymm5[2],ymm3[3],ymm5[3],ymm3[8],ymm5[8],ymm3[9],ymm5[9],ymm3[10],ymm5[10],ymm3[11],ymm5[11]9420; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm9 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]9421; AVX512DQ-NEXT: vpshufb %ymm9, %ymm1, %ymm39422; AVX512DQ-NEXT: vpermq {{.*#+}} ymm27 = ymm3[2,1,2,3]9423; AVX512DQ-NEXT: vpermt2d %zmm6, %zmm22, %zmm09424; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload9425; AVX512DQ-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload9426; AVX512DQ-NEXT: # xmm3 = xmm3[4],mem[4],xmm3[5],mem[5],xmm3[6],mem[6],xmm3[7],mem[7]9427; AVX512DQ-NEXT: vpermq {{.*#+}} ymm5 = ymm3[1,1,1,1]9428; AVX512DQ-NEXT: vmovdqa64 %ymm16, %ymm39429; AVX512DQ-NEXT: vmovdqa64 %ymm20, %ymm69430; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm3 = ymm3[0],ymm6[0],ymm3[1],ymm6[1],ymm3[2],ymm6[2],ymm3[3],ymm6[3],ymm3[8],ymm6[8],ymm3[9],ymm6[9],ymm3[10],ymm6[10],ymm3[11],ymm6[11]9431; AVX512DQ-NEXT: vpermq {{.*#+}} ymm6 = ymm3[2,2,2,3]9432; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload9433; AVX512DQ-NEXT: vpmovzxwd {{.*#+}} xmm16 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero9434; AVX512DQ-NEXT: vpshufb %xmm10, %xmm3, %xmm79435; AVX512DQ-NEXT: vmovdqa %xmm10, %xmm139436; AVX512DQ-NEXT: vpermq {{.*#+}} ymm7 = ymm7[0,1,0,1]9437; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]9438; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,2,2,2]9439; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm8 = xmm29[2,3,2,3]9440; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,2,2,1,4,5,6,7]9441; AVX512DQ-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,1,0,1]9442; AVX512DQ-NEXT: vmovdqa 96(%r9), %ymm119443; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm15 = ymm11[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]9444; AVX512DQ-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,2,2,2]9445; AVX512DQ-NEXT: vinserti32x8 $1, %ymm6, %zmm5, %zmm0 {%k2}9446; AVX512DQ-NEXT: vinserti32x8 $1, %ymm1, %zmm7, %zmm0 {%k1}9447; AVX512DQ-NEXT: vinserti64x4 $1, %ymm15, %zmm8, %zmm299448; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm29 = zmm29 ^ (zmm31 & (zmm29 ^ zmm0))9449; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm0 = ymm11[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]9450; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]9451; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]9452; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9453; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm0 = ymm11[2,3,2,3,6,7,6,7]9454; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm0 = ymm0[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]9455; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,1,2,3]9456; AVX512DQ-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9457; AVX512DQ-NEXT: vpermq {{.*#+}} ymm20 = ymm19[2,1,2,3]9458; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload9459; AVX512DQ-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload9460; AVX512DQ-NEXT: # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]9461; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm2[4],ymm4[4],ymm2[5],ymm4[5],ymm2[6],ymm4[6],ymm2[7],ymm4[7],ymm2[12],ymm4[12],ymm2[13],ymm4[13],ymm2[14],ymm4[14],ymm2[15],ymm4[15]9462; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[8],ymm4[8],ymm2[9],ymm4[9],ymm2[10],ymm4[10],ymm2[11],ymm4[11]9463; AVX512DQ-NEXT: vmovdqa64 %ymm26, %ymm79464; AVX512DQ-NEXT: vmovdqa64 %ymm24, %ymm89465; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm8[4],ymm7[4],ymm8[5],ymm7[5],ymm8[6],ymm7[6],ymm8[7],ymm7[7],ymm8[12],ymm7[12],ymm8[13],ymm7[13],ymm8[14],ymm7[14],ymm8[15],ymm7[15]9466; AVX512DQ-NEXT: vpermq {{.*#+}} ymm6 = ymm2[3,3,3,3]9467; AVX512DQ-NEXT: vpermq {{.*#+}} ymm19 = ymm17[2,2,2,2]9468; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm17 = ymm5[1,2,3,3,5,6,7,7]9469; AVX512DQ-NEXT: vpermt2d %zmm0, %zmm22, %zmm19470; AVX512DQ-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload9471; AVX512DQ-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm4 # 16-byte Folded Reload9472; AVX512DQ-NEXT: # xmm4 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]9473; AVX512DQ-NEXT: vmovdqa 64(%r8), %ymm39474; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm5 = ymm3[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]9475; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm5[2,2,2,3]9476; AVX512DQ-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9477; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload9478; AVX512DQ-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm15 # 16-byte Folded Reload9479; AVX512DQ-NEXT: # xmm15 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]9480; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm10 = ymm14[4],ymm12[4],ymm14[5],ymm12[5],ymm14[6],ymm12[6],ymm14[7],ymm12[7],ymm14[12],ymm12[12],ymm14[13],ymm12[13],ymm14[14],ymm12[14],ymm14[15],ymm12[15]9481; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm12 = ymm14[0],ymm12[0],ymm14[1],ymm12[1],ymm14[2],ymm12[2],ymm14[3],ymm12[3],ymm14[8],ymm12[8],ymm14[9],ymm12[9],ymm14[10],ymm12[10],ymm14[11],ymm12[11]9482; AVX512DQ-NEXT: vpshufb %ymm9, %ymm3, %ymm149483; AVX512DQ-NEXT: vpermq {{.*#+}} ymm24 = ymm14[2,1,2,3]9484; AVX512DQ-NEXT: vpermt2d %zmm15, %zmm22, %zmm129485; AVX512DQ-NEXT: vmovdqa 64(%r9), %ymm159486; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm14 = ymm15[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]9487; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm14 = ymm14[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]9488; AVX512DQ-NEXT: vpermq {{.*#+}} ymm28 = ymm14[2,2,2,3]9489; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm8[0],ymm7[0],ymm8[1],ymm7[1],ymm8[2],ymm7[2],ymm8[3],ymm7[3],ymm8[8],ymm7[8],ymm8[9],ymm7[9],ymm8[10],ymm7[10],ymm8[11],ymm7[11]9490; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm14 = ymm15[2,3,2,3,6,7,6,7]9491; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm14 = ymm14[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]9492; AVX512DQ-NEXT: vpermq {{.*#+}} ymm26 = ymm14[2,1,2,3]9493; AVX512DQ-NEXT: vpermq {{.*#+}} ymm14 = ymm4[1,1,1,1]9494; AVX512DQ-NEXT: vpermq {{.*#+}} ymm8 = ymm0[2,2,2,3]9495; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload9496; AVX512DQ-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm4 # 16-byte Folded Reload9497; AVX512DQ-NEXT: # xmm4 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]9498; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload9499; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload9500; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm9 = ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15]9501; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11]9502; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload9503; AVX512DQ-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero9504; AVX512DQ-NEXT: vmovdqa64 %xmm13, %xmm229505; AVX512DQ-NEXT: vpshufb %xmm13, %xmm5, %xmm139506; AVX512DQ-NEXT: vpermq {{.*#+}} ymm13 = ymm13[0,1,0,1]9507; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm3 = ymm3[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]9508; AVX512DQ-NEXT: vpermq {{.*#+}} ymm7 = ymm3[2,2,2,2]9509; AVX512DQ-NEXT: vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload9510; AVX512DQ-NEXT: # xmm11 = mem[2,3,2,3]9511; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[0,2,2,1,4,5,6,7]9512; AVX512DQ-NEXT: vpermq {{.*#+}} ymm11 = ymm11[0,1,0,1]9513; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm15 = ymm15[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]9514; AVX512DQ-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,2,2,2]9515; AVX512DQ-NEXT: vpermq {{.*#+}} ymm4 = ymm4[1,1,1,1]9516; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = ymm0[2,2,2,3]9517; AVX512DQ-NEXT: vpbroadcastq %xmm16, %ymm59518; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9519; AVX512DQ-NEXT: vinserti32x8 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm16 {%k2} # 32-byte Folded Reload9520; AVX512DQ-NEXT: vpbroadcastq %xmm2, %ymm29521; AVX512DQ-NEXT: vinserti32x8 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm18 {%k2} # 32-byte Folded Reload9522; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload9523; AVX512DQ-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero9524; AVX512DQ-NEXT: vpbroadcastq %xmm2, %ymm29525; AVX512DQ-NEXT: vinserti32x8 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm21 {%k2} # 32-byte Folded Reload9526; AVX512DQ-NEXT: vpbroadcastq %xmm30, %ymm29527; AVX512DQ-NEXT: vinserti32x8 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm25 {%k2} # 32-byte Folded Reload9528; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload9529; AVX512DQ-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm5 # 32-byte Folded Reload9530; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]9531; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm2 & (zmm5 ^ zmm16))9532; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9533; AVX512DQ-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm16, %zmm30 # 32-byte Folded Reload9534; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm30 = zmm30 ^ (zmm2 & (zmm30 ^ zmm18))9535; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9536; AVX512DQ-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm16, %zmm18 # 32-byte Folded Reload9537; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm18 = zmm18 ^ (zmm2 & (zmm18 ^ zmm21))9538; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9539; AVX512DQ-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm16, %zmm16 # 32-byte Folded Reload9540; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm16 = zmm16 ^ (zmm2 & (zmm16 ^ zmm25))9541; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload9542; AVX512DQ-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm21 # 32-byte Folded Reload9543; AVX512DQ-NEXT: vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload9544; AVX512DQ-NEXT: # ymm2 = mem[2,2,2,3]9545; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload9546; AVX512DQ-NEXT: vinserti32x8 $1, %ymm2, %zmm25, %zmm21 {%k2}9547; AVX512DQ-NEXT: movw $-28087, %ax # imm = 0x92499548; AVX512DQ-NEXT: kmovw %eax, %k39549; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload9550; AVX512DQ-NEXT: vinserti32x8 $1, %ymm27, %zmm2, %zmm21 {%k3}9551; AVX512DQ-NEXT: vinserti64x4 $1, %ymm6, %zmm20, %zmm69552; AVX512DQ-NEXT: vpermq {{.*#+}} ymm2 = ymm17[2,2,2,3]9553; AVX512DQ-NEXT: vinserti32x8 $1, %ymm2, %zmm19, %zmm6 {%k2}9554; AVX512DQ-NEXT: vinserti32x8 $1, %ymm8, %zmm14, %zmm1 {%k2}9555; AVX512DQ-NEXT: vinserti32x8 $1, %ymm7, %zmm13, %zmm1 {%k1}9556; AVX512DQ-NEXT: vinserti32x8 $1, %ymm3, %zmm4, %zmm12 {%k2}9557; AVX512DQ-NEXT: vinserti64x4 $1, %ymm15, %zmm11, %zmm29558; AVX512DQ-NEXT: vpermq $230, {{[-0-9]+}}(%r{{[sb]}}p), %ymm17 # 32-byte Folded Reload9559; AVX512DQ-NEXT: # ymm17 = mem[2,1,2,3]9560; AVX512DQ-NEXT: vpermq {{.*#+}} ymm20 = ymm9[3,3,3,3]9561; AVX512DQ-NEXT: vpermq $170, {{[-0-9]+}}(%r{{[sb]}}p), %ymm19 # 32-byte Folded Reload9562; AVX512DQ-NEXT: # ymm19 = mem[2,2,2,2]9563; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm25 = ymm10[1,2,3,3,5,6,7,7]9564; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm31 & (zmm2 ^ zmm1))9565; AVX512DQ-NEXT: vmovdqa64 %xmm22, %xmm19566; AVX512DQ-NEXT: vpshufb %xmm1, %xmm0, %xmm19567; AVX512DQ-NEXT: vmovdqa 32(%r8), %ymm89568; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]9569; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm9 = ymm8[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]9570; AVX512DQ-NEXT: vpermq {{.*#+}} ymm9 = ymm9[2,2,2,2]9571; AVX512DQ-NEXT: vinserti32x8 $1, %ymm9, %zmm1, %zmm12 {%k1}9572; AVX512DQ-NEXT: vmovdqa 32(%r9), %ymm19573; AVX512DQ-NEXT: vpshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload9574; AVX512DQ-NEXT: # xmm9 = mem[2,3,2,3]9575; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[0,2,2,1,4,5,6,7]9576; AVX512DQ-NEXT: vpermq {{.*#+}} ymm9 = ymm9[0,1,0,1]9577; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm11 = ymm1[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]9578; AVX512DQ-NEXT: vpermq {{.*#+}} ymm11 = ymm11[2,2,2,2]9579; AVX512DQ-NEXT: vinserti64x4 $1, %ymm11, %zmm9, %zmm99580; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm11 = ymm8[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]9581; AVX512DQ-NEXT: vpermq {{.*#+}} ymm11 = ymm11[2,2,2,3]9582; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm4 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]9583; AVX512DQ-NEXT: vpshufb %ymm4, %ymm8, %ymm89584; AVX512DQ-NEXT: vpermq {{.*#+}} ymm8 = ymm8[2,1,2,3]9585; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm9 = zmm9 ^ (zmm31 & (zmm9 ^ zmm12))9586; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload9587; AVX512DQ-NEXT: vpsrldq {{.*#+}} ymm12 = ymm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm3[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero9588; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload9589; AVX512DQ-NEXT: vpsrldq {{.*#+}} ymm13 = ymm0[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm0[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero9590; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} ymm12 = ymm13[0],ymm12[0],ymm13[1],ymm12[1],ymm13[2],ymm12[2],ymm13[3],ymm12[3],ymm13[8],ymm12[8],ymm13[9],ymm12[9],ymm13[10],ymm12[10],ymm13[11],ymm12[11]9591; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm13 = ymm1[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]9592; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm13 = ymm13[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]9593; AVX512DQ-NEXT: vpermq {{.*#+}} ymm13 = ymm13[2,2,2,3]9594; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[2,3,2,3,6,7,6,7]9595; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm1 = ymm1[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]9596; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,1,2,3]9597; AVX512DQ-NEXT: vpermq $230, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload9598; AVX512DQ-NEXT: # ymm14 = mem[2,1,2,3]9599; AVX512DQ-NEXT: vpermq $255, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Folded Reload9600; AVX512DQ-NEXT: # ymm15 = mem[3,3,3,3]9601; AVX512DQ-NEXT: vpermq {{.*#+}} ymm12 = ymm12[2,2,2,2]9602; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} ymm10 = ymm0[4],ymm3[4],ymm0[5],ymm3[5],ymm0[6],ymm3[6],ymm0[7],ymm3[7],ymm0[12],ymm3[12],ymm0[13],ymm3[13],ymm0[14],ymm3[14],ymm0[15],ymm3[15]9603; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm10 = ymm10[1,2,3,3,5,6,7,7]9604; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload9605; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm0 = ymm3[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]9606; AVX512DQ-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,3]9607; AVX512DQ-NEXT: vpshufb %ymm4, %ymm3, %ymm49608; AVX512DQ-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,1,2,3]9609; AVX512DQ-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload9610; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm3 = ymm7[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]9611; AVX512DQ-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]9612; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]9613; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm7 = ymm7[2,3,2,3,6,7,6,7]9614; AVX512DQ-NEXT: vpshuflw {{.*#+}} ymm7 = ymm7[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]9615; AVX512DQ-NEXT: vpermq {{.*#+}} ymm7 = ymm7[2,1,2,3]9616; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload9617; AVX512DQ-NEXT: vinserti32x8 $1, %ymm24, %zmm27, %zmm6 {%k3}9618; AVX512DQ-NEXT: vinserti64x4 $1, %ymm20, %zmm17, %zmm179619; AVX512DQ-NEXT: vpermq {{.*#+}} ymm20 = ymm25[2,2,2,3]9620; AVX512DQ-NEXT: vinserti32x8 $1, %ymm20, %zmm19, %zmm17 {%k2}9621; AVX512DQ-NEXT: vinserti32x8 $1, %ymm8, %zmm11, %zmm17 {%k3}9622; AVX512DQ-NEXT: vinserti64x4 $1, %ymm15, %zmm14, %zmm89623; AVX512DQ-NEXT: vpermq {{.*#+}} ymm10 = ymm10[2,2,2,3]9624; AVX512DQ-NEXT: vinserti32x8 $1, %ymm10, %zmm12, %zmm8 {%k2}9625; AVX512DQ-NEXT: vinserti32x8 $1, %ymm4, %zmm0, %zmm8 {%k3}9626; AVX512DQ-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9627; AVX512DQ-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload9628; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm4 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]9629; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm4 & (zmm0 ^ zmm21))9630; AVX512DQ-NEXT: vinserti64x4 $1, %ymm26, %zmm28, %zmm109631; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm4 & (zmm10 ^ zmm6))9632; AVX512DQ-NEXT: vinserti64x4 $1, %ymm1, %zmm13, %zmm19633; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm4 & (zmm1 ^ zmm17))9634; AVX512DQ-NEXT: vinserti64x4 $1, %ymm7, %zmm3, %zmm39635; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 ^ (zmm4 & (zmm3 ^ zmm8))9636; AVX512DQ-NEXT: movq {{[0-9]+}}(%rsp), %rax9637; AVX512DQ-NEXT: vmovdqa64 %zmm9, 256(%rax)9638; AVX512DQ-NEXT: vmovdqa64 %zmm2, 448(%rax)9639; AVX512DQ-NEXT: vmovdqa64 %zmm29, 640(%rax)9640; AVX512DQ-NEXT: vmovdqa64 %zmm23, 64(%rax)9641; AVX512DQ-NEXT: vmovdqa64 %zmm16, (%rax)9642; AVX512DQ-NEXT: vmovdqa64 %zmm18, 192(%rax)9643; AVX512DQ-NEXT: vmovdqa64 %zmm30, 384(%rax)9644; AVX512DQ-NEXT: vmovdqa64 %zmm5, 576(%rax)9645; AVX512DQ-NEXT: vmovdqa64 %zmm3, 128(%rax)9646; AVX512DQ-NEXT: vmovdqa64 %zmm1, 320(%rax)9647; AVX512DQ-NEXT: vmovdqa64 %zmm10, 512(%rax)9648; AVX512DQ-NEXT: vmovdqa64 %zmm0, 704(%rax)9649; AVX512DQ-NEXT: addq $1400, %rsp # imm = 0x5789650; AVX512DQ-NEXT: vzeroupper9651; AVX512DQ-NEXT: retq9652;9653; AVX512DQ-FCP-LABEL: store_i16_stride6_vf64:9654; AVX512DQ-FCP: # %bb.0:9655; AVX512DQ-FCP-NEXT: subq $1224, %rsp # imm = 0x4C89656; AVX512DQ-FCP-NEXT: vmovdqa 96(%rcx), %ymm19657; AVX512DQ-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9658; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdx), %ymm29659; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9660; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15]9661; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} ymm1 = ymm1[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm1[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero9662; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} ymm2 = ymm2[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm2[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero9663; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]9664; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm22 = [4,5,4,5,4,5,4,5,21,22,21,22,21,22,23,23]9665; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm22, %zmm29666; AVX512DQ-FCP-NEXT: vmovdqa 96(%rsi), %ymm49667; AVX512DQ-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9668; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [u,u,u,u,4,5,10,11,u,u,u,u,u,u,u,u,24,25,22,23,20,21,26,27,u,u,u,u,u,u,u,u]9669; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm4, %ymm09670; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %ymm59671; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9672; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm5, %ymm39673; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11]9674; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm5[4],ymm4[4],ymm5[5],ymm4[5],ymm5[6],ymm4[6],ymm5[7],ymm4[7],ymm5[12],ymm4[12],ymm5[13],ymm4[13],ymm5[14],ymm4[14],ymm5[15],ymm4[15]9675; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm23 = [2,1,2,3,11,11,11,11]9676; AVX512DQ-FCP-NEXT: vpermt2q %zmm3, %zmm23, %zmm09677; AVX512DQ-FCP-NEXT: movw $18724, %ax # imm = 0x49249678; AVX512DQ-FCP-NEXT: kmovw %eax, %k19679; AVX512DQ-FCP-NEXT: vmovdqa32 %zmm2, %zmm0 {%k1}9680; AVX512DQ-FCP-NEXT: vmovdqa 96(%r8), %ymm49681; AVX512DQ-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9682; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm2 = ymm4[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]9683; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm21 = [12,1,2,13,4,5,14,7]9684; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, %ymm39685; AVX512DQ-FCP-NEXT: vpermt2d %ymm2, %ymm21, %ymm39686; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm24 = [8,21,10,11,20,13,14,23]9687; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm8 = [u,u,u,u,u,u,u,u,14,15,14,15,14,15,14,15,28,29,26,27,26,27,30,31,30,31,30,31,30,31,30,31]9688; AVX512DQ-FCP-NEXT: vpshufb %ymm8, %ymm4, %ymm29689; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm24, %zmm09690; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm09691; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9692; AVX512DQ-FCP-NEXT: vmovdqa 96(%r9), %ymm29693; AVX512DQ-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9694; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm9 = [u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,24,25,28,29,28,29,26,27,24,25,26,27,28,29,30,31]9695; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm2, %ymm09696; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm15 = [16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25,16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25]9697; AVX512DQ-FCP-NEXT: # ymm15 = mem[0,1,0,1]9698; AVX512DQ-FCP-NEXT: vpshufb %ymm15, %ymm2, %ymm29699; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm20 = [2,2,0,3,10,0,10,11]9700; AVX512DQ-FCP-NEXT: vpermi2q %zmm0, %zmm2, %zmm209701; AVX512DQ-FCP-NEXT: vmovdqa 64(%rcx), %ymm39702; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9703; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdx), %ymm49704; AVX512DQ-FCP-NEXT: vmovdqa 96(%r9), %xmm29705; AVX512DQ-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9706; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm10 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]9707; AVX512DQ-FCP-NEXT: vpshufb %xmm10, %xmm2, %xmm09708; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,1,4,5,6,7]9709; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm26 = [0,0,0,0,8,8,0,9]9710; AVX512DQ-FCP-NEXT: vpermi2q %zmm0, %zmm2, %zmm269711; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} ymm0 = ymm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm3[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero9712; AVX512DQ-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9713; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} ymm2 = ymm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm4[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero9714; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11]9715; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]9716; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm22, %zmm09717; AVX512DQ-FCP-NEXT: vmovdqa 64(%rsi), %ymm49718; AVX512DQ-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9719; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm4, %ymm29720; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm59721; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9722; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm5, %ymm39723; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11]9724; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm5[4],ymm4[4],ymm5[5],ymm4[5],ymm5[6],ymm4[6],ymm5[7],ymm4[7],ymm5[12],ymm4[12],ymm5[13],ymm4[13],ymm5[14],ymm4[14],ymm5[15],ymm4[15]9725; AVX512DQ-FCP-NEXT: vpermt2q %zmm3, %zmm23, %zmm29726; AVX512DQ-FCP-NEXT: vmovdqa32 %zmm0, %zmm2 {%k1}9727; AVX512DQ-FCP-NEXT: vmovdqa %ymm2, %ymm09728; AVX512DQ-FCP-NEXT: vmovdqa 64(%r8), %ymm49729; AVX512DQ-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9730; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm3 = ymm4[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]9731; AVX512DQ-FCP-NEXT: vpermt2d %ymm3, %ymm21, %ymm09732; AVX512DQ-FCP-NEXT: vpshufb %ymm8, %ymm4, %ymm39733; AVX512DQ-FCP-NEXT: vpermt2d %zmm3, %zmm24, %zmm29734; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm09735; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9736; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm0 = [2,2,2,3,10,9,10,11]9737; AVX512DQ-FCP-NEXT: vmovdqa 64(%r9), %ymm39738; AVX512DQ-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9739; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm3, %ymm29740; AVX512DQ-FCP-NEXT: vpshufb %ymm15, %ymm3, %ymm39741; AVX512DQ-FCP-NEXT: vpermt2q %zmm2, %zmm0, %zmm39742; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9743; AVX512DQ-FCP-NEXT: vmovdqa 96(%rcx), %xmm49744; AVX512DQ-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9745; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdx), %xmm59746; AVX512DQ-FCP-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9747; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} xmm2 = xmm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero9748; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} xmm3 = xmm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero9749; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]9750; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm30 = [1,0,2,2,1,0,2,2,16,17,16,17,16,17,16,17]9751; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]9752; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm30, %zmm39753; AVX512DQ-FCP-NEXT: vmovdqa 96(%rsi), %xmm69754; AVX512DQ-FCP-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9755; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7,8,9,6,7,4,5,10,11]9756; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm6, %xmm49757; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %xmm79758; AVX512DQ-FCP-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9759; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm7, %xmm59760; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]9761; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm29 = [8,8,0,9,0,1,0,1]9762; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]9763; AVX512DQ-FCP-NEXT: vpermt2q %zmm5, %zmm29, %zmm49764; AVX512DQ-FCP-NEXT: movw $9362, %ax # imm = 0x24929765; AVX512DQ-FCP-NEXT: kmovw %eax, %k29766; AVX512DQ-FCP-NEXT: vmovdqa32 %zmm3, %zmm4 {%k2}9767; AVX512DQ-FCP-NEXT: vmovdqa %ymm4, %ymm39768; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm28 = [16,9,10,17,12,13,18,15]9769; AVX512DQ-FCP-NEXT: vmovdqa 96(%r8), %xmm69770; AVX512DQ-FCP-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9771; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm5 = xmm6[2,1,3,3,4,5,6,7]9772; AVX512DQ-FCP-NEXT: vpermt2d %zmm5, %zmm28, %zmm49773; AVX512DQ-FCP-NEXT: vpmovzxwd {{.*#+}} xmm5 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero9774; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm27 = [0,1,8,3,4,9,6,7]9775; AVX512DQ-FCP-NEXT: vpermt2d %ymm5, %ymm27, %ymm39776; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm4, %zmm3, %zmm39777; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9778; AVX512DQ-FCP-NEXT: vmovdqa 64(%rcx), %xmm59779; AVX512DQ-FCP-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9780; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdx), %xmm69781; AVX512DQ-FCP-NEXT: vmovdqa %xmm6, (%rsp) # 16-byte Spill9782; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} xmm3 = xmm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero9783; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} xmm4 = xmm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero9784; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]9785; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]9786; AVX512DQ-FCP-NEXT: vpermt2d %zmm3, %zmm30, %zmm49787; AVX512DQ-FCP-NEXT: vmovdqa 64(%rsi), %xmm69788; AVX512DQ-FCP-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9789; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm6, %xmm39790; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %xmm79791; AVX512DQ-FCP-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9792; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm7, %xmm59793; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]9794; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]9795; AVX512DQ-FCP-NEXT: vpermt2q %zmm5, %zmm29, %zmm39796; AVX512DQ-FCP-NEXT: vmovdqa32 %zmm4, %zmm3 {%k2}9797; AVX512DQ-FCP-NEXT: vmovdqa %ymm3, %ymm49798; AVX512DQ-FCP-NEXT: vmovdqa 64(%r8), %xmm69799; AVX512DQ-FCP-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9800; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm5 = xmm6[2,1,3,3,4,5,6,7]9801; AVX512DQ-FCP-NEXT: vpermt2d %zmm5, %zmm28, %zmm39802; AVX512DQ-FCP-NEXT: vpmovzxwd {{.*#+}} xmm5 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero9803; AVX512DQ-FCP-NEXT: vpermt2d %ymm5, %ymm27, %ymm49804; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm39805; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9806; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm31 = [0,0,0,0,8,8,0,9]9807; AVX512DQ-FCP-NEXT: vmovdqa 64(%r9), %xmm49808; AVX512DQ-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill9809; AVX512DQ-FCP-NEXT: vpshufb %xmm10, %xmm4, %xmm39810; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,0,2,1,4,5,6,7]9811; AVX512DQ-FCP-NEXT: vpermt2q %zmm3, %zmm31, %zmm49812; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9813; AVX512DQ-FCP-NEXT: vmovdqa 32(%rcx), %ymm59814; AVX512DQ-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9815; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdx), %ymm69816; AVX512DQ-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9817; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} ymm3 = ymm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm5[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero9818; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} ymm4 = ymm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm6[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero9819; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]9820; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm6[4],ymm5[4],ymm6[5],ymm5[5],ymm6[6],ymm5[6],ymm6[7],ymm5[7],ymm6[12],ymm5[12],ymm6[13],ymm5[13],ymm6[14],ymm5[14],ymm6[15],ymm5[15]9821; AVX512DQ-FCP-NEXT: vpermt2d %zmm4, %zmm22, %zmm39822; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %ymm69823; AVX512DQ-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9824; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm6, %ymm49825; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm79826; AVX512DQ-FCP-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9827; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm7, %ymm59828; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm4 = ymm5[0],ymm4[0],ymm5[1],ymm4[1],ymm5[2],ymm4[2],ymm5[3],ymm4[3],ymm5[8],ymm4[8],ymm5[9],ymm4[9],ymm5[10],ymm4[10],ymm5[11],ymm4[11]9829; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm7[4],ymm6[4],ymm7[5],ymm6[5],ymm7[6],ymm6[6],ymm7[7],ymm6[7],ymm7[12],ymm6[12],ymm7[13],ymm6[13],ymm7[14],ymm6[14],ymm7[15],ymm6[15]9830; AVX512DQ-FCP-NEXT: vpermt2q %zmm5, %zmm23, %zmm49831; AVX512DQ-FCP-NEXT: vmovdqa32 %zmm3, %zmm4 {%k1}9832; AVX512DQ-FCP-NEXT: vmovdqa %ymm4, %ymm39833; AVX512DQ-FCP-NEXT: vmovdqa 32(%r8), %ymm69834; AVX512DQ-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9835; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm5 = ymm6[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]9836; AVX512DQ-FCP-NEXT: vpermt2d %ymm5, %ymm21, %ymm39837; AVX512DQ-FCP-NEXT: vpshufb %ymm8, %ymm6, %ymm59838; AVX512DQ-FCP-NEXT: vpermt2d %zmm5, %zmm24, %zmm49839; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm4, %zmm3, %zmm39840; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9841; AVX512DQ-FCP-NEXT: vmovdqa 32(%r9), %ymm49842; AVX512DQ-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill9843; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm4, %ymm39844; AVX512DQ-FCP-NEXT: vpshufb %ymm15, %ymm4, %ymm49845; AVX512DQ-FCP-NEXT: vpermt2q %zmm3, %zmm0, %zmm49846; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9847; AVX512DQ-FCP-NEXT: vmovdqa (%rcx), %ymm149848; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %ymm139849; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} ymm3 = ymm14[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm14[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero9850; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} ymm4 = ymm13[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,ymm13[22,23,24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero9851; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]9852; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm13[4],ymm14[4],ymm13[5],ymm14[5],ymm13[6],ymm14[6],ymm13[7],ymm14[7],ymm13[12],ymm14[12],ymm13[13],ymm14[13],ymm13[14],ymm14[14],ymm13[15],ymm14[15]9853; AVX512DQ-FCP-NEXT: vpermt2d %zmm4, %zmm22, %zmm39854; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %ymm109855; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm10, %ymm49856; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm79857; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm7, %ymm19858; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm4[0],ymm1[1],ymm4[1],ymm1[2],ymm4[2],ymm1[3],ymm4[3],ymm1[8],ymm4[8],ymm1[9],ymm4[9],ymm1[10],ymm4[10],ymm1[11],ymm4[11]9859; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm7[4],ymm10[4],ymm7[5],ymm10[5],ymm7[6],ymm10[6],ymm7[7],ymm10[7],ymm7[12],ymm10[12],ymm7[13],ymm10[13],ymm7[14],ymm10[14],ymm7[15],ymm10[15]9860; AVX512DQ-FCP-NEXT: vpermt2q %zmm4, %zmm23, %zmm19861; AVX512DQ-FCP-NEXT: vmovdqa32 %zmm3, %zmm1 {%k1}9862; AVX512DQ-FCP-NEXT: vmovdqa (%r8), %ymm129863; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm3 = ymm12[2,1,3,3,4,5,6,7,10,9,11,11,12,13,14,15]9864; AVX512DQ-FCP-NEXT: vpermi2d %ymm3, %ymm1, %ymm219865; AVX512DQ-FCP-NEXT: vpshufb %ymm8, %ymm12, %ymm39866; AVX512DQ-FCP-NEXT: vpermt2d %zmm3, %zmm24, %zmm19867; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm21, %zmm259868; AVX512DQ-FCP-NEXT: vmovdqa (%r9), %ymm119869; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm11, %ymm19870; AVX512DQ-FCP-NEXT: vpshufb %ymm15, %ymm11, %ymm39871; AVX512DQ-FCP-NEXT: vpermt2q %zmm1, %zmm0, %zmm39872; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9873; AVX512DQ-FCP-NEXT: vmovdqa 32(%rcx), %xmm39874; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdx), %xmm49875; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} xmm0 = xmm3[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero9876; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} xmm1 = xmm4[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero9877; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]9878; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]9879; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm4, %xmm219880; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm3, %xmm199881; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm30, %zmm19882; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %xmm49883; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %xmm59884; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm4, %xmm09885; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm5, %xmm39886; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]9887; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]9888; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm5, %xmm189889; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm4, %xmm179890; AVX512DQ-FCP-NEXT: vpermt2q %zmm3, %zmm29, %zmm09891; AVX512DQ-FCP-NEXT: vmovdqa32 %zmm1, %zmm0 {%k2}9892; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, %ymm19893; AVX512DQ-FCP-NEXT: vmovdqa 32(%r8), %xmm49894; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm4[2,1,3,3,4,5,6,7]9895; AVX512DQ-FCP-NEXT: vpermt2d %zmm3, %zmm28, %zmm09896; AVX512DQ-FCP-NEXT: vpmovzxwd {{.*#+}} xmm3 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero9897; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm4, %xmm169898; AVX512DQ-FCP-NEXT: vpermt2d %ymm3, %ymm27, %ymm19899; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm229900; AVX512DQ-FCP-NEXT: vmovdqa (%rcx), %xmm69901; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %xmm59902; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} xmm0 = xmm6[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero9903; AVX512DQ-FCP-NEXT: vpsrldq {{.*#+}} xmm1 = xmm5[6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero9904; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]9905; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]9906; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm30, %zmm39907; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %xmm49908; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm19909; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm4, %xmm89910; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm1, %xmm29911; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm8 = xmm2[4],xmm8[4],xmm2[5],xmm8[5],xmm2[6],xmm8[6],xmm2[7],xmm8[7]9912; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]9913; AVX512DQ-FCP-NEXT: vpermt2q %zmm2, %zmm29, %zmm89914; AVX512DQ-FCP-NEXT: vmovdqa32 %zmm3, %zmm8 {%k2}9915; AVX512DQ-FCP-NEXT: vmovdqa (%r8), %xmm29916; AVX512DQ-FCP-NEXT: vpmovzxwd {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero9917; AVX512DQ-FCP-NEXT: vpermi2d %ymm3, %ymm8, %ymm279918; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[2,1,3,3,4,5,6,7]9919; AVX512DQ-FCP-NEXT: vpermt2d %zmm3, %zmm28, %zmm89920; AVX512DQ-FCP-NEXT: vmovdqa 32(%r9), %xmm159921; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]9922; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm15, %xmm39923; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm9 = xmm15[0,0,2,1,4,5,6,7]9924; AVX512DQ-FCP-NEXT: vpermt2q %zmm3, %zmm31, %zmm99925; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm8, %zmm27, %zmm239926; AVX512DQ-FCP-NEXT: vmovdqa (%r9), %xmm39927; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm09928; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm8 = xmm3[0,0,2,1,4,5,6,7]9929; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm31, %zmm89930; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm7[0],ymm10[0],ymm7[1],ymm10[1],ymm7[2],ymm10[2],ymm7[3],ymm10[3],ymm7[8],ymm10[8],ymm7[9],ymm10[9],ymm7[10],ymm10[10],ymm7[11],ymm10[11]9931; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]9932; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm24 = [1,1,1,1,10,10,10,11]9933; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm24, %zmm19934; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]9935; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm4 = ymm13[0],ymm14[0],ymm13[1],ymm14[1],ymm13[2],ymm14[2],ymm13[3],ymm14[3],ymm13[8],ymm14[8],ymm13[9],ymm14[9],ymm13[10],ymm14[10],ymm13[11],ymm14[11]9936; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm30 = [17,18,17,18,0,0,19,19,5,4,2,2,5,4,6,6]9937; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm30, %zmm49938; AVX512DQ-FCP-NEXT: vmovdqa32 %zmm1, %zmm4 {%k1}9939; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm5 = [12,13,10,11,10,11,14,15,14,15,14,15,14,15,14,15]9940; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm2, %xmm09941; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm6 = [0,9,2,3,8,5,6,11]9942; AVX512DQ-FCP-NEXT: vmovdqa %ymm4, %ymm19943; AVX512DQ-FCP-NEXT: vpermt2d %ymm0, %ymm6, %ymm19944; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm0 = ymm12[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]9945; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm27 = [8,9,20,11,12,21,14,15]9946; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm27, %zmm49947; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm4, %zmm1, %zmm289948; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} ymm0 = ymm11[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]9949; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm4 = [8,9,12,13,12,13,10,11,8,9,10,11,12,13,14,15]9950; AVX512DQ-FCP-NEXT: vpshufb %xmm4, %xmm3, %xmm29951; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm1 = [0,0,0,1,0,10,10,0]9952; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm29953; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload9954; AVX512DQ-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload9955; AVX512DQ-FCP-NEXT: # ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11]9956; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload9957; AVX512DQ-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload9958; AVX512DQ-FCP-NEXT: # xmm3 = xmm3[4],mem[4],xmm3[5],mem[5],xmm3[6],mem[6],xmm3[7],mem[7]9959; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm24, %zmm39960; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload9961; AVX512DQ-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload9962; AVX512DQ-FCP-NEXT: # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]9963; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload9964; AVX512DQ-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload9965; AVX512DQ-FCP-NEXT: # ymm7 = ymm7[0],mem[0],ymm7[1],mem[1],ymm7[2],mem[2],ymm7[3],mem[3],ymm7[8],mem[8],ymm7[9],mem[9],ymm7[10],mem[10],ymm7[11],mem[11]9966; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm30, %zmm79967; AVX512DQ-FCP-NEXT: vmovdqa32 %zmm3, %zmm7 {%k1}9968; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload9969; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm0, %xmm09970; AVX512DQ-FCP-NEXT: vmovdqa %ymm7, %ymm39971; AVX512DQ-FCP-NEXT: vpermt2d %ymm0, %ymm6, %ymm39972; AVX512DQ-FCP-NEXT: vpshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload9973; AVX512DQ-FCP-NEXT: # ymm0 = mem[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]9974; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm27, %zmm79975; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm3, %zmm299976; AVX512DQ-FCP-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload9977; AVX512DQ-FCP-NEXT: # ymm0 = mem[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]9978; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload9979; AVX512DQ-FCP-NEXT: vpshufb %xmm4, %xmm3, %xmm79980; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm1, %zmm79981; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload9982; AVX512DQ-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload9983; AVX512DQ-FCP-NEXT: # ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11]9984; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload9985; AVX512DQ-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload9986; AVX512DQ-FCP-NEXT: # xmm3 = xmm3[4],mem[4],xmm3[5],mem[5],xmm3[6],mem[6],xmm3[7],mem[7]9987; AVX512DQ-FCP-NEXT: vpermt2q %zmm0, %zmm24, %zmm39988; AVX512DQ-FCP-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload9989; AVX512DQ-FCP-NEXT: vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload9990; AVX512DQ-FCP-NEXT: # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]9991; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload9992; AVX512DQ-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload9993; AVX512DQ-FCP-NEXT: # ymm10 = ymm10[0],mem[0],ymm10[1],mem[1],ymm10[2],mem[2],ymm10[3],mem[3],ymm10[8],mem[8],ymm10[9],mem[9],ymm10[10],mem[10],ymm10[11],mem[11]9994; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm30, %zmm109995; AVX512DQ-FCP-NEXT: vmovdqa32 %zmm3, %zmm10 {%k1}9996; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload9997; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm0, %xmm09998; AVX512DQ-FCP-NEXT: vmovdqa %ymm10, %ymm39999; AVX512DQ-FCP-NEXT: vpermt2d %ymm0, %ymm6, %ymm310000; AVX512DQ-FCP-NEXT: vpshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload10001; AVX512DQ-FCP-NEXT: # ymm0 = mem[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]10002; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm27, %zmm1010003; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm10, %zmm3, %zmm310004; AVX512DQ-FCP-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload10005; AVX512DQ-FCP-NEXT: # ymm10 = mem[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]10006; AVX512DQ-FCP-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload10007; AVX512DQ-FCP-NEXT: vpshufb %xmm4, %xmm0, %xmm010008; AVX512DQ-FCP-NEXT: vpermt2q %zmm10, %zmm1, %zmm010009; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload10010; AVX512DQ-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload10011; AVX512DQ-FCP-NEXT: # ymm10 = ymm10[0],mem[0],ymm10[1],mem[1],ymm10[2],mem[2],ymm10[3],mem[3],ymm10[8],mem[8],ymm10[9],mem[9],ymm10[10],mem[10],ymm10[11],mem[11]10012; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm18, %xmm1110013; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm17, %xmm1210014; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm11 = xmm11[4],xmm12[4],xmm11[5],xmm12[5],xmm11[6],xmm12[6],xmm11[7],xmm12[7]10015; AVX512DQ-FCP-NEXT: vpermt2q %zmm10, %zmm24, %zmm1110016; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm21, %xmm1010017; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm19, %xmm1210018; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm10 = xmm10[4],xmm12[4],xmm10[5],xmm12[5],xmm10[6],xmm12[6],xmm10[7],xmm12[7]10019; AVX512DQ-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload10020; AVX512DQ-FCP-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm12 # 32-byte Folded Reload10021; AVX512DQ-FCP-NEXT: # ymm12 = ymm12[0],mem[0],ymm12[1],mem[1],ymm12[2],mem[2],ymm12[3],mem[3],ymm12[8],mem[8],ymm12[9],mem[9],ymm12[10],mem[10],ymm12[11],mem[11]10022; AVX512DQ-FCP-NEXT: vpermt2d %zmm10, %zmm30, %zmm1210023; AVX512DQ-FCP-NEXT: vmovdqa32 %zmm11, %zmm12 {%k1}10024; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm16, %xmm1010025; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm10, %xmm510026; AVX512DQ-FCP-NEXT: vpermi2d %ymm5, %ymm12, %ymm610027; AVX512DQ-FCP-NEXT: vpshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload10028; AVX512DQ-FCP-NEXT: # ymm5 = mem[0,1,1,3,4,5,6,7,8,9,9,11,12,13,14,15]10029; AVX512DQ-FCP-NEXT: vpermt2d %zmm5, %zmm27, %zmm1210030; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm12, %zmm6, %zmm510031; AVX512DQ-FCP-NEXT: vpshufb %xmm4, %xmm15, %xmm410032; AVX512DQ-FCP-NEXT: vpshuflw $96, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload10033; AVX512DQ-FCP-NEXT: # ymm6 = mem[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]10034; AVX512DQ-FCP-NEXT: vpermt2q %zmm6, %zmm1, %zmm410035; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]10036; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm1 & (zmm4 ^ zmm5))10037; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax10038; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm4, 256(%rax)10039; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm1 & (zmm0 ^ zmm3))10040; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, 448(%rax)10041; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm1 & (zmm7 ^ zmm29))10042; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm7, 640(%rax)10043; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm1 & (zmm2 ^ zmm28))10044; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, 64(%rax)10045; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]10046; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm8 = zmm8 ^ (zmm0 & (zmm8 ^ zmm23))10047; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm8, (%rax)10048; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm9 = zmm9 ^ (zmm0 & (zmm9 ^ zmm22))10049; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm9, 192(%rax)10050; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm1 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]10051; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload10052; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm1 & (zmm2 ^ zmm25))10053; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, 128(%rax)10054; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload10055; AVX512DQ-FCP-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm2 # 64-byte Folded Reload10056; AVX512DQ-FCP-NEXT: # zmm2 = zmm2 ^ (zmm1 & (zmm2 ^ mem))10057; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, 320(%rax)10058; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload10059; AVX512DQ-FCP-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload10060; AVX512DQ-FCP-NEXT: # zmm2 = zmm2 ^ (zmm0 & (zmm2 ^ mem))10061; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, 384(%rax)10062; AVX512DQ-FCP-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload10063; AVX512DQ-FCP-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm2 # 64-byte Folded Reload10064; AVX512DQ-FCP-NEXT: # zmm2 = zmm2 ^ (zmm1 & (zmm2 ^ mem))10065; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, 512(%rax)10066; AVX512DQ-FCP-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm20 # 64-byte Folded Reload10067; AVX512DQ-FCP-NEXT: # zmm20 = zmm20 ^ (zmm1 & (zmm20 ^ mem))10068; AVX512DQ-FCP-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm26 # 64-byte Folded Reload10069; AVX512DQ-FCP-NEXT: # zmm26 = zmm26 ^ (zmm0 & (zmm26 ^ mem))10070; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm26, 576(%rax)10071; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm20, 704(%rax)10072; AVX512DQ-FCP-NEXT: addq $1224, %rsp # imm = 0x4C810073; AVX512DQ-FCP-NEXT: vzeroupper10074; AVX512DQ-FCP-NEXT: retq10075;10076; AVX512BW-LABEL: store_i16_stride6_vf64:10077; AVX512BW: # %bb.0:10078; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm310079; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm010080; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm1610081; AVX512BW-NEXT: vmovdqa64 64(%rsi), %zmm2410082; AVX512BW-NEXT: vmovdqa64 (%rdx), %zmm110083; AVX512BW-NEXT: vmovdqa64 64(%rdx), %zmm710084; AVX512BW-NEXT: vmovdqa64 (%rcx), %zmm1210085; AVX512BW-NEXT: vmovdqa64 64(%rcx), %zmm1510086; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm19 = [0,0,0,32,3,35,0,0,1,33,4,36,0,0,2,34,0,0,0,32,3,35,0,0,1,33,4,36,0,0,2,34]10087; AVX512BW-NEXT: # zmm19 = mem[0,1,2,3,0,1,2,3]10088; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm1310089; AVX512BW-NEXT: vpermt2w %zmm12, %zmm19, %zmm1310090; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm26 = [0,32,3,35,0,0,1,33,4,36,0,0,2,34,5,37,0,32,3,35,0,0,1,33,4,36,0,0,2,34,5,37]10091; AVX512BW-NEXT: # zmm26 = mem[0,1,2,3,0,1,2,3]10092; AVX512BW-NEXT: vmovdqa64 %zmm3, %zmm210093; AVX512BW-NEXT: vpermt2w %zmm16, %zmm26, %zmm210094; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm11 = [29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0]10095; AVX512BW-NEXT: # zmm11 = mem[0,1,2,3,0,1,2,3]10096; AVX512BW-NEXT: vmovdqa64 %zmm7, %zmm2010097; AVX512BW-NEXT: vpermt2w %zmm15, %zmm11, %zmm2010098; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm5 = [0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61]10099; AVX512BW-NEXT: # zmm5 = mem[0,1,2,3,0,1,2,3]10100; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm410101; AVX512BW-NEXT: vpermt2w %zmm24, %zmm5, %zmm410102; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm18 = [24,56,0,0,22,54,25,57,0,0,23,55,26,58,0,0,24,56,0,0,22,54,25,57,0,0,23,55,26,58,0,0]10103; AVX512BW-NEXT: # zmm18 = mem[0,1,2,3,0,1,2,3]10104; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm2210105; AVX512BW-NEXT: vpermt2w %zmm24, %zmm18, %zmm2210106; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm8 = [21,53,24,56,0,0,22,54,25,57,0,0,23,55,26,58,21,53,24,56,0,0,22,54,25,57,0,0,23,55,26,58]10107; AVX512BW-NEXT: # zmm8 = mem[0,1,2,3,0,1,2,3]10108; AVX512BW-NEXT: vmovdqa64 %zmm7, %zmm610109; AVX512BW-NEXT: vpermt2w %zmm15, %zmm8, %zmm610110; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm21 = [0,0,16,48,19,51,0,0,17,49,20,52,0,0,18,50,0,0,16,48,19,51,0,0,17,49,20,52,0,0,18,50]10111; AVX512BW-NEXT: # zmm21 = mem[0,1,2,3,0,1,2,3]10112; AVX512BW-NEXT: vmovdqa64 %zmm7, %zmm2310113; AVX512BW-NEXT: vpermt2w %zmm15, %zmm21, %zmm2310114; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm10 = [16,48,19,51,0,0,17,49,20,52,0,0,18,50,21,53,16,48,19,51,0,0,17,49,20,52,0,0,18,50,21,53]10115; AVX512BW-NEXT: # zmm10 = mem[0,1,2,3,0,1,2,3]10116; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm910117; AVX512BW-NEXT: vpermt2w %zmm24, %zmm10, %zmm910118; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm17 = [0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45]10119; AVX512BW-NEXT: # zmm17 = mem[0,1,2,3,0,1,2,3]10120; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm1410121; AVX512BW-NEXT: vpermt2w %zmm24, %zmm17, %zmm1410122; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm2510123; AVX512BW-NEXT: vpermt2w %zmm24, %zmm26, %zmm010124; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm26 = [8,40,0,0,6,38,9,41,0,0,7,39,10,42,0,0,8,40,0,0,6,38,9,41,0,0,7,39,10,42,0,0]10125; AVX512BW-NEXT: # zmm26 = mem[0,1,2,3,0,1,2,3]10126; AVX512BW-NEXT: vpermt2w %zmm24, %zmm26, %zmm2510127; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm24 = [13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0]10128; AVX512BW-NEXT: # zmm24 = mem[0,1,2,3,0,1,2,3]10129; AVX512BW-NEXT: vpermi2w %zmm16, %zmm3, %zmm510130; AVX512BW-NEXT: vpermi2w %zmm16, %zmm3, %zmm1810131; AVX512BW-NEXT: vpermi2w %zmm16, %zmm3, %zmm1010132; AVX512BW-NEXT: vpermi2w %zmm16, %zmm3, %zmm1710133; AVX512BW-NEXT: vpermt2w %zmm16, %zmm26, %zmm310134; AVX512BW-NEXT: vmovdqa64 %zmm7, %zmm2610135; AVX512BW-NEXT: vpermt2w %zmm15, %zmm24, %zmm2610136; AVX512BW-NEXT: vmovdqa64 %zmm7, %zmm1610137; AVX512BW-NEXT: vpermt2w %zmm15, %zmm19, %zmm710138; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm19 = [5,37,8,40,0,0,6,38,9,41,0,0,7,39,10,42,5,37,8,40,0,0,6,38,9,41,0,0,7,39,10,42]10139; AVX512BW-NEXT: # zmm19 = mem[0,1,2,3,0,1,2,3]10140; AVX512BW-NEXT: vpermt2w %zmm15, %zmm19, %zmm1610141; AVX512BW-NEXT: vpermi2w %zmm12, %zmm1, %zmm1110142; AVX512BW-NEXT: vpermi2w %zmm12, %zmm1, %zmm810143; AVX512BW-NEXT: vpermi2w %zmm12, %zmm1, %zmm2110144; AVX512BW-NEXT: vpermi2w %zmm12, %zmm1, %zmm2410145; AVX512BW-NEXT: vpermt2w %zmm12, %zmm19, %zmm110146; AVX512BW-NEXT: movw $9362, %ax # imm = 0x249210147; AVX512BW-NEXT: kmovd %eax, %k210148; AVX512BW-NEXT: vmovdqa32 %zmm13, %zmm2 {%k2}10149; AVX512BW-NEXT: vmovdqa64 (%r8), %zmm1210150; AVX512BW-NEXT: movw $18724, %ax # imm = 0x492410151; AVX512BW-NEXT: kmovd %eax, %k110152; AVX512BW-NEXT: vmovdqa32 %zmm20, %zmm4 {%k1}10153; AVX512BW-NEXT: vmovdqa64 64(%r8), %zmm1310154; AVX512BW-NEXT: vmovdqa32 %zmm22, %zmm6 {%k1}10155; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm15 = [0,1,2,3,32,0,6,7,8,9,33,0,12,13,14,15,34,0,18,19,20,21,35,0,24,25,26,27,36,0,30,31]10156; AVX512BW-NEXT: vpermt2w %zmm12, %zmm15, %zmm210157; AVX512BW-NEXT: vmovdqa32 %zmm23, %zmm9 {%k2}10158; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm19 = [58,0,2,3,4,5,59,0,8,9,10,11,60,0,14,15,16,17,61,0,20,21,22,23,62,0,26,27,28,29,63,0]10159; AVX512BW-NEXT: vpermt2w %zmm13, %zmm19, %zmm410160; AVX512BW-NEXT: vmovdqa32 %zmm26, %zmm14 {%k1}10161; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm20 = [0,1,53,0,4,5,6,7,54,0,10,11,12,13,55,0,16,17,18,19,56,0,22,23,24,25,57,0,28,29,30,31]10162; AVX512BW-NEXT: vpermt2w %zmm13, %zmm20, %zmm610163; AVX512BW-NEXT: vmovdqa32 %zmm25, %zmm16 {%k1}10164; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm22 = [0,1,2,3,48,0,6,7,8,9,49,0,12,13,14,15,50,0,18,19,20,21,51,0,24,25,26,27,52,0,30,31]10165; AVX512BW-NEXT: vpermt2w %zmm13, %zmm22, %zmm910166; AVX512BW-NEXT: vmovdqa32 %zmm7, %zmm0 {%k2}10167; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm7 = [42,0,2,3,4,5,43,0,8,9,10,11,44,0,14,15,16,17,45,0,20,21,22,23,46,0,26,27,28,29,47,0]10168; AVX512BW-NEXT: vpermt2w %zmm13, %zmm7, %zmm1410169; AVX512BW-NEXT: vpermt2w %zmm13, %zmm15, %zmm010170; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm15 = [0,1,37,0,4,5,6,7,38,0,10,11,12,13,39,0,16,17,18,19,40,0,22,23,24,25,41,0,28,29,30,31]10171; AVX512BW-NEXT: vpermt2w %zmm13, %zmm15, %zmm1610172; AVX512BW-NEXT: vmovdqa32 %zmm11, %zmm5 {%k1}10173; AVX512BW-NEXT: vpermt2w %zmm12, %zmm19, %zmm510174; AVX512BW-NEXT: vmovdqa32 %zmm18, %zmm8 {%k1}10175; AVX512BW-NEXT: vpermt2w %zmm12, %zmm20, %zmm810176; AVX512BW-NEXT: vmovdqa32 %zmm21, %zmm10 {%k2}10177; AVX512BW-NEXT: vmovdqa64 (%r9), %zmm1110178; AVX512BW-NEXT: vpermt2w %zmm12, %zmm22, %zmm1010179; AVX512BW-NEXT: vmovdqa64 64(%r9), %zmm1310180; AVX512BW-NEXT: vmovdqa32 %zmm24, %zmm17 {%k1}10181; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm18 = [0,1,2,3,4,32,6,7,8,9,10,33,12,13,14,15,16,34,18,19,20,21,22,35,24,25,26,27,28,36,30,31]10182; AVX512BW-NEXT: vpermt2w %zmm11, %zmm18, %zmm210183; AVX512BW-NEXT: vpermt2w %zmm12, %zmm7, %zmm1710184; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm7 = [0,58,2,3,4,5,6,59,8,9,10,11,12,60,14,15,16,17,18,61,20,21,22,23,24,62,26,27,28,29,30,63]10185; AVX512BW-NEXT: vpermt2w %zmm13, %zmm7, %zmm410186; AVX512BW-NEXT: vmovdqa32 %zmm3, %zmm1 {%k1}10187; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,1,2,53,4,5,6,7,8,54,10,11,12,13,14,55,16,17,18,19,20,56,22,23,24,25,26,57,28,29,30,31]10188; AVX512BW-NEXT: vpermt2w %zmm13, %zmm3, %zmm610189; AVX512BW-NEXT: vpermt2w %zmm12, %zmm15, %zmm110190; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm12 = [0,1,2,3,4,48,6,7,8,9,10,49,12,13,14,15,16,50,18,19,20,21,22,51,24,25,26,27,28,52,30,31]10191; AVX512BW-NEXT: vpermt2w %zmm13, %zmm12, %zmm910192; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm15 = [0,42,2,3,4,5,6,43,8,9,10,11,12,44,14,15,16,17,18,45,20,21,22,23,24,46,26,27,28,29,30,47]10193; AVX512BW-NEXT: vpermt2w %zmm13, %zmm15, %zmm1410194; AVX512BW-NEXT: vpermt2w %zmm13, %zmm18, %zmm010195; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm18 = [0,1,2,37,4,5,6,7,8,38,10,11,12,13,14,39,16,17,18,19,20,40,22,23,24,25,26,41,28,29,30,31]10196; AVX512BW-NEXT: vpermt2w %zmm13, %zmm18, %zmm1610197; AVX512BW-NEXT: vpermt2w %zmm11, %zmm7, %zmm510198; AVX512BW-NEXT: vpermt2w %zmm11, %zmm3, %zmm810199; AVX512BW-NEXT: vpermt2w %zmm11, %zmm12, %zmm1010200; AVX512BW-NEXT: vpermt2w %zmm11, %zmm15, %zmm1710201; AVX512BW-NEXT: vpermt2w %zmm11, %zmm18, %zmm110202; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rax10203; AVX512BW-NEXT: vmovdqa64 %zmm1, 64(%rax)10204; AVX512BW-NEXT: vmovdqa64 %zmm17, 128(%rax)10205; AVX512BW-NEXT: vmovdqa64 %zmm10, 192(%rax)10206; AVX512BW-NEXT: vmovdqa64 %zmm8, 256(%rax)10207; AVX512BW-NEXT: vmovdqa64 %zmm5, 320(%rax)10208; AVX512BW-NEXT: vmovdqa64 %zmm0, 384(%rax)10209; AVX512BW-NEXT: vmovdqa64 %zmm16, 448(%rax)10210; AVX512BW-NEXT: vmovdqa64 %zmm14, 512(%rax)10211; AVX512BW-NEXT: vmovdqa64 %zmm9, 576(%rax)10212; AVX512BW-NEXT: vmovdqa64 %zmm6, 640(%rax)10213; AVX512BW-NEXT: vmovdqa64 %zmm4, 704(%rax)10214; AVX512BW-NEXT: vmovdqa64 %zmm2, (%rax)10215; AVX512BW-NEXT: vzeroupper10216; AVX512BW-NEXT: retq10217;10218; AVX512BW-FCP-LABEL: store_i16_stride6_vf64:10219; AVX512BW-FCP: # %bb.0:10220; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm310221; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm010222; AVX512BW-FCP-NEXT: vmovdqa64 (%rsi), %zmm1610223; AVX512BW-FCP-NEXT: vmovdqa64 64(%rsi), %zmm2410224; AVX512BW-FCP-NEXT: vmovdqa64 (%rdx), %zmm110225; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdx), %zmm710226; AVX512BW-FCP-NEXT: vmovdqa64 (%rcx), %zmm1210227; AVX512BW-FCP-NEXT: vmovdqa64 64(%rcx), %zmm1510228; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm19 = [0,0,0,32,3,35,0,0,1,33,4,36,0,0,2,34,0,0,0,32,3,35,0,0,1,33,4,36,0,0,2,34]10229; AVX512BW-FCP-NEXT: # zmm19 = mem[0,1,2,3,0,1,2,3]10230; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm1310231; AVX512BW-FCP-NEXT: vpermt2w %zmm12, %zmm19, %zmm1310232; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm26 = [0,32,3,35,0,0,1,33,4,36,0,0,2,34,5,37,0,32,3,35,0,0,1,33,4,36,0,0,2,34,5,37]10233; AVX512BW-FCP-NEXT: # zmm26 = mem[0,1,2,3,0,1,2,3]10234; AVX512BW-FCP-NEXT: vmovdqa64 %zmm3, %zmm210235; AVX512BW-FCP-NEXT: vpermt2w %zmm16, %zmm26, %zmm210236; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm11 = [29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0]10237; AVX512BW-FCP-NEXT: # zmm11 = mem[0,1,2,3,0,1,2,3]10238; AVX512BW-FCP-NEXT: vmovdqa64 %zmm7, %zmm2010239; AVX512BW-FCP-NEXT: vpermt2w %zmm15, %zmm11, %zmm2010240; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm5 = [0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61]10241; AVX512BW-FCP-NEXT: # zmm5 = mem[0,1,2,3,0,1,2,3]10242; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm410243; AVX512BW-FCP-NEXT: vpermt2w %zmm24, %zmm5, %zmm410244; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm18 = [24,56,0,0,22,54,25,57,0,0,23,55,26,58,0,0,24,56,0,0,22,54,25,57,0,0,23,55,26,58,0,0]10245; AVX512BW-FCP-NEXT: # zmm18 = mem[0,1,2,3,0,1,2,3]10246; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm2210247; AVX512BW-FCP-NEXT: vpermt2w %zmm24, %zmm18, %zmm2210248; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm8 = [21,53,24,56,0,0,22,54,25,57,0,0,23,55,26,58,21,53,24,56,0,0,22,54,25,57,0,0,23,55,26,58]10249; AVX512BW-FCP-NEXT: # zmm8 = mem[0,1,2,3,0,1,2,3]10250; AVX512BW-FCP-NEXT: vmovdqa64 %zmm7, %zmm610251; AVX512BW-FCP-NEXT: vpermt2w %zmm15, %zmm8, %zmm610252; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm21 = [0,0,16,48,19,51,0,0,17,49,20,52,0,0,18,50,0,0,16,48,19,51,0,0,17,49,20,52,0,0,18,50]10253; AVX512BW-FCP-NEXT: # zmm21 = mem[0,1,2,3,0,1,2,3]10254; AVX512BW-FCP-NEXT: vmovdqa64 %zmm7, %zmm2310255; AVX512BW-FCP-NEXT: vpermt2w %zmm15, %zmm21, %zmm2310256; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm10 = [16,48,19,51,0,0,17,49,20,52,0,0,18,50,21,53,16,48,19,51,0,0,17,49,20,52,0,0,18,50,21,53]10257; AVX512BW-FCP-NEXT: # zmm10 = mem[0,1,2,3,0,1,2,3]10258; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm910259; AVX512BW-FCP-NEXT: vpermt2w %zmm24, %zmm10, %zmm910260; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm17 = [0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45]10261; AVX512BW-FCP-NEXT: # zmm17 = mem[0,1,2,3,0,1,2,3]10262; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm1410263; AVX512BW-FCP-NEXT: vpermt2w %zmm24, %zmm17, %zmm1410264; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm2510265; AVX512BW-FCP-NEXT: vpermt2w %zmm24, %zmm26, %zmm010266; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm26 = [8,40,0,0,6,38,9,41,0,0,7,39,10,42,0,0,8,40,0,0,6,38,9,41,0,0,7,39,10,42,0,0]10267; AVX512BW-FCP-NEXT: # zmm26 = mem[0,1,2,3,0,1,2,3]10268; AVX512BW-FCP-NEXT: vpermt2w %zmm24, %zmm26, %zmm2510269; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm24 = [13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0]10270; AVX512BW-FCP-NEXT: # zmm24 = mem[0,1,2,3,0,1,2,3]10271; AVX512BW-FCP-NEXT: vpermi2w %zmm16, %zmm3, %zmm510272; AVX512BW-FCP-NEXT: vpermi2w %zmm16, %zmm3, %zmm1810273; AVX512BW-FCP-NEXT: vpermi2w %zmm16, %zmm3, %zmm1010274; AVX512BW-FCP-NEXT: vpermi2w %zmm16, %zmm3, %zmm1710275; AVX512BW-FCP-NEXT: vpermt2w %zmm16, %zmm26, %zmm310276; AVX512BW-FCP-NEXT: vmovdqa64 %zmm7, %zmm2610277; AVX512BW-FCP-NEXT: vpermt2w %zmm15, %zmm24, %zmm2610278; AVX512BW-FCP-NEXT: vmovdqa64 %zmm7, %zmm1610279; AVX512BW-FCP-NEXT: vpermt2w %zmm15, %zmm19, %zmm710280; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm19 = [5,37,8,40,0,0,6,38,9,41,0,0,7,39,10,42,5,37,8,40,0,0,6,38,9,41,0,0,7,39,10,42]10281; AVX512BW-FCP-NEXT: # zmm19 = mem[0,1,2,3,0,1,2,3]10282; AVX512BW-FCP-NEXT: vpermt2w %zmm15, %zmm19, %zmm1610283; AVX512BW-FCP-NEXT: vpermi2w %zmm12, %zmm1, %zmm1110284; AVX512BW-FCP-NEXT: vpermi2w %zmm12, %zmm1, %zmm810285; AVX512BW-FCP-NEXT: vpermi2w %zmm12, %zmm1, %zmm2110286; AVX512BW-FCP-NEXT: vpermi2w %zmm12, %zmm1, %zmm2410287; AVX512BW-FCP-NEXT: vpermt2w %zmm12, %zmm19, %zmm110288; AVX512BW-FCP-NEXT: movw $9362, %ax # imm = 0x249210289; AVX512BW-FCP-NEXT: kmovd %eax, %k210290; AVX512BW-FCP-NEXT: vmovdqa32 %zmm13, %zmm2 {%k2}10291; AVX512BW-FCP-NEXT: vmovdqa64 (%r8), %zmm1210292; AVX512BW-FCP-NEXT: movw $18724, %ax # imm = 0x492410293; AVX512BW-FCP-NEXT: kmovd %eax, %k110294; AVX512BW-FCP-NEXT: vmovdqa32 %zmm20, %zmm4 {%k1}10295; AVX512BW-FCP-NEXT: vmovdqa64 64(%r8), %zmm1310296; AVX512BW-FCP-NEXT: vmovdqa32 %zmm22, %zmm6 {%k1}10297; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm15 = [0,1,2,3,32,0,6,7,8,9,33,0,12,13,14,15,34,0,18,19,20,21,35,0,24,25,26,27,36,0,30,31]10298; AVX512BW-FCP-NEXT: vpermt2w %zmm12, %zmm15, %zmm210299; AVX512BW-FCP-NEXT: vmovdqa32 %zmm23, %zmm9 {%k2}10300; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm19 = [58,0,2,3,4,5,59,0,8,9,10,11,60,0,14,15,16,17,61,0,20,21,22,23,62,0,26,27,28,29,63,0]10301; AVX512BW-FCP-NEXT: vpermt2w %zmm13, %zmm19, %zmm410302; AVX512BW-FCP-NEXT: vmovdqa32 %zmm26, %zmm14 {%k1}10303; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm20 = [0,1,53,0,4,5,6,7,54,0,10,11,12,13,55,0,16,17,18,19,56,0,22,23,24,25,57,0,28,29,30,31]10304; AVX512BW-FCP-NEXT: vpermt2w %zmm13, %zmm20, %zmm610305; AVX512BW-FCP-NEXT: vmovdqa32 %zmm25, %zmm16 {%k1}10306; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm22 = [0,1,2,3,48,0,6,7,8,9,49,0,12,13,14,15,50,0,18,19,20,21,51,0,24,25,26,27,52,0,30,31]10307; AVX512BW-FCP-NEXT: vpermt2w %zmm13, %zmm22, %zmm910308; AVX512BW-FCP-NEXT: vmovdqa32 %zmm7, %zmm0 {%k2}10309; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm7 = [42,0,2,3,4,5,43,0,8,9,10,11,44,0,14,15,16,17,45,0,20,21,22,23,46,0,26,27,28,29,47,0]10310; AVX512BW-FCP-NEXT: vpermt2w %zmm13, %zmm7, %zmm1410311; AVX512BW-FCP-NEXT: vpermt2w %zmm13, %zmm15, %zmm010312; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm15 = [0,1,37,0,4,5,6,7,38,0,10,11,12,13,39,0,16,17,18,19,40,0,22,23,24,25,41,0,28,29,30,31]10313; AVX512BW-FCP-NEXT: vpermt2w %zmm13, %zmm15, %zmm1610314; AVX512BW-FCP-NEXT: vmovdqa32 %zmm11, %zmm5 {%k1}10315; AVX512BW-FCP-NEXT: vpermt2w %zmm12, %zmm19, %zmm510316; AVX512BW-FCP-NEXT: vmovdqa32 %zmm18, %zmm8 {%k1}10317; AVX512BW-FCP-NEXT: vpermt2w %zmm12, %zmm20, %zmm810318; AVX512BW-FCP-NEXT: vmovdqa32 %zmm21, %zmm10 {%k2}10319; AVX512BW-FCP-NEXT: vmovdqa64 (%r9), %zmm1110320; AVX512BW-FCP-NEXT: vpermt2w %zmm12, %zmm22, %zmm1010321; AVX512BW-FCP-NEXT: vmovdqa64 64(%r9), %zmm1310322; AVX512BW-FCP-NEXT: vmovdqa32 %zmm24, %zmm17 {%k1}10323; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm18 = [0,1,2,3,4,32,6,7,8,9,10,33,12,13,14,15,16,34,18,19,20,21,22,35,24,25,26,27,28,36,30,31]10324; AVX512BW-FCP-NEXT: vpermt2w %zmm11, %zmm18, %zmm210325; AVX512BW-FCP-NEXT: vpermt2w %zmm12, %zmm7, %zmm1710326; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm7 = [0,58,2,3,4,5,6,59,8,9,10,11,12,60,14,15,16,17,18,61,20,21,22,23,24,62,26,27,28,29,30,63]10327; AVX512BW-FCP-NEXT: vpermt2w %zmm13, %zmm7, %zmm410328; AVX512BW-FCP-NEXT: vmovdqa32 %zmm3, %zmm1 {%k1}10329; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,1,2,53,4,5,6,7,8,54,10,11,12,13,14,55,16,17,18,19,20,56,22,23,24,25,26,57,28,29,30,31]10330; AVX512BW-FCP-NEXT: vpermt2w %zmm13, %zmm3, %zmm610331; AVX512BW-FCP-NEXT: vpermt2w %zmm12, %zmm15, %zmm110332; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm12 = [0,1,2,3,4,48,6,7,8,9,10,49,12,13,14,15,16,50,18,19,20,21,22,51,24,25,26,27,28,52,30,31]10333; AVX512BW-FCP-NEXT: vpermt2w %zmm13, %zmm12, %zmm910334; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm15 = [0,42,2,3,4,5,6,43,8,9,10,11,12,44,14,15,16,17,18,45,20,21,22,23,24,46,26,27,28,29,30,47]10335; AVX512BW-FCP-NEXT: vpermt2w %zmm13, %zmm15, %zmm1410336; AVX512BW-FCP-NEXT: vpermt2w %zmm13, %zmm18, %zmm010337; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm18 = [0,1,2,37,4,5,6,7,8,38,10,11,12,13,14,39,16,17,18,19,20,40,22,23,24,25,26,41,28,29,30,31]10338; AVX512BW-FCP-NEXT: vpermt2w %zmm13, %zmm18, %zmm1610339; AVX512BW-FCP-NEXT: vpermt2w %zmm11, %zmm7, %zmm510340; AVX512BW-FCP-NEXT: vpermt2w %zmm11, %zmm3, %zmm810341; AVX512BW-FCP-NEXT: vpermt2w %zmm11, %zmm12, %zmm1010342; AVX512BW-FCP-NEXT: vpermt2w %zmm11, %zmm15, %zmm1710343; AVX512BW-FCP-NEXT: vpermt2w %zmm11, %zmm18, %zmm110344; AVX512BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax10345; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, 64(%rax)10346; AVX512BW-FCP-NEXT: vmovdqa64 %zmm17, 128(%rax)10347; AVX512BW-FCP-NEXT: vmovdqa64 %zmm10, 192(%rax)10348; AVX512BW-FCP-NEXT: vmovdqa64 %zmm8, 256(%rax)10349; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, 320(%rax)10350; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, 384(%rax)10351; AVX512BW-FCP-NEXT: vmovdqa64 %zmm16, 448(%rax)10352; AVX512BW-FCP-NEXT: vmovdqa64 %zmm14, 512(%rax)10353; AVX512BW-FCP-NEXT: vmovdqa64 %zmm9, 576(%rax)10354; AVX512BW-FCP-NEXT: vmovdqa64 %zmm6, 640(%rax)10355; AVX512BW-FCP-NEXT: vmovdqa64 %zmm4, 704(%rax)10356; AVX512BW-FCP-NEXT: vmovdqa64 %zmm2, (%rax)10357; AVX512BW-FCP-NEXT: vzeroupper10358; AVX512BW-FCP-NEXT: retq10359;10360; AVX512DQ-BW-LABEL: store_i16_stride6_vf64:10361; AVX512DQ-BW: # %bb.0:10362; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm310363; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %zmm010364; AVX512DQ-BW-NEXT: vmovdqa64 (%rsi), %zmm1610365; AVX512DQ-BW-NEXT: vmovdqa64 64(%rsi), %zmm2410366; AVX512DQ-BW-NEXT: vmovdqa64 (%rdx), %zmm110367; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdx), %zmm710368; AVX512DQ-BW-NEXT: vmovdqa64 (%rcx), %zmm1210369; AVX512DQ-BW-NEXT: vmovdqa64 64(%rcx), %zmm1510370; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm19 = [0,0,0,32,3,35,0,0,1,33,4,36,0,0,2,34,0,0,0,32,3,35,0,0,1,33,4,36,0,0,2,34]10371; AVX512DQ-BW-NEXT: # zmm19 = mem[0,1,2,3,0,1,2,3]10372; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, %zmm1310373; AVX512DQ-BW-NEXT: vpermt2w %zmm12, %zmm19, %zmm1310374; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm26 = [0,32,3,35,0,0,1,33,4,36,0,0,2,34,5,37,0,32,3,35,0,0,1,33,4,36,0,0,2,34,5,37]10375; AVX512DQ-BW-NEXT: # zmm26 = mem[0,1,2,3,0,1,2,3]10376; AVX512DQ-BW-NEXT: vmovdqa64 %zmm3, %zmm210377; AVX512DQ-BW-NEXT: vpermt2w %zmm16, %zmm26, %zmm210378; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm11 = [29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0]10379; AVX512DQ-BW-NEXT: # zmm11 = mem[0,1,2,3,0,1,2,3]10380; AVX512DQ-BW-NEXT: vmovdqa64 %zmm7, %zmm2010381; AVX512DQ-BW-NEXT: vpermt2w %zmm15, %zmm11, %zmm2010382; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm5 = [0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61]10383; AVX512DQ-BW-NEXT: # zmm5 = mem[0,1,2,3,0,1,2,3]10384; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm410385; AVX512DQ-BW-NEXT: vpermt2w %zmm24, %zmm5, %zmm410386; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm18 = [24,56,0,0,22,54,25,57,0,0,23,55,26,58,0,0,24,56,0,0,22,54,25,57,0,0,23,55,26,58,0,0]10387; AVX512DQ-BW-NEXT: # zmm18 = mem[0,1,2,3,0,1,2,3]10388; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm2210389; AVX512DQ-BW-NEXT: vpermt2w %zmm24, %zmm18, %zmm2210390; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm8 = [21,53,24,56,0,0,22,54,25,57,0,0,23,55,26,58,21,53,24,56,0,0,22,54,25,57,0,0,23,55,26,58]10391; AVX512DQ-BW-NEXT: # zmm8 = mem[0,1,2,3,0,1,2,3]10392; AVX512DQ-BW-NEXT: vmovdqa64 %zmm7, %zmm610393; AVX512DQ-BW-NEXT: vpermt2w %zmm15, %zmm8, %zmm610394; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm21 = [0,0,16,48,19,51,0,0,17,49,20,52,0,0,18,50,0,0,16,48,19,51,0,0,17,49,20,52,0,0,18,50]10395; AVX512DQ-BW-NEXT: # zmm21 = mem[0,1,2,3,0,1,2,3]10396; AVX512DQ-BW-NEXT: vmovdqa64 %zmm7, %zmm2310397; AVX512DQ-BW-NEXT: vpermt2w %zmm15, %zmm21, %zmm2310398; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm10 = [16,48,19,51,0,0,17,49,20,52,0,0,18,50,21,53,16,48,19,51,0,0,17,49,20,52,0,0,18,50,21,53]10399; AVX512DQ-BW-NEXT: # zmm10 = mem[0,1,2,3,0,1,2,3]10400; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm910401; AVX512DQ-BW-NEXT: vpermt2w %zmm24, %zmm10, %zmm910402; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm17 = [0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45]10403; AVX512DQ-BW-NEXT: # zmm17 = mem[0,1,2,3,0,1,2,3]10404; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm1410405; AVX512DQ-BW-NEXT: vpermt2w %zmm24, %zmm17, %zmm1410406; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm2510407; AVX512DQ-BW-NEXT: vpermt2w %zmm24, %zmm26, %zmm010408; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm26 = [8,40,0,0,6,38,9,41,0,0,7,39,10,42,0,0,8,40,0,0,6,38,9,41,0,0,7,39,10,42,0,0]10409; AVX512DQ-BW-NEXT: # zmm26 = mem[0,1,2,3,0,1,2,3]10410; AVX512DQ-BW-NEXT: vpermt2w %zmm24, %zmm26, %zmm2510411; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm24 = [13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0]10412; AVX512DQ-BW-NEXT: # zmm24 = mem[0,1,2,3,0,1,2,3]10413; AVX512DQ-BW-NEXT: vpermi2w %zmm16, %zmm3, %zmm510414; AVX512DQ-BW-NEXT: vpermi2w %zmm16, %zmm3, %zmm1810415; AVX512DQ-BW-NEXT: vpermi2w %zmm16, %zmm3, %zmm1010416; AVX512DQ-BW-NEXT: vpermi2w %zmm16, %zmm3, %zmm1710417; AVX512DQ-BW-NEXT: vpermt2w %zmm16, %zmm26, %zmm310418; AVX512DQ-BW-NEXT: vmovdqa64 %zmm7, %zmm2610419; AVX512DQ-BW-NEXT: vpermt2w %zmm15, %zmm24, %zmm2610420; AVX512DQ-BW-NEXT: vmovdqa64 %zmm7, %zmm1610421; AVX512DQ-BW-NEXT: vpermt2w %zmm15, %zmm19, %zmm710422; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm19 = [5,37,8,40,0,0,6,38,9,41,0,0,7,39,10,42,5,37,8,40,0,0,6,38,9,41,0,0,7,39,10,42]10423; AVX512DQ-BW-NEXT: # zmm19 = mem[0,1,2,3,0,1,2,3]10424; AVX512DQ-BW-NEXT: vpermt2w %zmm15, %zmm19, %zmm1610425; AVX512DQ-BW-NEXT: vpermi2w %zmm12, %zmm1, %zmm1110426; AVX512DQ-BW-NEXT: vpermi2w %zmm12, %zmm1, %zmm810427; AVX512DQ-BW-NEXT: vpermi2w %zmm12, %zmm1, %zmm2110428; AVX512DQ-BW-NEXT: vpermi2w %zmm12, %zmm1, %zmm2410429; AVX512DQ-BW-NEXT: vpermt2w %zmm12, %zmm19, %zmm110430; AVX512DQ-BW-NEXT: movw $9362, %ax # imm = 0x249210431; AVX512DQ-BW-NEXT: kmovd %eax, %k210432; AVX512DQ-BW-NEXT: vmovdqa32 %zmm13, %zmm2 {%k2}10433; AVX512DQ-BW-NEXT: vmovdqa64 (%r8), %zmm1210434; AVX512DQ-BW-NEXT: movw $18724, %ax # imm = 0x492410435; AVX512DQ-BW-NEXT: kmovd %eax, %k110436; AVX512DQ-BW-NEXT: vmovdqa32 %zmm20, %zmm4 {%k1}10437; AVX512DQ-BW-NEXT: vmovdqa64 64(%r8), %zmm1310438; AVX512DQ-BW-NEXT: vmovdqa32 %zmm22, %zmm6 {%k1}10439; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm15 = [0,1,2,3,32,0,6,7,8,9,33,0,12,13,14,15,34,0,18,19,20,21,35,0,24,25,26,27,36,0,30,31]10440; AVX512DQ-BW-NEXT: vpermt2w %zmm12, %zmm15, %zmm210441; AVX512DQ-BW-NEXT: vmovdqa32 %zmm23, %zmm9 {%k2}10442; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm19 = [58,0,2,3,4,5,59,0,8,9,10,11,60,0,14,15,16,17,61,0,20,21,22,23,62,0,26,27,28,29,63,0]10443; AVX512DQ-BW-NEXT: vpermt2w %zmm13, %zmm19, %zmm410444; AVX512DQ-BW-NEXT: vmovdqa32 %zmm26, %zmm14 {%k1}10445; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm20 = [0,1,53,0,4,5,6,7,54,0,10,11,12,13,55,0,16,17,18,19,56,0,22,23,24,25,57,0,28,29,30,31]10446; AVX512DQ-BW-NEXT: vpermt2w %zmm13, %zmm20, %zmm610447; AVX512DQ-BW-NEXT: vmovdqa32 %zmm25, %zmm16 {%k1}10448; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm22 = [0,1,2,3,48,0,6,7,8,9,49,0,12,13,14,15,50,0,18,19,20,21,51,0,24,25,26,27,52,0,30,31]10449; AVX512DQ-BW-NEXT: vpermt2w %zmm13, %zmm22, %zmm910450; AVX512DQ-BW-NEXT: vmovdqa32 %zmm7, %zmm0 {%k2}10451; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm7 = [42,0,2,3,4,5,43,0,8,9,10,11,44,0,14,15,16,17,45,0,20,21,22,23,46,0,26,27,28,29,47,0]10452; AVX512DQ-BW-NEXT: vpermt2w %zmm13, %zmm7, %zmm1410453; AVX512DQ-BW-NEXT: vpermt2w %zmm13, %zmm15, %zmm010454; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm15 = [0,1,37,0,4,5,6,7,38,0,10,11,12,13,39,0,16,17,18,19,40,0,22,23,24,25,41,0,28,29,30,31]10455; AVX512DQ-BW-NEXT: vpermt2w %zmm13, %zmm15, %zmm1610456; AVX512DQ-BW-NEXT: vmovdqa32 %zmm11, %zmm5 {%k1}10457; AVX512DQ-BW-NEXT: vpermt2w %zmm12, %zmm19, %zmm510458; AVX512DQ-BW-NEXT: vmovdqa32 %zmm18, %zmm8 {%k1}10459; AVX512DQ-BW-NEXT: vpermt2w %zmm12, %zmm20, %zmm810460; AVX512DQ-BW-NEXT: vmovdqa32 %zmm21, %zmm10 {%k2}10461; AVX512DQ-BW-NEXT: vmovdqa64 (%r9), %zmm1110462; AVX512DQ-BW-NEXT: vpermt2w %zmm12, %zmm22, %zmm1010463; AVX512DQ-BW-NEXT: vmovdqa64 64(%r9), %zmm1310464; AVX512DQ-BW-NEXT: vmovdqa32 %zmm24, %zmm17 {%k1}10465; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm18 = [0,1,2,3,4,32,6,7,8,9,10,33,12,13,14,15,16,34,18,19,20,21,22,35,24,25,26,27,28,36,30,31]10466; AVX512DQ-BW-NEXT: vpermt2w %zmm11, %zmm18, %zmm210467; AVX512DQ-BW-NEXT: vpermt2w %zmm12, %zmm7, %zmm1710468; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm7 = [0,58,2,3,4,5,6,59,8,9,10,11,12,60,14,15,16,17,18,61,20,21,22,23,24,62,26,27,28,29,30,63]10469; AVX512DQ-BW-NEXT: vpermt2w %zmm13, %zmm7, %zmm410470; AVX512DQ-BW-NEXT: vmovdqa32 %zmm3, %zmm1 {%k1}10471; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,1,2,53,4,5,6,7,8,54,10,11,12,13,14,55,16,17,18,19,20,56,22,23,24,25,26,57,28,29,30,31]10472; AVX512DQ-BW-NEXT: vpermt2w %zmm13, %zmm3, %zmm610473; AVX512DQ-BW-NEXT: vpermt2w %zmm12, %zmm15, %zmm110474; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm12 = [0,1,2,3,4,48,6,7,8,9,10,49,12,13,14,15,16,50,18,19,20,21,22,51,24,25,26,27,28,52,30,31]10475; AVX512DQ-BW-NEXT: vpermt2w %zmm13, %zmm12, %zmm910476; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm15 = [0,42,2,3,4,5,6,43,8,9,10,11,12,44,14,15,16,17,18,45,20,21,22,23,24,46,26,27,28,29,30,47]10477; AVX512DQ-BW-NEXT: vpermt2w %zmm13, %zmm15, %zmm1410478; AVX512DQ-BW-NEXT: vpermt2w %zmm13, %zmm18, %zmm010479; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm18 = [0,1,2,37,4,5,6,7,8,38,10,11,12,13,14,39,16,17,18,19,20,40,22,23,24,25,26,41,28,29,30,31]10480; AVX512DQ-BW-NEXT: vpermt2w %zmm13, %zmm18, %zmm1610481; AVX512DQ-BW-NEXT: vpermt2w %zmm11, %zmm7, %zmm510482; AVX512DQ-BW-NEXT: vpermt2w %zmm11, %zmm3, %zmm810483; AVX512DQ-BW-NEXT: vpermt2w %zmm11, %zmm12, %zmm1010484; AVX512DQ-BW-NEXT: vpermt2w %zmm11, %zmm15, %zmm1710485; AVX512DQ-BW-NEXT: vpermt2w %zmm11, %zmm18, %zmm110486; AVX512DQ-BW-NEXT: movq {{[0-9]+}}(%rsp), %rax10487; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, 64(%rax)10488; AVX512DQ-BW-NEXT: vmovdqa64 %zmm17, 128(%rax)10489; AVX512DQ-BW-NEXT: vmovdqa64 %zmm10, 192(%rax)10490; AVX512DQ-BW-NEXT: vmovdqa64 %zmm8, 256(%rax)10491; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, 320(%rax)10492; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, 384(%rax)10493; AVX512DQ-BW-NEXT: vmovdqa64 %zmm16, 448(%rax)10494; AVX512DQ-BW-NEXT: vmovdqa64 %zmm14, 512(%rax)10495; AVX512DQ-BW-NEXT: vmovdqa64 %zmm9, 576(%rax)10496; AVX512DQ-BW-NEXT: vmovdqa64 %zmm6, 640(%rax)10497; AVX512DQ-BW-NEXT: vmovdqa64 %zmm4, 704(%rax)10498; AVX512DQ-BW-NEXT: vmovdqa64 %zmm2, (%rax)10499; AVX512DQ-BW-NEXT: vzeroupper10500; AVX512DQ-BW-NEXT: retq10501;10502; AVX512DQ-BW-FCP-LABEL: store_i16_stride6_vf64:10503; AVX512DQ-BW-FCP: # %bb.0:10504; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm310505; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm010506; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rsi), %zmm1610507; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rsi), %zmm2410508; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdx), %zmm110509; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdx), %zmm710510; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rcx), %zmm1210511; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rcx), %zmm1510512; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm19 = [0,0,0,32,3,35,0,0,1,33,4,36,0,0,2,34,0,0,0,32,3,35,0,0,1,33,4,36,0,0,2,34]10513; AVX512DQ-BW-FCP-NEXT: # zmm19 = mem[0,1,2,3,0,1,2,3]10514; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm1310515; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm12, %zmm19, %zmm1310516; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm26 = [0,32,3,35,0,0,1,33,4,36,0,0,2,34,5,37,0,32,3,35,0,0,1,33,4,36,0,0,2,34,5,37]10517; AVX512DQ-BW-FCP-NEXT: # zmm26 = mem[0,1,2,3,0,1,2,3]10518; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm3, %zmm210519; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm16, %zmm26, %zmm210520; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm11 = [29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0]10521; AVX512DQ-BW-FCP-NEXT: # zmm11 = mem[0,1,2,3,0,1,2,3]10522; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm7, %zmm2010523; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm15, %zmm11, %zmm2010524; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm5 = [0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61,0,0,27,59,30,62,0,0,28,60,31,63,0,0,29,61]10525; AVX512DQ-BW-FCP-NEXT: # zmm5 = mem[0,1,2,3,0,1,2,3]10526; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm410527; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm24, %zmm5, %zmm410528; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm18 = [24,56,0,0,22,54,25,57,0,0,23,55,26,58,0,0,24,56,0,0,22,54,25,57,0,0,23,55,26,58,0,0]10529; AVX512DQ-BW-FCP-NEXT: # zmm18 = mem[0,1,2,3,0,1,2,3]10530; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm2210531; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm24, %zmm18, %zmm2210532; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm8 = [21,53,24,56,0,0,22,54,25,57,0,0,23,55,26,58,21,53,24,56,0,0,22,54,25,57,0,0,23,55,26,58]10533; AVX512DQ-BW-FCP-NEXT: # zmm8 = mem[0,1,2,3,0,1,2,3]10534; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm7, %zmm610535; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm15, %zmm8, %zmm610536; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm21 = [0,0,16,48,19,51,0,0,17,49,20,52,0,0,18,50,0,0,16,48,19,51,0,0,17,49,20,52,0,0,18,50]10537; AVX512DQ-BW-FCP-NEXT: # zmm21 = mem[0,1,2,3,0,1,2,3]10538; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm7, %zmm2310539; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm15, %zmm21, %zmm2310540; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm10 = [16,48,19,51,0,0,17,49,20,52,0,0,18,50,21,53,16,48,19,51,0,0,17,49,20,52,0,0,18,50,21,53]10541; AVX512DQ-BW-FCP-NEXT: # zmm10 = mem[0,1,2,3,0,1,2,3]10542; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm910543; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm24, %zmm10, %zmm910544; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm17 = [0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45]10545; AVX512DQ-BW-FCP-NEXT: # zmm17 = mem[0,1,2,3,0,1,2,3]10546; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm1410547; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm24, %zmm17, %zmm1410548; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm2510549; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm24, %zmm26, %zmm010550; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm26 = [8,40,0,0,6,38,9,41,0,0,7,39,10,42,0,0,8,40,0,0,6,38,9,41,0,0,7,39,10,42,0,0]10551; AVX512DQ-BW-FCP-NEXT: # zmm26 = mem[0,1,2,3,0,1,2,3]10552; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm24, %zmm26, %zmm2510553; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm24 = [13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0,13,45,0,0,11,43,14,46,0,0,12,44,15,47,0,0]10554; AVX512DQ-BW-FCP-NEXT: # zmm24 = mem[0,1,2,3,0,1,2,3]10555; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm16, %zmm3, %zmm510556; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm16, %zmm3, %zmm1810557; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm16, %zmm3, %zmm1010558; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm16, %zmm3, %zmm1710559; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm16, %zmm26, %zmm310560; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm7, %zmm2610561; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm15, %zmm24, %zmm2610562; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm7, %zmm1610563; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm15, %zmm19, %zmm710564; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm19 = [5,37,8,40,0,0,6,38,9,41,0,0,7,39,10,42,5,37,8,40,0,0,6,38,9,41,0,0,7,39,10,42]10565; AVX512DQ-BW-FCP-NEXT: # zmm19 = mem[0,1,2,3,0,1,2,3]10566; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm15, %zmm19, %zmm1610567; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm12, %zmm1, %zmm1110568; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm12, %zmm1, %zmm810569; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm12, %zmm1, %zmm2110570; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm12, %zmm1, %zmm2410571; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm12, %zmm19, %zmm110572; AVX512DQ-BW-FCP-NEXT: movw $9362, %ax # imm = 0x249210573; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k210574; AVX512DQ-BW-FCP-NEXT: vmovdqa32 %zmm13, %zmm2 {%k2}10575; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%r8), %zmm1210576; AVX512DQ-BW-FCP-NEXT: movw $18724, %ax # imm = 0x492410577; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k110578; AVX512DQ-BW-FCP-NEXT: vmovdqa32 %zmm20, %zmm4 {%k1}10579; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%r8), %zmm1310580; AVX512DQ-BW-FCP-NEXT: vmovdqa32 %zmm22, %zmm6 {%k1}10581; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm15 = [0,1,2,3,32,0,6,7,8,9,33,0,12,13,14,15,34,0,18,19,20,21,35,0,24,25,26,27,36,0,30,31]10582; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm12, %zmm15, %zmm210583; AVX512DQ-BW-FCP-NEXT: vmovdqa32 %zmm23, %zmm9 {%k2}10584; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm19 = [58,0,2,3,4,5,59,0,8,9,10,11,60,0,14,15,16,17,61,0,20,21,22,23,62,0,26,27,28,29,63,0]10585; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm13, %zmm19, %zmm410586; AVX512DQ-BW-FCP-NEXT: vmovdqa32 %zmm26, %zmm14 {%k1}10587; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm20 = [0,1,53,0,4,5,6,7,54,0,10,11,12,13,55,0,16,17,18,19,56,0,22,23,24,25,57,0,28,29,30,31]10588; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm13, %zmm20, %zmm610589; AVX512DQ-BW-FCP-NEXT: vmovdqa32 %zmm25, %zmm16 {%k1}10590; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm22 = [0,1,2,3,48,0,6,7,8,9,49,0,12,13,14,15,50,0,18,19,20,21,51,0,24,25,26,27,52,0,30,31]10591; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm13, %zmm22, %zmm910592; AVX512DQ-BW-FCP-NEXT: vmovdqa32 %zmm7, %zmm0 {%k2}10593; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm7 = [42,0,2,3,4,5,43,0,8,9,10,11,44,0,14,15,16,17,45,0,20,21,22,23,46,0,26,27,28,29,47,0]10594; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm13, %zmm7, %zmm1410595; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm13, %zmm15, %zmm010596; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm15 = [0,1,37,0,4,5,6,7,38,0,10,11,12,13,39,0,16,17,18,19,40,0,22,23,24,25,41,0,28,29,30,31]10597; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm13, %zmm15, %zmm1610598; AVX512DQ-BW-FCP-NEXT: vmovdqa32 %zmm11, %zmm5 {%k1}10599; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm12, %zmm19, %zmm510600; AVX512DQ-BW-FCP-NEXT: vmovdqa32 %zmm18, %zmm8 {%k1}10601; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm12, %zmm20, %zmm810602; AVX512DQ-BW-FCP-NEXT: vmovdqa32 %zmm21, %zmm10 {%k2}10603; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%r9), %zmm1110604; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm12, %zmm22, %zmm1010605; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%r9), %zmm1310606; AVX512DQ-BW-FCP-NEXT: vmovdqa32 %zmm24, %zmm17 {%k1}10607; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm18 = [0,1,2,3,4,32,6,7,8,9,10,33,12,13,14,15,16,34,18,19,20,21,22,35,24,25,26,27,28,36,30,31]10608; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm11, %zmm18, %zmm210609; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm12, %zmm7, %zmm1710610; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm7 = [0,58,2,3,4,5,6,59,8,9,10,11,12,60,14,15,16,17,18,61,20,21,22,23,24,62,26,27,28,29,30,63]10611; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm13, %zmm7, %zmm410612; AVX512DQ-BW-FCP-NEXT: vmovdqa32 %zmm3, %zmm1 {%k1}10613; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,1,2,53,4,5,6,7,8,54,10,11,12,13,14,55,16,17,18,19,20,56,22,23,24,25,26,57,28,29,30,31]10614; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm13, %zmm3, %zmm610615; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm12, %zmm15, %zmm110616; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm12 = [0,1,2,3,4,48,6,7,8,9,10,49,12,13,14,15,16,50,18,19,20,21,22,51,24,25,26,27,28,52,30,31]10617; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm13, %zmm12, %zmm910618; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm15 = [0,42,2,3,4,5,6,43,8,9,10,11,12,44,14,15,16,17,18,45,20,21,22,23,24,46,26,27,28,29,30,47]10619; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm13, %zmm15, %zmm1410620; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm13, %zmm18, %zmm010621; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm18 = [0,1,2,37,4,5,6,7,8,38,10,11,12,13,14,39,16,17,18,19,20,40,22,23,24,25,26,41,28,29,30,31]10622; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm13, %zmm18, %zmm1610623; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm11, %zmm7, %zmm510624; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm11, %zmm3, %zmm810625; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm11, %zmm12, %zmm1010626; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm11, %zmm15, %zmm1710627; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm11, %zmm18, %zmm110628; AVX512DQ-BW-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax10629; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, 64(%rax)10630; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm17, 128(%rax)10631; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm10, 192(%rax)10632; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm8, 256(%rax)10633; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, 320(%rax)10634; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, 384(%rax)10635; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm16, 448(%rax)10636; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm14, 512(%rax)10637; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm9, 576(%rax)10638; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm6, 640(%rax)10639; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm4, 704(%rax)10640; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm2, (%rax)10641; AVX512DQ-BW-FCP-NEXT: vzeroupper10642; AVX512DQ-BW-FCP-NEXT: retq10643 %in.vec0 = load <64 x i16>, ptr %in.vecptr0, align 6410644 %in.vec1 = load <64 x i16>, ptr %in.vecptr1, align 6410645 %in.vec2 = load <64 x i16>, ptr %in.vecptr2, align 6410646 %in.vec3 = load <64 x i16>, ptr %in.vecptr3, align 6410647 %in.vec4 = load <64 x i16>, ptr %in.vecptr4, align 6410648 %in.vec5 = load <64 x i16>, ptr %in.vecptr5, align 6410649 %1 = shufflevector <64 x i16> %in.vec0, <64 x i16> %in.vec1, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>10650 %2 = shufflevector <64 x i16> %in.vec2, <64 x i16> %in.vec3, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>10651 %3 = shufflevector <64 x i16> %in.vec4, <64 x i16> %in.vec5, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>10652 %4 = shufflevector <128 x i16> %1, <128 x i16> %2, <256 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127, i32 128, i32 129, i32 130, i32 131, i32 132, i32 133, i32 134, i32 135, i32 136, i32 137, i32 138, i32 139, i32 140, i32 141, i32 142, i32 143, i32 144, i32 145, i32 146, i32 147, i32 148, i32 149, i32 150, i32 151, i32 152, i32 153, i32 154, i32 155, i32 156, i32 157, i32 158, i32 159, i32 160, i32 161, i32 162, i32 163, i32 164, i32 165, i32 166, i32 167, i32 168, i32 169, i32 170, i32 171, i32 172, i32 173, i32 174, i32 175, i32 176, i32 177, i32 178, i32 179, i32 180, i32 181, i32 182, i32 183, i32 184, i32 185, i32 186, i32 187, i32 188, i32 189, i32 190, i32 191, i32 192, i32 193, i32 194, i32 195, i32 196, i32 197, i32 198, i32 199, i32 200, i32 201, i32 202, i32 203, i32 204, i32 205, i32 206, i32 207, i32 208, i32 209, i32 210, i32 211, i32 212, i32 213, i32 214, i32 215, i32 216, i32 217, i32 218, i32 219, i32 220, i32 221, i32 222, i32 223, i32 224, i32 225, i32 226, i32 227, i32 228, i32 229, i32 230, i32 231, i32 232, i32 233, i32 234, i32 235, i32 236, i32 237, i32 238, i32 239, i32 240, i32 241, i32 242, i32 243, i32 244, i32 245, i32 246, i32 247, i32 248, i32 249, i32 250, i32 251, i32 252, i32 253, i32 254, i32 255>10653 %5 = shufflevector <128 x i16> %3, <128 x i16> poison, <256 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>10654 %6 = shufflevector <256 x i16> %4, <256 x i16> %5, <384 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127, i32 128, i32 129, i32 130, i32 131, i32 132, i32 133, i32 134, i32 135, i32 136, i32 137, i32 138, i32 139, i32 140, i32 141, i32 142, i32 143, i32 144, i32 145, i32 146, i32 147, i32 148, i32 149, i32 150, i32 151, i32 152, i32 153, i32 154, i32 155, i32 156, i32 157, i32 158, i32 159, i32 160, i32 161, i32 162, i32 163, i32 164, i32 165, i32 166, i32 167, i32 168, i32 169, i32 170, i32 171, i32 172, i32 173, i32 174, i32 175, i32 176, i32 177, i32 178, i32 179, i32 180, i32 181, i32 182, i32 183, i32 184, i32 185, i32 186, i32 187, i32 188, i32 189, i32 190, i32 191, i32 192, i32 193, i32 194, i32 195, i32 196, i32 197, i32 198, i32 199, i32 200, i32 201, i32 202, i32 203, i32 204, i32 205, i32 206, i32 207, i32 208, i32 209, i32 210, i32 211, i32 212, i32 213, i32 214, i32 215, i32 216, i32 217, i32 218, i32 219, i32 220, i32 221, i32 222, i32 223, i32 224, i32 225, i32 226, i32 227, i32 228, i32 229, i32 230, i32 231, i32 232, i32 233, i32 234, i32 235, i32 236, i32 237, i32 238, i32 239, i32 240, i32 241, i32 242, i32 243, i32 244, i32 245, i32 246, i32 247, i32 248, i32 249, i32 250, i32 251, i32 252, i32 253, i32 254, i32 255, i32 256, i32 257, i32 258, i32 259, i32 260, i32 261, i32 262, i32 263, i32 264, i32 265, i32 266, i32 267, i32 268, i32 269, i32 270, i32 271, i32 272, i32 273, i32 274, i32 275, i32 276, i32 277, i32 278, i32 279, i32 280, i32 281, i32 282, i32 283, i32 284, i32 285, i32 286, i32 287, i32 288, i32 289, i32 290, i32 291, i32 292, i32 293, i32 294, i32 295, i32 296, i32 297, i32 298, i32 299, i32 300, i32 301, i32 302, i32 303, i32 304, i32 305, i32 306, i32 307, i32 308, i32 309, i32 310, i32 311, i32 312, i32 313, i32 314, i32 315, i32 316, i32 317, i32 318, i32 319, i32 320, i32 321, i32 322, i32 323, i32 324, i32 325, i32 326, i32 327, i32 328, i32 329, i32 330, i32 331, i32 332, i32 333, i32 334, i32 335, i32 336, i32 337, i32 338, i32 339, i32 340, i32 341, i32 342, i32 343, i32 344, i32 345, i32 346, i32 347, i32 348, i32 349, i32 350, i32 351, i32 352, i32 353, i32 354, i32 355, i32 356, i32 357, i32 358, i32 359, i32 360, i32 361, i32 362, i32 363, i32 364, i32 365, i32 366, i32 367, i32 368, i32 369, i32 370, i32 371, i32 372, i32 373, i32 374, i32 375, i32 376, i32 377, i32 378, i32 379, i32 380, i32 381, i32 382, i32 383>10655 %interleaved.vec = shufflevector <384 x i16> %6, <384 x i16> poison, <384 x i32> <i32 0, i32 64, i32 128, i32 192, i32 256, i32 320, i32 1, i32 65, i32 129, i32 193, i32 257, i32 321, i32 2, i32 66, i32 130, i32 194, i32 258, i32 322, i32 3, i32 67, i32 131, i32 195, i32 259, i32 323, i32 4, i32 68, i32 132, i32 196, i32 260, i32 324, i32 5, i32 69, i32 133, i32 197, i32 261, i32 325, i32 6, i32 70, i32 134, i32 198, i32 262, i32 326, i32 7, i32 71, i32 135, i32 199, i32 263, i32 327, i32 8, i32 72, i32 136, i32 200, i32 264, i32 328, i32 9, i32 73, i32 137, i32 201, i32 265, i32 329, i32 10, i32 74, i32 138, i32 202, i32 266, i32 330, i32 11, i32 75, i32 139, i32 203, i32 267, i32 331, i32 12, i32 76, i32 140, i32 204, i32 268, i32 332, i32 13, i32 77, i32 141, i32 205, i32 269, i32 333, i32 14, i32 78, i32 142, i32 206, i32 270, i32 334, i32 15, i32 79, i32 143, i32 207, i32 271, i32 335, i32 16, i32 80, i32 144, i32 208, i32 272, i32 336, i32 17, i32 81, i32 145, i32 209, i32 273, i32 337, i32 18, i32 82, i32 146, i32 210, i32 274, i32 338, i32 19, i32 83, i32 147, i32 211, i32 275, i32 339, i32 20, i32 84, i32 148, i32 212, i32 276, i32 340, i32 21, i32 85, i32 149, i32 213, i32 277, i32 341, i32 22, i32 86, i32 150, i32 214, i32 278, i32 342, i32 23, i32 87, i32 151, i32 215, i32 279, i32 343, i32 24, i32 88, i32 152, i32 216, i32 280, i32 344, i32 25, i32 89, i32 153, i32 217, i32 281, i32 345, i32 26, i32 90, i32 154, i32 218, i32 282, i32 346, i32 27, i32 91, i32 155, i32 219, i32 283, i32 347, i32 28, i32 92, i32 156, i32 220, i32 284, i32 348, i32 29, i32 93, i32 157, i32 221, i32 285, i32 349, i32 30, i32 94, i32 158, i32 222, i32 286, i32 350, i32 31, i32 95, i32 159, i32 223, i32 287, i32 351, i32 32, i32 96, i32 160, i32 224, i32 288, i32 352, i32 33, i32 97, i32 161, i32 225, i32 289, i32 353, i32 34, i32 98, i32 162, i32 226, i32 290, i32 354, i32 35, i32 99, i32 163, i32 227, i32 291, i32 355, i32 36, i32 100, i32 164, i32 228, i32 292, i32 356, i32 37, i32 101, i32 165, i32 229, i32 293, i32 357, i32 38, i32 102, i32 166, i32 230, i32 294, i32 358, i32 39, i32 103, i32 167, i32 231, i32 295, i32 359, i32 40, i32 104, i32 168, i32 232, i32 296, i32 360, i32 41, i32 105, i32 169, i32 233, i32 297, i32 361, i32 42, i32 106, i32 170, i32 234, i32 298, i32 362, i32 43, i32 107, i32 171, i32 235, i32 299, i32 363, i32 44, i32 108, i32 172, i32 236, i32 300, i32 364, i32 45, i32 109, i32 173, i32 237, i32 301, i32 365, i32 46, i32 110, i32 174, i32 238, i32 302, i32 366, i32 47, i32 111, i32 175, i32 239, i32 303, i32 367, i32 48, i32 112, i32 176, i32 240, i32 304, i32 368, i32 49, i32 113, i32 177, i32 241, i32 305, i32 369, i32 50, i32 114, i32 178, i32 242, i32 306, i32 370, i32 51, i32 115, i32 179, i32 243, i32 307, i32 371, i32 52, i32 116, i32 180, i32 244, i32 308, i32 372, i32 53, i32 117, i32 181, i32 245, i32 309, i32 373, i32 54, i32 118, i32 182, i32 246, i32 310, i32 374, i32 55, i32 119, i32 183, i32 247, i32 311, i32 375, i32 56, i32 120, i32 184, i32 248, i32 312, i32 376, i32 57, i32 121, i32 185, i32 249, i32 313, i32 377, i32 58, i32 122, i32 186, i32 250, i32 314, i32 378, i32 59, i32 123, i32 187, i32 251, i32 315, i32 379, i32 60, i32 124, i32 188, i32 252, i32 316, i32 380, i32 61, i32 125, i32 189, i32 253, i32 317, i32 381, i32 62, i32 126, i32 190, i32 254, i32 318, i32 382, i32 63, i32 127, i32 191, i32 255, i32 319, i32 383>10656 store <384 x i16> %interleaved.vec, ptr %out.vec, align 6410657 ret void10658}10659