6616 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE3; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX4; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX25; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FP6; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FCP7; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX5128; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512-FCP9; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ10; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ-FCP11; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW12; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW-FCP13; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512DQ-BW14; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ-BW-FCP15 16; These patterns are produced by LoopVectorizer for interleaved loads.17 18define void @load_i16_stride4_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3) nounwind {19; SSE-LABEL: load_i16_stride4_vf2:20; SSE: # %bb.0:21; SSE-NEXT: movdqa (%rdi), %xmm022; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]23; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm1[0,2,2,3,4,5,6,7]24; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]25; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]26; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm0[2,0,2,3,4,5,6,7]27; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7]28; SSE-NEXT: movd %xmm2, (%rsi)29; SSE-NEXT: movd %xmm1, (%rdx)30; SSE-NEXT: movd %xmm3, (%rcx)31; SSE-NEXT: movd %xmm0, (%r8)32; SSE-NEXT: retq33;34; AVX-LABEL: load_i16_stride4_vf2:35; AVX: # %bb.0:36; AVX-NEXT: vmovdqa (%rdi), %xmm037; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]38; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm1[0,2,2,3,4,5,6,7]39; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]40; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]41; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm0[2,0,2,3,4,5,6,7]42; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7]43; AVX-NEXT: vmovd %xmm2, (%rsi)44; AVX-NEXT: vmovd %xmm1, (%rdx)45; AVX-NEXT: vmovd %xmm3, (%rcx)46; AVX-NEXT: vmovd %xmm0, (%r8)47; AVX-NEXT: retq48;49; AVX2-LABEL: load_i16_stride4_vf2:50; AVX2: # %bb.0:51; AVX2-NEXT: vmovdqa (%rdi), %xmm052; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]53; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm1[0,2,2,3,4,5,6,7]54; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]55; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]56; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm0[2,0,2,3,4,5,6,7]57; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7]58; AVX2-NEXT: vmovd %xmm2, (%rsi)59; AVX2-NEXT: vmovd %xmm1, (%rdx)60; AVX2-NEXT: vmovd %xmm3, (%rcx)61; AVX2-NEXT: vmovd %xmm0, (%r8)62; AVX2-NEXT: retq63;64; AVX2-FP-LABEL: load_i16_stride4_vf2:65; AVX2-FP: # %bb.0:66; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm067; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]68; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm1[0,2,2,3,4,5,6,7]69; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]70; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]71; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm0[2,0,2,3,4,5,6,7]72; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7]73; AVX2-FP-NEXT: vmovd %xmm2, (%rsi)74; AVX2-FP-NEXT: vmovd %xmm1, (%rdx)75; AVX2-FP-NEXT: vmovd %xmm3, (%rcx)76; AVX2-FP-NEXT: vmovd %xmm0, (%r8)77; AVX2-FP-NEXT: retq78;79; AVX2-FCP-LABEL: load_i16_stride4_vf2:80; AVX2-FCP: # %bb.0:81; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm082; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]83; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm1[0,2,2,3,4,5,6,7]84; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]85; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]86; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm0[2,0,2,3,4,5,6,7]87; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7]88; AVX2-FCP-NEXT: vmovd %xmm2, (%rsi)89; AVX2-FCP-NEXT: vmovd %xmm1, (%rdx)90; AVX2-FCP-NEXT: vmovd %xmm3, (%rcx)91; AVX2-FCP-NEXT: vmovd %xmm0, (%r8)92; AVX2-FCP-NEXT: retq93;94; AVX512-LABEL: load_i16_stride4_vf2:95; AVX512: # %bb.0:96; AVX512-NEXT: vmovdqa (%rdi), %xmm097; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]98; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]99; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]100; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[2,0,2,3,4,5,6,7]101; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]102; AVX512-NEXT: vpmovqw %xmm0, (%rsi)103; AVX512-NEXT: vmovd %xmm1, (%rdx)104; AVX512-NEXT: vmovd %xmm3, (%rcx)105; AVX512-NEXT: vmovd %xmm2, (%r8)106; AVX512-NEXT: retq107;108; AVX512-FCP-LABEL: load_i16_stride4_vf2:109; AVX512-FCP: # %bb.0:110; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm0111; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[2,3,10,11,u,u,u,u,u,u,u,u,u,u,u,u]112; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]113; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[2,0,2,3,4,5,6,7]114; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]115; AVX512-FCP-NEXT: vpmovqw %xmm0, (%rsi)116; AVX512-FCP-NEXT: vmovd %xmm1, (%rdx)117; AVX512-FCP-NEXT: vmovd %xmm3, (%rcx)118; AVX512-FCP-NEXT: vmovd %xmm2, (%r8)119; AVX512-FCP-NEXT: retq120;121; AVX512DQ-LABEL: load_i16_stride4_vf2:122; AVX512DQ: # %bb.0:123; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0124; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]125; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]126; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]127; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[2,0,2,3,4,5,6,7]128; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]129; AVX512DQ-NEXT: vpmovqw %xmm0, (%rsi)130; AVX512DQ-NEXT: vmovd %xmm1, (%rdx)131; AVX512DQ-NEXT: vmovd %xmm3, (%rcx)132; AVX512DQ-NEXT: vmovd %xmm2, (%r8)133; AVX512DQ-NEXT: retq134;135; AVX512DQ-FCP-LABEL: load_i16_stride4_vf2:136; AVX512DQ-FCP: # %bb.0:137; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm0138; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[2,3,10,11,u,u,u,u,u,u,u,u,u,u,u,u]139; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]140; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[2,0,2,3,4,5,6,7]141; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]142; AVX512DQ-FCP-NEXT: vpmovqw %xmm0, (%rsi)143; AVX512DQ-FCP-NEXT: vmovd %xmm1, (%rdx)144; AVX512DQ-FCP-NEXT: vmovd %xmm3, (%rcx)145; AVX512DQ-FCP-NEXT: vmovd %xmm2, (%r8)146; AVX512DQ-FCP-NEXT: retq147;148; AVX512BW-LABEL: load_i16_stride4_vf2:149; AVX512BW: # %bb.0:150; AVX512BW-NEXT: vmovdqa (%rdi), %xmm0151; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]152; AVX512BW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]153; AVX512BW-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]154; AVX512BW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[2,0,2,3,4,5,6,7]155; AVX512BW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]156; AVX512BW-NEXT: vpmovqw %xmm0, (%rsi)157; AVX512BW-NEXT: vmovd %xmm1, (%rdx)158; AVX512BW-NEXT: vmovd %xmm3, (%rcx)159; AVX512BW-NEXT: vmovd %xmm2, (%r8)160; AVX512BW-NEXT: retq161;162; AVX512BW-FCP-LABEL: load_i16_stride4_vf2:163; AVX512BW-FCP: # %bb.0:164; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %xmm0165; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[2,3,10,11,u,u,u,u,u,u,u,u,u,u,u,u]166; AVX512BW-FCP-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]167; AVX512BW-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[2,0,2,3,4,5,6,7]168; AVX512BW-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]169; AVX512BW-FCP-NEXT: vpmovqw %xmm0, (%rsi)170; AVX512BW-FCP-NEXT: vmovd %xmm1, (%rdx)171; AVX512BW-FCP-NEXT: vmovd %xmm3, (%rcx)172; AVX512BW-FCP-NEXT: vmovd %xmm2, (%r8)173; AVX512BW-FCP-NEXT: retq174;175; AVX512DQ-BW-LABEL: load_i16_stride4_vf2:176; AVX512DQ-BW: # %bb.0:177; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %xmm0178; AVX512DQ-BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]179; AVX512DQ-BW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]180; AVX512DQ-BW-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]181; AVX512DQ-BW-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[2,0,2,3,4,5,6,7]182; AVX512DQ-BW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]183; AVX512DQ-BW-NEXT: vpmovqw %xmm0, (%rsi)184; AVX512DQ-BW-NEXT: vmovd %xmm1, (%rdx)185; AVX512DQ-BW-NEXT: vmovd %xmm3, (%rcx)186; AVX512DQ-BW-NEXT: vmovd %xmm2, (%r8)187; AVX512DQ-BW-NEXT: retq188;189; AVX512DQ-BW-FCP-LABEL: load_i16_stride4_vf2:190; AVX512DQ-BW-FCP: # %bb.0:191; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %xmm0192; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[2,3,10,11,u,u,u,u,u,u,u,u,u,u,u,u]193; AVX512DQ-BW-FCP-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]194; AVX512DQ-BW-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[2,0,2,3,4,5,6,7]195; AVX512DQ-BW-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]196; AVX512DQ-BW-FCP-NEXT: vpmovqw %xmm0, (%rsi)197; AVX512DQ-BW-FCP-NEXT: vmovd %xmm1, (%rdx)198; AVX512DQ-BW-FCP-NEXT: vmovd %xmm3, (%rcx)199; AVX512DQ-BW-FCP-NEXT: vmovd %xmm2, (%r8)200; AVX512DQ-BW-FCP-NEXT: retq201 %wide.vec = load <8 x i16>, ptr %in.vec, align 64202 %strided.vec0 = shufflevector <8 x i16> %wide.vec, <8 x i16> poison, <2 x i32> <i32 0, i32 4>203 %strided.vec1 = shufflevector <8 x i16> %wide.vec, <8 x i16> poison, <2 x i32> <i32 1, i32 5>204 %strided.vec2 = shufflevector <8 x i16> %wide.vec, <8 x i16> poison, <2 x i32> <i32 2, i32 6>205 %strided.vec3 = shufflevector <8 x i16> %wide.vec, <8 x i16> poison, <2 x i32> <i32 3, i32 7>206 store <2 x i16> %strided.vec0, ptr %out.vec0, align 64207 store <2 x i16> %strided.vec1, ptr %out.vec1, align 64208 store <2 x i16> %strided.vec2, ptr %out.vec2, align 64209 store <2 x i16> %strided.vec3, ptr %out.vec3, align 64210 ret void211}212 213define void @load_i16_stride4_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3) nounwind {214; SSE-LABEL: load_i16_stride4_vf4:215; SSE: # %bb.0:216; SSE-NEXT: movdqa (%rdi), %xmm0217; SSE-NEXT: movdqa 16(%rdi), %xmm1218; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3]219; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm2[0,2,2,3,4,5,6,7]220; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,2,2,3]221; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm4[0,2,2,3,4,5,6,7]222; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]223; SSE-NEXT: movq %xmm5, (%rsi)224; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]225; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm4[1,3,2,3,4,5,6,7]226; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]227; SSE-NEXT: movq %xmm3, (%rdx)228; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]229; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm1[2,0,2,3,4,5,6,7]230; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]231; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm0[2,0,2,3,4,5,6,7]232; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]233; SSE-NEXT: movq %xmm3, (%rcx)234; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]235; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7]236; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]237; SSE-NEXT: movq %xmm0, (%r8)238; SSE-NEXT: retq239;240; AVX-LABEL: load_i16_stride4_vf4:241; AVX: # %bb.0:242; AVX-NEXT: vpxor %xmm0, %xmm0, %xmm0243; AVX-NEXT: vmovdqa (%rdi), %xmm1244; AVX-NEXT: vmovdqa 16(%rdi), %xmm2245; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1,2,3],xmm2[4],xmm0[5,6,7]246; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3],xmm1[4],xmm0[5,6,7]247; AVX-NEXT: vpackusdw %xmm3, %xmm0, %xmm0248; AVX-NEXT: vpackusdw %xmm0, %xmm0, %xmm0249; AVX-NEXT: vmovq %xmm0, (%rsi)250; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]251; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]252; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[0,2,2,3]253; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,3,2,3,4,5,6,7]254; AVX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]255; AVX-NEXT: vmovq %xmm0, (%rdx)256; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm2[3,1,2,3]257; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[2,0,2,3,4,5,6,7]258; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]259; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm1[2,0,2,3,4,5,6,7]260; AVX-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]261; AVX-NEXT: vmovq %xmm2, (%rcx)262; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7]263; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]264; AVX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]265; AVX-NEXT: vmovq %xmm0, (%r8)266; AVX-NEXT: retq267;268; AVX2-LABEL: load_i16_stride4_vf4:269; AVX2: # %bb.0:270; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0271; AVX2-NEXT: vmovdqa (%rdi), %xmm1272; AVX2-NEXT: vmovdqa 16(%rdi), %xmm2273; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1,2,3],xmm2[4],xmm0[5,6,7]274; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3],xmm1[4],xmm0[5,6,7]275; AVX2-NEXT: vpackusdw %xmm3, %xmm0, %xmm0276; AVX2-NEXT: vpackusdw %xmm0, %xmm0, %xmm0277; AVX2-NEXT: vmovq %xmm0, (%rsi)278; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]279; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]280; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[0,2,2,3]281; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,3,2,3,4,5,6,7]282; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]283; AVX2-NEXT: vmovq %xmm0, (%rdx)284; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm2[3,1,2,3]285; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[2,0,2,3,4,5,6,7]286; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]287; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm1[2,0,2,3,4,5,6,7]288; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]289; AVX2-NEXT: vmovq %xmm2, (%rcx)290; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7]291; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]292; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]293; AVX2-NEXT: vmovq %xmm0, (%r8)294; AVX2-NEXT: retq295;296; AVX2-FP-LABEL: load_i16_stride4_vf4:297; AVX2-FP: # %bb.0:298; AVX2-FP-NEXT: vpxor %xmm0, %xmm0, %xmm0299; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm1300; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm2301; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1,2,3],xmm2[4],xmm0[5,6,7]302; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3],xmm1[4],xmm0[5,6,7]303; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm0, %xmm0304; AVX2-FP-NEXT: vpackusdw %xmm0, %xmm0, %xmm0305; AVX2-FP-NEXT: vmovq %xmm0, (%rsi)306; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm0 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]307; AVX2-FP-NEXT: vpshufb %xmm0, %xmm2, %xmm3308; AVX2-FP-NEXT: vpshufb %xmm0, %xmm1, %xmm0309; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]310; AVX2-FP-NEXT: vmovq %xmm0, (%rdx)311; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm0 = xmm2[3,1,2,3]312; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[2,0,2,3,4,5,6,7]313; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]314; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm1[2,0,2,3,4,5,6,7]315; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]316; AVX2-FP-NEXT: vmovq %xmm2, (%rcx)317; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7]318; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]319; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]320; AVX2-FP-NEXT: vmovq %xmm0, (%r8)321; AVX2-FP-NEXT: retq322;323; AVX2-FCP-LABEL: load_i16_stride4_vf4:324; AVX2-FCP: # %bb.0:325; AVX2-FCP-NEXT: vpxor %xmm0, %xmm0, %xmm0326; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm1327; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm2328; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm0[1,2,3],xmm2[4],xmm0[5,6,7]329; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3],xmm1[4],xmm0[5,6,7]330; AVX2-FCP-NEXT: vpackusdw %xmm3, %xmm0, %xmm0331; AVX2-FCP-NEXT: vpackusdw %xmm0, %xmm0, %xmm0332; AVX2-FCP-NEXT: vmovq %xmm0, (%rsi)333; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]334; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm2, %xmm3335; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm1, %xmm0336; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]337; AVX2-FCP-NEXT: vmovq %xmm0, (%rdx)338; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm0 = xmm2[3,1,2,3]339; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[2,0,2,3,4,5,6,7]340; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]341; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm1[2,0,2,3,4,5,6,7]342; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]343; AVX2-FCP-NEXT: vmovq %xmm2, (%rcx)344; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7]345; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]346; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]347; AVX2-FCP-NEXT: vmovq %xmm0, (%r8)348; AVX2-FCP-NEXT: retq349;350; AVX512-LABEL: load_i16_stride4_vf4:351; AVX512: # %bb.0:352; AVX512-NEXT: vmovdqa (%rdi), %ymm0353; AVX512-NEXT: vpmovqw %ymm0, (%rsi)354; AVX512-NEXT: vpsrlq $16, %ymm0, %ymm1355; AVX512-NEXT: vpmovqw %ymm1, (%rdx)356; AVX512-NEXT: vpsrlq $32, %ymm0, %ymm1357; AVX512-NEXT: vpmovqw %ymm1, (%rcx)358; AVX512-NEXT: vpsrlq $48, %ymm0, %ymm0359; AVX512-NEXT: vpmovqw %ymm0, (%r8)360; AVX512-NEXT: vzeroupper361; AVX512-NEXT: retq362;363; AVX512-FCP-LABEL: load_i16_stride4_vf4:364; AVX512-FCP: # %bb.0:365; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm0366; AVX512-FCP-NEXT: vpmovqw %ymm0, (%rsi)367; AVX512-FCP-NEXT: vpsrlq $16, %ymm0, %ymm1368; AVX512-FCP-NEXT: vpmovqw %ymm1, (%rdx)369; AVX512-FCP-NEXT: vpsrlq $32, %ymm0, %ymm1370; AVX512-FCP-NEXT: vpmovqw %ymm1, (%rcx)371; AVX512-FCP-NEXT: vpsrlq $48, %ymm0, %ymm0372; AVX512-FCP-NEXT: vpmovqw %ymm0, (%r8)373; AVX512-FCP-NEXT: vzeroupper374; AVX512-FCP-NEXT: retq375;376; AVX512DQ-LABEL: load_i16_stride4_vf4:377; AVX512DQ: # %bb.0:378; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm0379; AVX512DQ-NEXT: vpmovqw %ymm0, (%rsi)380; AVX512DQ-NEXT: vpsrlq $16, %ymm0, %ymm1381; AVX512DQ-NEXT: vpmovqw %ymm1, (%rdx)382; AVX512DQ-NEXT: vpsrlq $32, %ymm0, %ymm1383; AVX512DQ-NEXT: vpmovqw %ymm1, (%rcx)384; AVX512DQ-NEXT: vpsrlq $48, %ymm0, %ymm0385; AVX512DQ-NEXT: vpmovqw %ymm0, (%r8)386; AVX512DQ-NEXT: vzeroupper387; AVX512DQ-NEXT: retq388;389; AVX512DQ-FCP-LABEL: load_i16_stride4_vf4:390; AVX512DQ-FCP: # %bb.0:391; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm0392; AVX512DQ-FCP-NEXT: vpmovqw %ymm0, (%rsi)393; AVX512DQ-FCP-NEXT: vpsrlq $16, %ymm0, %ymm1394; AVX512DQ-FCP-NEXT: vpmovqw %ymm1, (%rdx)395; AVX512DQ-FCP-NEXT: vpsrlq $32, %ymm0, %ymm1396; AVX512DQ-FCP-NEXT: vpmovqw %ymm1, (%rcx)397; AVX512DQ-FCP-NEXT: vpsrlq $48, %ymm0, %ymm0398; AVX512DQ-FCP-NEXT: vpmovqw %ymm0, (%r8)399; AVX512DQ-FCP-NEXT: vzeroupper400; AVX512DQ-FCP-NEXT: retq401;402; AVX512BW-LABEL: load_i16_stride4_vf4:403; AVX512BW: # %bb.0:404; AVX512BW-NEXT: vmovdqa (%rdi), %ymm0405; AVX512BW-NEXT: vpmovqw %ymm0, (%rsi)406; AVX512BW-NEXT: vpsrlq $16, %ymm0, %ymm1407; AVX512BW-NEXT: vpmovqw %ymm1, (%rdx)408; AVX512BW-NEXT: vpsrlq $32, %ymm0, %ymm1409; AVX512BW-NEXT: vpmovqw %ymm1, (%rcx)410; AVX512BW-NEXT: vpsrlq $48, %ymm0, %ymm0411; AVX512BW-NEXT: vpmovqw %ymm0, (%r8)412; AVX512BW-NEXT: vzeroupper413; AVX512BW-NEXT: retq414;415; AVX512BW-FCP-LABEL: load_i16_stride4_vf4:416; AVX512BW-FCP: # %bb.0:417; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %ymm0418; AVX512BW-FCP-NEXT: vpmovqw %ymm0, (%rsi)419; AVX512BW-FCP-NEXT: vpsrlq $16, %ymm0, %ymm1420; AVX512BW-FCP-NEXT: vpmovqw %ymm1, (%rdx)421; AVX512BW-FCP-NEXT: vpsrlq $32, %ymm0, %ymm1422; AVX512BW-FCP-NEXT: vpmovqw %ymm1, (%rcx)423; AVX512BW-FCP-NEXT: vpsrlq $48, %ymm0, %ymm0424; AVX512BW-FCP-NEXT: vpmovqw %ymm0, (%r8)425; AVX512BW-FCP-NEXT: vzeroupper426; AVX512BW-FCP-NEXT: retq427;428; AVX512DQ-BW-LABEL: load_i16_stride4_vf4:429; AVX512DQ-BW: # %bb.0:430; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %ymm0431; AVX512DQ-BW-NEXT: vpmovqw %ymm0, (%rsi)432; AVX512DQ-BW-NEXT: vpsrlq $16, %ymm0, %ymm1433; AVX512DQ-BW-NEXT: vpmovqw %ymm1, (%rdx)434; AVX512DQ-BW-NEXT: vpsrlq $32, %ymm0, %ymm1435; AVX512DQ-BW-NEXT: vpmovqw %ymm1, (%rcx)436; AVX512DQ-BW-NEXT: vpsrlq $48, %ymm0, %ymm0437; AVX512DQ-BW-NEXT: vpmovqw %ymm0, (%r8)438; AVX512DQ-BW-NEXT: vzeroupper439; AVX512DQ-BW-NEXT: retq440;441; AVX512DQ-BW-FCP-LABEL: load_i16_stride4_vf4:442; AVX512DQ-BW-FCP: # %bb.0:443; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %ymm0444; AVX512DQ-BW-FCP-NEXT: vpmovqw %ymm0, (%rsi)445; AVX512DQ-BW-FCP-NEXT: vpsrlq $16, %ymm0, %ymm1446; AVX512DQ-BW-FCP-NEXT: vpmovqw %ymm1, (%rdx)447; AVX512DQ-BW-FCP-NEXT: vpsrlq $32, %ymm0, %ymm1448; AVX512DQ-BW-FCP-NEXT: vpmovqw %ymm1, (%rcx)449; AVX512DQ-BW-FCP-NEXT: vpsrlq $48, %ymm0, %ymm0450; AVX512DQ-BW-FCP-NEXT: vpmovqw %ymm0, (%r8)451; AVX512DQ-BW-FCP-NEXT: vzeroupper452; AVX512DQ-BW-FCP-NEXT: retq453 %wide.vec = load <16 x i16>, ptr %in.vec, align 64454 %strided.vec0 = shufflevector <16 x i16> %wide.vec, <16 x i16> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>455 %strided.vec1 = shufflevector <16 x i16> %wide.vec, <16 x i16> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>456 %strided.vec2 = shufflevector <16 x i16> %wide.vec, <16 x i16> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>457 %strided.vec3 = shufflevector <16 x i16> %wide.vec, <16 x i16> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>458 store <4 x i16> %strided.vec0, ptr %out.vec0, align 64459 store <4 x i16> %strided.vec1, ptr %out.vec1, align 64460 store <4 x i16> %strided.vec2, ptr %out.vec2, align 64461 store <4 x i16> %strided.vec3, ptr %out.vec3, align 64462 ret void463}464 465define void @load_i16_stride4_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3) nounwind {466; SSE-LABEL: load_i16_stride4_vf8:467; SSE: # %bb.0:468; SSE-NEXT: movdqa (%rdi), %xmm2469; SSE-NEXT: movdqa 16(%rdi), %xmm3470; SSE-NEXT: movdqa 32(%rdi), %xmm1471; SSE-NEXT: movdqa 48(%rdi), %xmm4472; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,2,2,3]473; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm5[0,2,2,3,4,5,6,7]474; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm2[0,2,2,3]475; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm6[0,2,2,3,4,5,6,7]476; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1]477; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm4[0,2,2,3]478; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm8[0,1,0,2,4,5,6,7]479; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm1[0,2,2,3]480; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm10[0,1,0,2,4,5,6,7]481; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1]482; SSE-NEXT: movsd {{.*#+}} xmm0 = xmm7[0],xmm0[1]483; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7]484; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[1,3,2,3,4,5,6,7]485; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]486; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm8[0,1,1,3,4,5,6,7]487; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm10[0,1,1,3,4,5,6,7]488; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]489; SSE-NEXT: movsd {{.*#+}} xmm7 = xmm6[0],xmm7[1]490; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]491; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm3[2,0,2,3,4,5,6,7]492; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]493; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm2[2,0,2,3,4,5,6,7]494; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]495; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]496; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm4[0,1,2,0,4,5,6,7]497; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]498; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm1[0,1,2,0,4,5,6,7]499; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm5[0],xmm8[1],xmm5[1]500; SSE-NEXT: movsd {{.*#+}} xmm8 = xmm6[0],xmm8[1]501; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7]502; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]503; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]504; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm4[0,1,3,1,4,5,6,7]505; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7]506; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]507; SSE-NEXT: movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]508; SSE-NEXT: movapd %xmm0, (%rsi)509; SSE-NEXT: movapd %xmm7, (%rdx)510; SSE-NEXT: movapd %xmm8, (%rcx)511; SSE-NEXT: movapd %xmm1, (%r8)512; SSE-NEXT: retq513;514; AVX-LABEL: load_i16_stride4_vf8:515; AVX: # %bb.0:516; AVX-NEXT: vpxor %xmm0, %xmm0, %xmm0517; AVX-NEXT: vmovdqa (%rdi), %xmm1518; AVX-NEXT: vmovdqa 16(%rdi), %xmm2519; AVX-NEXT: vmovdqa 32(%rdi), %xmm3520; AVX-NEXT: vmovdqa 48(%rdi), %xmm4521; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm4[0],xmm0[1,2,3],xmm4[4],xmm0[5,6,7]522; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm3[0],xmm0[1,2,3],xmm3[4],xmm0[5,6,7]523; AVX-NEXT: vpackusdw %xmm5, %xmm6, %xmm5524; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm2[0],xmm0[1,2,3],xmm2[4],xmm0[5,6,7]525; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3],xmm1[4],xmm0[5,6,7]526; AVX-NEXT: vpackusdw %xmm6, %xmm0, %xmm0527; AVX-NEXT: vpackusdw %xmm5, %xmm0, %xmm0528; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[0,2,2,3]529; AVX-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,1,3,4,5,6,7]530; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[0,2,2,3]531; AVX-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]532; AVX-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]533; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm2[0,2,2,3]534; AVX-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[1,3,2,3,4,5,6,7]535; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm1[0,2,2,3]536; AVX-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[1,3,2,3,4,5,6,7]537; AVX-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]538; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1,2,3],xmm5[4,5,6,7]539; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]540; AVX-NEXT: vpshuflw {{.*#+}} xmm6 = xmm4[0,1,2,0,4,5,6,7]541; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]542; AVX-NEXT: vpshuflw {{.*#+}} xmm7 = xmm3[0,1,2,0,4,5,6,7]543; AVX-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]544; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]545; AVX-NEXT: vpshuflw {{.*#+}} xmm7 = xmm2[2,0,2,3,4,5,6,7]546; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]547; AVX-NEXT: vpshuflw {{.*#+}} xmm8 = xmm1[2,0,2,3,4,5,6,7]548; AVX-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]549; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0,1,2,3],xmm6[4,5,6,7]550; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]551; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]552; AVX-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]553; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]554; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]555; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]556; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5,6,7]557; AVX-NEXT: vmovdqa %xmm0, (%rsi)558; AVX-NEXT: vmovdqa %xmm5, (%rdx)559; AVX-NEXT: vmovdqa %xmm6, (%rcx)560; AVX-NEXT: vmovdqa %xmm1, (%r8)561; AVX-NEXT: retq562;563; AVX2-LABEL: load_i16_stride4_vf8:564; AVX2: # %bb.0:565; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0566; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]567; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2568; AVX2-NEXT: vpackusdw %xmm2, %xmm1, %xmm1569; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]570; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2571; AVX2-NEXT: vpackusdw %xmm2, %xmm0, %xmm0572; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0573; AVX2-NEXT: vmovdqa (%rdi), %xmm1574; AVX2-NEXT: vmovdqa 16(%rdi), %xmm2575; AVX2-NEXT: vmovdqa 32(%rdi), %xmm3576; AVX2-NEXT: vmovdqa 48(%rdi), %xmm4577; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[0,2,2,3]578; AVX2-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,1,3,4,5,6,7]579; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[0,2,2,3]580; AVX2-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]581; AVX2-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]582; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm2[0,2,2,3]583; AVX2-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[1,3,2,3,4,5,6,7]584; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm1[0,2,2,3]585; AVX2-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[1,3,2,3,4,5,6,7]586; AVX2-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]587; AVX2-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]588; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]589; AVX2-NEXT: vpshuflw {{.*#+}} xmm6 = xmm4[0,1,2,0,4,5,6,7]590; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]591; AVX2-NEXT: vpshuflw {{.*#+}} xmm7 = xmm3[0,1,2,0,4,5,6,7]592; AVX2-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]593; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]594; AVX2-NEXT: vpshuflw {{.*#+}} xmm7 = xmm2[2,0,2,3,4,5,6,7]595; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]596; AVX2-NEXT: vpshuflw {{.*#+}} xmm8 = xmm1[2,0,2,3,4,5,6,7]597; AVX2-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]598; AVX2-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3]599; AVX2-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]600; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]601; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]602; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]603; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]604; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]605; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3]606; AVX2-NEXT: vmovdqa %xmm0, (%rsi)607; AVX2-NEXT: vmovdqa %xmm5, (%rdx)608; AVX2-NEXT: vmovdqa %xmm6, (%rcx)609; AVX2-NEXT: vmovdqa %xmm1, (%r8)610; AVX2-NEXT: vzeroupper611; AVX2-NEXT: retq612;613; AVX2-FP-LABEL: load_i16_stride4_vf8:614; AVX2-FP: # %bb.0:615; AVX2-FP-NEXT: vpxor %xmm0, %xmm0, %xmm0616; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]617; AVX2-FP-NEXT: vextracti128 $1, %ymm1, %xmm2618; AVX2-FP-NEXT: vpackusdw %xmm2, %xmm1, %xmm1619; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]620; AVX2-FP-NEXT: vextracti128 $1, %ymm0, %xmm2621; AVX2-FP-NEXT: vpackusdw %xmm2, %xmm0, %xmm0622; AVX2-FP-NEXT: vpackusdw %xmm1, %xmm0, %xmm0623; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm1624; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm2625; AVX2-FP-NEXT: vmovdqa 32(%rdi), %xmm3626; AVX2-FP-NEXT: vmovdqa 48(%rdi), %xmm4627; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm5 = [0,1,2,3,2,3,10,11,8,9,10,11,12,13,14,15]628; AVX2-FP-NEXT: vpshufb %xmm5, %xmm4, %xmm6629; AVX2-FP-NEXT: vpshufb %xmm5, %xmm3, %xmm5630; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]631; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm6 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]632; AVX2-FP-NEXT: vpshufb %xmm6, %xmm2, %xmm7633; AVX2-FP-NEXT: vpshufb %xmm6, %xmm1, %xmm6634; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]635; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]636; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]637; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm6 = xmm4[0,1,2,0,4,5,6,7]638; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]639; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm7 = xmm3[0,1,2,0,4,5,6,7]640; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]641; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]642; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm7 = xmm2[2,0,2,3,4,5,6,7]643; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]644; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm8 = xmm1[2,0,2,3,4,5,6,7]645; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]646; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3]647; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]648; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]649; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]650; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]651; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]652; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]653; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3]654; AVX2-FP-NEXT: vmovdqa %xmm0, (%rsi)655; AVX2-FP-NEXT: vmovdqa %xmm5, (%rdx)656; AVX2-FP-NEXT: vmovdqa %xmm6, (%rcx)657; AVX2-FP-NEXT: vmovdqa %xmm1, (%r8)658; AVX2-FP-NEXT: vzeroupper659; AVX2-FP-NEXT: retq660;661; AVX2-FCP-LABEL: load_i16_stride4_vf8:662; AVX2-FCP: # %bb.0:663; AVX2-FCP-NEXT: vpxor %xmm0, %xmm0, %xmm0664; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]665; AVX2-FCP-NEXT: vextracti128 $1, %ymm1, %xmm2666; AVX2-FCP-NEXT: vpackusdw %xmm2, %xmm1, %xmm1667; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]668; AVX2-FCP-NEXT: vextracti128 $1, %ymm0, %xmm2669; AVX2-FCP-NEXT: vpackusdw %xmm2, %xmm0, %xmm0670; AVX2-FCP-NEXT: vpackusdw %xmm1, %xmm0, %xmm0671; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm1672; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm2673; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %xmm3674; AVX2-FCP-NEXT: vmovdqa 48(%rdi), %xmm4675; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm5 = [0,1,2,3,2,3,10,11,8,9,10,11,12,13,14,15]676; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm4, %xmm6677; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm3, %xmm5678; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]679; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm6 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]680; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm2, %xmm7681; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm1, %xmm6682; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]683; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]684; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]685; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm6 = xmm4[0,1,2,0,4,5,6,7]686; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]687; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm7 = xmm3[0,1,2,0,4,5,6,7]688; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]689; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]690; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm7 = xmm2[2,0,2,3,4,5,6,7]691; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]692; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm8 = xmm1[2,0,2,3,4,5,6,7]693; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]694; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3]695; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]696; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]697; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]698; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]699; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]700; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]701; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3]702; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rsi)703; AVX2-FCP-NEXT: vmovdqa %xmm5, (%rdx)704; AVX2-FCP-NEXT: vmovdqa %xmm6, (%rcx)705; AVX2-FCP-NEXT: vmovdqa %xmm1, (%r8)706; AVX2-FCP-NEXT: vzeroupper707; AVX2-FCP-NEXT: retq708;709; AVX512-LABEL: load_i16_stride4_vf8:710; AVX512: # %bb.0:711; AVX512-NEXT: vmovdqa64 (%rdi), %zmm0712; AVX512-NEXT: vpsrlq $16, %zmm0, %zmm1713; AVX512-NEXT: vpsrlq $32, %zmm0, %zmm2714; AVX512-NEXT: vpsrlq $48, %zmm0, %zmm3715; AVX512-NEXT: vpmovqw %zmm0, (%rsi)716; AVX512-NEXT: vpmovqw %zmm1, (%rdx)717; AVX512-NEXT: vpmovqw %zmm2, (%rcx)718; AVX512-NEXT: vpmovqw %zmm3, (%r8)719; AVX512-NEXT: vzeroupper720; AVX512-NEXT: retq721;722; AVX512-FCP-LABEL: load_i16_stride4_vf8:723; AVX512-FCP: # %bb.0:724; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm0725; AVX512-FCP-NEXT: vpsrlq $16, %zmm0, %zmm1726; AVX512-FCP-NEXT: vpsrlq $32, %zmm0, %zmm2727; AVX512-FCP-NEXT: vpsrlq $48, %zmm0, %zmm3728; AVX512-FCP-NEXT: vpmovqw %zmm0, (%rsi)729; AVX512-FCP-NEXT: vpmovqw %zmm1, (%rdx)730; AVX512-FCP-NEXT: vpmovqw %zmm2, (%rcx)731; AVX512-FCP-NEXT: vpmovqw %zmm3, (%r8)732; AVX512-FCP-NEXT: vzeroupper733; AVX512-FCP-NEXT: retq734;735; AVX512DQ-LABEL: load_i16_stride4_vf8:736; AVX512DQ: # %bb.0:737; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm0738; AVX512DQ-NEXT: vpsrlq $16, %zmm0, %zmm1739; AVX512DQ-NEXT: vpsrlq $32, %zmm0, %zmm2740; AVX512DQ-NEXT: vpsrlq $48, %zmm0, %zmm3741; AVX512DQ-NEXT: vpmovqw %zmm0, (%rsi)742; AVX512DQ-NEXT: vpmovqw %zmm1, (%rdx)743; AVX512DQ-NEXT: vpmovqw %zmm2, (%rcx)744; AVX512DQ-NEXT: vpmovqw %zmm3, (%r8)745; AVX512DQ-NEXT: vzeroupper746; AVX512DQ-NEXT: retq747;748; AVX512DQ-FCP-LABEL: load_i16_stride4_vf8:749; AVX512DQ-FCP: # %bb.0:750; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm0751; AVX512DQ-FCP-NEXT: vpsrlq $16, %zmm0, %zmm1752; AVX512DQ-FCP-NEXT: vpsrlq $32, %zmm0, %zmm2753; AVX512DQ-FCP-NEXT: vpsrlq $48, %zmm0, %zmm3754; AVX512DQ-FCP-NEXT: vpmovqw %zmm0, (%rsi)755; AVX512DQ-FCP-NEXT: vpmovqw %zmm1, (%rdx)756; AVX512DQ-FCP-NEXT: vpmovqw %zmm2, (%rcx)757; AVX512DQ-FCP-NEXT: vpmovqw %zmm3, (%r8)758; AVX512DQ-FCP-NEXT: vzeroupper759; AVX512DQ-FCP-NEXT: retq760;761; AVX512BW-LABEL: load_i16_stride4_vf8:762; AVX512BW: # %bb.0:763; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0764; AVX512BW-NEXT: vpsrlq $16, %zmm0, %zmm1765; AVX512BW-NEXT: vpsrlq $32, %zmm0, %zmm2766; AVX512BW-NEXT: vpsrlq $48, %zmm0, %zmm3767; AVX512BW-NEXT: vpmovqw %zmm0, (%rsi)768; AVX512BW-NEXT: vpmovqw %zmm1, (%rdx)769; AVX512BW-NEXT: vpmovqw %zmm2, (%rcx)770; AVX512BW-NEXT: vpmovqw %zmm3, (%r8)771; AVX512BW-NEXT: vzeroupper772; AVX512BW-NEXT: retq773;774; AVX512BW-FCP-LABEL: load_i16_stride4_vf8:775; AVX512BW-FCP: # %bb.0:776; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0777; AVX512BW-FCP-NEXT: vpsrlq $16, %zmm0, %zmm1778; AVX512BW-FCP-NEXT: vpsrlq $32, %zmm0, %zmm2779; AVX512BW-FCP-NEXT: vpsrlq $48, %zmm0, %zmm3780; AVX512BW-FCP-NEXT: vpmovqw %zmm0, (%rsi)781; AVX512BW-FCP-NEXT: vpmovqw %zmm1, (%rdx)782; AVX512BW-FCP-NEXT: vpmovqw %zmm2, (%rcx)783; AVX512BW-FCP-NEXT: vpmovqw %zmm3, (%r8)784; AVX512BW-FCP-NEXT: vzeroupper785; AVX512BW-FCP-NEXT: retq786;787; AVX512DQ-BW-LABEL: load_i16_stride4_vf8:788; AVX512DQ-BW: # %bb.0:789; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm0790; AVX512DQ-BW-NEXT: vpsrlq $16, %zmm0, %zmm1791; AVX512DQ-BW-NEXT: vpsrlq $32, %zmm0, %zmm2792; AVX512DQ-BW-NEXT: vpsrlq $48, %zmm0, %zmm3793; AVX512DQ-BW-NEXT: vpmovqw %zmm0, (%rsi)794; AVX512DQ-BW-NEXT: vpmovqw %zmm1, (%rdx)795; AVX512DQ-BW-NEXT: vpmovqw %zmm2, (%rcx)796; AVX512DQ-BW-NEXT: vpmovqw %zmm3, (%r8)797; AVX512DQ-BW-NEXT: vzeroupper798; AVX512DQ-BW-NEXT: retq799;800; AVX512DQ-BW-FCP-LABEL: load_i16_stride4_vf8:801; AVX512DQ-BW-FCP: # %bb.0:802; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm0803; AVX512DQ-BW-FCP-NEXT: vpsrlq $16, %zmm0, %zmm1804; AVX512DQ-BW-FCP-NEXT: vpsrlq $32, %zmm0, %zmm2805; AVX512DQ-BW-FCP-NEXT: vpsrlq $48, %zmm0, %zmm3806; AVX512DQ-BW-FCP-NEXT: vpmovqw %zmm0, (%rsi)807; AVX512DQ-BW-FCP-NEXT: vpmovqw %zmm1, (%rdx)808; AVX512DQ-BW-FCP-NEXT: vpmovqw %zmm2, (%rcx)809; AVX512DQ-BW-FCP-NEXT: vpmovqw %zmm3, (%r8)810; AVX512DQ-BW-FCP-NEXT: vzeroupper811; AVX512DQ-BW-FCP-NEXT: retq812 %wide.vec = load <32 x i16>, ptr %in.vec, align 64813 %strided.vec0 = shufflevector <32 x i16> %wide.vec, <32 x i16> poison, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>814 %strided.vec1 = shufflevector <32 x i16> %wide.vec, <32 x i16> poison, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>815 %strided.vec2 = shufflevector <32 x i16> %wide.vec, <32 x i16> poison, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>816 %strided.vec3 = shufflevector <32 x i16> %wide.vec, <32 x i16> poison, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>817 store <8 x i16> %strided.vec0, ptr %out.vec0, align 64818 store <8 x i16> %strided.vec1, ptr %out.vec1, align 64819 store <8 x i16> %strided.vec2, ptr %out.vec2, align 64820 store <8 x i16> %strided.vec3, ptr %out.vec3, align 64821 ret void822}823 824define void @load_i16_stride4_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3) nounwind {825; SSE-LABEL: load_i16_stride4_vf16:826; SSE: # %bb.0:827; SSE-NEXT: movdqa 96(%rdi), %xmm4828; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill829; SSE-NEXT: movdqa 64(%rdi), %xmm1830; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill831; SSE-NEXT: movdqa 80(%rdi), %xmm3832; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill833; SSE-NEXT: movdqa (%rdi), %xmm8834; SSE-NEXT: movdqa 16(%rdi), %xmm10835; SSE-NEXT: movdqa 32(%rdi), %xmm7836; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill837; SSE-NEXT: movdqa 48(%rdi), %xmm6838; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm10[0,2,2,3]839; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm5[0,2,2,3,4,5,6,7]840; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,2,2,3]841; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm9[0,2,2,3,4,5,6,7]842; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]843; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm6[0,2,2,3]844; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm12[0,1,0,2,4,5,6,7]845; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm7[0,2,2,3]846; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm13[0,1,0,2,4,5,6,7]847; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm11[0],xmm7[1],xmm11[1]848; SSE-NEXT: movsd {{.*#+}} xmm7 = xmm2[0],xmm7[1]849; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm3[0,2,2,3]850; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm14[0,2,2,3,4,5,6,7]851; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm1[0,2,2,3]852; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm15[0,2,2,3,4,5,6,7]853; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]854; SSE-NEXT: movdqa 112(%rdi), %xmm11855; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm11[0,2,2,3]856; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm3[0,1,0,2,4,5,6,7]857; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]858; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm2[0,1,0,2,4,5,6,7]859; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]860; SSE-NEXT: movsd {{.*#+}} xmm4 = xmm0[0],xmm4[1]861; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm5[1,3,2,3,4,5,6,7]862; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm9[1,3,2,3,4,5,6,7]863; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]864; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm12[0,1,1,3,4,5,6,7]865; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm13[0,1,1,3,4,5,6,7]866; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]867; SSE-NEXT: movsd {{.*#+}} xmm5 = xmm1[0],xmm5[1]868; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm14[1,3,2,3,4,5,6,7]869; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm15[1,3,2,3,4,5,6,7]870; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]871; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm3[0,1,1,3,4,5,6,7]872; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm2[0,1,1,3,4,5,6,7]873; SSE-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm0[0],xmm9[1],xmm0[1]874; SSE-NEXT: movsd {{.*#+}} xmm9 = xmm1[0],xmm9[1]875; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm10[3,1,2,3]876; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm0[2,0,2,3,4,5,6,7]877; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm8[3,1,2,3]878; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm2[2,0,2,3,4,5,6,7]879; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]880; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm6[3,1,2,3]881; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm1[0,1,2,0,4,5,6,7]882; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload883; SSE-NEXT: # xmm10 = mem[3,1,2,3]884; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm10[0,1,2,0,4,5,6,7]885; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]886; SSE-NEXT: movsd {{.*#+}} xmm6 = xmm3[0],xmm6[1]887; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload888; SSE-NEXT: # xmm3 = mem[3,1,2,3]889; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm3[2,0,2,3,4,5,6,7]890; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload891; SSE-NEXT: # xmm12 = mem[3,1,2,3]892; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm12[2,0,2,3,4,5,6,7]893; SSE-NEXT: punpckldq {{.*#+}} xmm13 = xmm13[0],xmm8[0],xmm13[1],xmm8[1]894; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm11[3,1,2,3]895; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm8[0,1,2,0,4,5,6,7]896; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload897; SSE-NEXT: # xmm14 = mem[3,1,2,3]898; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm14[0,1,2,0,4,5,6,7]899; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm11[0],xmm15[1],xmm11[1]900; SSE-NEXT: movsd {{.*#+}} xmm15 = xmm13[0],xmm15[1]901; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7]902; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]903; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]904; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm1[0,1,3,1,4,5,6,7]905; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm10[0,1,3,1,4,5,6,7]906; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]907; SSE-NEXT: movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]908; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm3[3,1,2,3,4,5,6,7]909; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm12[3,1,2,3,4,5,6,7]910; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]911; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm8[0,1,3,1,4,5,6,7]912; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm14[0,1,3,1,4,5,6,7]913; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]914; SSE-NEXT: movsd {{.*#+}} xmm3 = xmm2[0],xmm3[1]915; SSE-NEXT: movapd %xmm4, 16(%rsi)916; SSE-NEXT: movapd %xmm7, (%rsi)917; SSE-NEXT: movapd %xmm9, 16(%rdx)918; SSE-NEXT: movapd %xmm5, (%rdx)919; SSE-NEXT: movapd %xmm15, 16(%rcx)920; SSE-NEXT: movapd %xmm6, (%rcx)921; SSE-NEXT: movapd %xmm3, 16(%r8)922; SSE-NEXT: movapd %xmm1, (%r8)923; SSE-NEXT: retq924;925; AVX-LABEL: load_i16_stride4_vf16:926; AVX: # %bb.0:927; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1928; AVX-NEXT: vmovdqa 112(%rdi), %xmm5929; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm5[0],xmm1[1,2,3],xmm5[4],xmm1[5,6,7]930; AVX-NEXT: vmovdqa 96(%rdi), %xmm6931; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm6[0],xmm1[1,2,3],xmm6[4],xmm1[5,6,7]932; AVX-NEXT: vpackusdw %xmm0, %xmm2, %xmm0933; AVX-NEXT: vmovdqa 80(%rdi), %xmm7934; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm7[0],xmm1[1,2,3],xmm7[4],xmm1[5,6,7]935; AVX-NEXT: vmovdqa 64(%rdi), %xmm8936; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm8[0],xmm1[1,2,3],xmm8[4],xmm1[5,6,7]937; AVX-NEXT: vpackusdw %xmm2, %xmm3, %xmm2938; AVX-NEXT: vpackusdw %xmm0, %xmm2, %xmm0939; AVX-NEXT: vmovdqa (%rdi), %xmm2940; AVX-NEXT: vmovdqa 16(%rdi), %xmm3941; AVX-NEXT: vmovdqa 32(%rdi), %xmm4942; AVX-NEXT: vmovdqa 48(%rdi), %xmm9943; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm9[0],xmm1[1,2,3],xmm9[4],xmm1[5,6,7]944; AVX-NEXT: vpblendw {{.*#+}} xmm11 = xmm4[0],xmm1[1,2,3],xmm4[4],xmm1[5,6,7]945; AVX-NEXT: vpackusdw %xmm10, %xmm11, %xmm10946; AVX-NEXT: vpblendw {{.*#+}} xmm11 = xmm3[0],xmm1[1,2,3],xmm3[4],xmm1[5,6,7]947; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3],xmm2[4],xmm1[5,6,7]948; AVX-NEXT: vpackusdw %xmm11, %xmm1, %xmm1949; AVX-NEXT: vpackusdw %xmm10, %xmm1, %xmm1950; AVX-NEXT: vpshufd {{.*#+}} xmm10 = xmm5[0,2,2,3]951; AVX-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]952; AVX-NEXT: vpshufd {{.*#+}} xmm11 = xmm6[0,2,2,3]953; AVX-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[0,1,1,3,4,5,6,7]954; AVX-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]955; AVX-NEXT: vpshufd {{.*#+}} xmm11 = xmm7[0,2,2,3]956; AVX-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7]957; AVX-NEXT: vpshufd {{.*#+}} xmm12 = xmm8[0,2,2,3]958; AVX-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[1,3,2,3,4,5,6,7]959; AVX-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]960; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1,2,3],xmm10[4,5,6,7]961; AVX-NEXT: vinsertf128 $1, %xmm10, %ymm0, %ymm10962; AVX-NEXT: vpshufd {{.*#+}} xmm11 = xmm9[0,2,2,3]963; AVX-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[0,1,1,3,4,5,6,7]964; AVX-NEXT: vpshufd {{.*#+}} xmm12 = xmm4[0,2,2,3]965; AVX-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[0,1,1,3,4,5,6,7]966; AVX-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]967; AVX-NEXT: vpshufd {{.*#+}} xmm12 = xmm3[0,2,2,3]968; AVX-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[1,3,2,3,4,5,6,7]969; AVX-NEXT: vpshufd {{.*#+}} xmm13 = xmm2[0,2,2,3]970; AVX-NEXT: vpshuflw {{.*#+}} xmm13 = xmm13[1,3,2,3,4,5,6,7]971; AVX-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]972; AVX-NEXT: vpblendw {{.*#+}} xmm11 = xmm12[0,1,2,3],xmm11[4,5,6,7]973; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7]974; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]975; AVX-NEXT: vpshuflw {{.*#+}} xmm11 = xmm5[0,1,2,0,4,5,6,7]976; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,1,2,3]977; AVX-NEXT: vpshuflw {{.*#+}} xmm12 = xmm6[0,1,2,0,4,5,6,7]978; AVX-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]979; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,1,2,3]980; AVX-NEXT: vpshuflw {{.*#+}} xmm12 = xmm7[2,0,2,3,4,5,6,7]981; AVX-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,1,2,3]982; AVX-NEXT: vpshuflw {{.*#+}} xmm13 = xmm8[2,0,2,3,4,5,6,7]983; AVX-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]984; AVX-NEXT: vpblendw {{.*#+}} xmm11 = xmm12[0,1,2,3],xmm11[4,5,6,7]985; AVX-NEXT: vinsertf128 $1, %xmm11, %ymm0, %ymm11986; AVX-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[3,1,2,3]987; AVX-NEXT: vpshuflw {{.*#+}} xmm12 = xmm9[0,1,2,0,4,5,6,7]988; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]989; AVX-NEXT: vpshuflw {{.*#+}} xmm13 = xmm4[0,1,2,0,4,5,6,7]990; AVX-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]991; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]992; AVX-NEXT: vpshuflw {{.*#+}} xmm13 = xmm3[2,0,2,3,4,5,6,7]993; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]994; AVX-NEXT: vpshuflw {{.*#+}} xmm14 = xmm2[2,0,2,3,4,5,6,7]995; AVX-NEXT: vpunpckldq {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1]996; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm13[0,1,2,3],xmm12[4,5,6,7]997; AVX-NEXT: vblendps {{.*#+}} ymm11 = ymm12[0,1,2,3],ymm11[4,5,6,7]998; AVX-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]999; AVX-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,3,1,4,5,6,7]1000; AVX-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]1001; AVX-NEXT: vpshuflw {{.*#+}} xmm6 = xmm7[3,1,2,3,4,5,6,7]1002; AVX-NEXT: vpshuflw {{.*#+}} xmm7 = xmm8[3,1,2,3,4,5,6,7]1003; AVX-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]1004; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1,2,3],xmm5[4,5,6,7]1005; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm51006; AVX-NEXT: vpshuflw {{.*#+}} xmm6 = xmm9[0,1,3,1,4,5,6,7]1007; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]1008; AVX-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]1009; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7]1010; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]1011; AVX-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]1012; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm4[4,5,6,7]1013; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm5[4,5,6,7]1014; AVX-NEXT: vmovdqa %xmm1, (%rsi)1015; AVX-NEXT: vmovdqa %xmm0, 16(%rsi)1016; AVX-NEXT: vmovaps %ymm10, (%rdx)1017; AVX-NEXT: vmovaps %ymm11, (%rcx)1018; AVX-NEXT: vmovaps %ymm2, (%r8)1019; AVX-NEXT: vzeroupper1020; AVX-NEXT: retq1021;1022; AVX2-LABEL: load_i16_stride4_vf16:1023; AVX2: # %bb.0:1024; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm01025; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]1026; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm21027; AVX2-NEXT: vpackusdw %xmm2, %xmm1, %xmm11028; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]1029; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm31030; AVX2-NEXT: vpackusdw %xmm3, %xmm2, %xmm21031; AVX2-NEXT: vpackusdw %xmm1, %xmm2, %xmm11032; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]1033; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm31034; AVX2-NEXT: vpackusdw %xmm3, %xmm2, %xmm21035; AVX2-NEXT: vpackusdw %xmm2, %xmm2, %xmm21036; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm21037; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]1038; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm31039; AVX2-NEXT: vpackusdw %xmm3, %xmm0, %xmm01040; AVX2-NEXT: vpackusdw %xmm0, %xmm0, %xmm01041; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm01042; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]1043; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]1044; AVX2-NEXT: vmovdqa (%rdi), %xmm11045; AVX2-NEXT: vmovdqa 16(%rdi), %xmm21046; AVX2-NEXT: vmovdqa 32(%rdi), %xmm31047; AVX2-NEXT: vmovdqa 48(%rdi), %xmm51048; AVX2-NEXT: vmovdqa 112(%rdi), %xmm61049; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm6[0,2,2,3]1050; AVX2-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7]1051; AVX2-NEXT: vmovdqa 96(%rdi), %xmm71052; AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[0,2,2,3]1053; AVX2-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,1,1,3,4,5,6,7]1054; AVX2-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm8[0],xmm4[0],xmm8[1],xmm4[1]1055; AVX2-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm41056; AVX2-NEXT: vmovdqa 80(%rdi), %xmm81057; AVX2-NEXT: vpshufd {{.*#+}} xmm9 = xmm8[0,2,2,3]1058; AVX2-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7]1059; AVX2-NEXT: vmovdqa 64(%rdi), %xmm101060; AVX2-NEXT: vpshufd {{.*#+}} xmm11 = xmm10[0,2,2,3]1061; AVX2-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7]1062; AVX2-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]1063; AVX2-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm91064; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5],ymm4[6,7]1065; AVX2-NEXT: vpshufd {{.*#+}} xmm9 = xmm5[0,2,2,3]1066; AVX2-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]1067; AVX2-NEXT: vpshufd {{.*#+}} xmm11 = xmm3[0,2,2,3]1068; AVX2-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[0,1,1,3,4,5,6,7]1069; AVX2-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]1070; AVX2-NEXT: vpshufd {{.*#+}} xmm11 = xmm2[0,2,2,3]1071; AVX2-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7]1072; AVX2-NEXT: vpshufd {{.*#+}} xmm12 = xmm1[0,2,2,3]1073; AVX2-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[1,3,2,3,4,5,6,7]1074; AVX2-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]1075; AVX2-NEXT: vpblendd {{.*#+}} xmm9 = xmm11[0,1],xmm9[2,3]1076; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3],ymm4[4,5,6,7]1077; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,1,2,3]1078; AVX2-NEXT: vpshuflw {{.*#+}} xmm9 = xmm6[0,1,2,0,4,5,6,7]1079; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,1,2,3]1080; AVX2-NEXT: vpshuflw {{.*#+}} xmm11 = xmm7[0,1,2,0,4,5,6,7]1081; AVX2-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]1082; AVX2-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm91083; AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,1,2,3]1084; AVX2-NEXT: vpshuflw {{.*#+}} xmm11 = xmm8[2,0,2,3,4,5,6,7]1085; AVX2-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[3,1,2,3]1086; AVX2-NEXT: vpshuflw {{.*#+}} xmm12 = xmm10[2,0,2,3,4,5,6,7]1087; AVX2-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]1088; AVX2-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm111089; AVX2-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3,4,5],ymm9[6,7]1090; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]1091; AVX2-NEXT: vpshuflw {{.*#+}} xmm11 = xmm5[0,1,2,0,4,5,6,7]1092; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]1093; AVX2-NEXT: vpshuflw {{.*#+}} xmm12 = xmm3[0,1,2,0,4,5,6,7]1094; AVX2-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]1095; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]1096; AVX2-NEXT: vpshuflw {{.*#+}} xmm12 = xmm2[2,0,2,3,4,5,6,7]1097; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]1098; AVX2-NEXT: vpshuflw {{.*#+}} xmm13 = xmm1[2,0,2,3,4,5,6,7]1099; AVX2-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]1100; AVX2-NEXT: vpblendd {{.*#+}} xmm11 = xmm12[0,1],xmm11[2,3]1101; AVX2-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3],ymm9[4,5,6,7]1102; AVX2-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,3,1,4,5,6,7]1103; AVX2-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,3,1,4,5,6,7]1104; AVX2-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]1105; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm61106; AVX2-NEXT: vpshuflw {{.*#+}} xmm7 = xmm8[3,1,2,3,4,5,6,7]1107; AVX2-NEXT: vpshuflw {{.*#+}} xmm8 = xmm10[3,1,2,3,4,5,6,7]1108; AVX2-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]1109; AVX2-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm71110; AVX2-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3,4,5],ymm6[6,7]1111; AVX2-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]1112; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]1113; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]1114; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]1115; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]1116; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]1117; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3]1118; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm6[4,5,6,7]1119; AVX2-NEXT: vmovdqa %ymm0, (%rsi)1120; AVX2-NEXT: vmovdqa %ymm4, (%rdx)1121; AVX2-NEXT: vmovdqa %ymm9, (%rcx)1122; AVX2-NEXT: vmovdqa %ymm1, (%r8)1123; AVX2-NEXT: vzeroupper1124; AVX2-NEXT: retq1125;1126; AVX2-FP-LABEL: load_i16_stride4_vf16:1127; AVX2-FP: # %bb.0:1128; AVX2-FP-NEXT: vpxor %xmm0, %xmm0, %xmm01129; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]1130; AVX2-FP-NEXT: vextracti128 $1, %ymm1, %xmm21131; AVX2-FP-NEXT: vpackusdw %xmm2, %xmm1, %xmm11132; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]1133; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm31134; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm2, %xmm21135; AVX2-FP-NEXT: vpackusdw %xmm1, %xmm2, %xmm11136; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]1137; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm31138; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm2, %xmm21139; AVX2-FP-NEXT: vpackusdw %xmm2, %xmm2, %xmm21140; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm21141; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]1142; AVX2-FP-NEXT: vextracti128 $1, %ymm0, %xmm31143; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm0, %xmm01144; AVX2-FP-NEXT: vpackusdw %xmm0, %xmm0, %xmm01145; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm01146; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]1147; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]1148; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm11149; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm31150; AVX2-FP-NEXT: vmovdqa 32(%rdi), %xmm41151; AVX2-FP-NEXT: vmovdqa 48(%rdi), %xmm51152; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,2,3,10,11,8,9,10,11,12,13,14,15]1153; AVX2-FP-NEXT: vmovdqa 112(%rdi), %xmm61154; AVX2-FP-NEXT: vpshufb %xmm2, %xmm6, %xmm71155; AVX2-FP-NEXT: vmovdqa 96(%rdi), %xmm81156; AVX2-FP-NEXT: vpshufb %xmm2, %xmm8, %xmm91157; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1]1158; AVX2-FP-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm71159; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm9 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]1160; AVX2-FP-NEXT: vmovdqa 80(%rdi), %xmm101161; AVX2-FP-NEXT: vpshufb %xmm9, %xmm10, %xmm111162; AVX2-FP-NEXT: vmovdqa 64(%rdi), %xmm121163; AVX2-FP-NEXT: vpshufb %xmm9, %xmm12, %xmm131164; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1]1165; AVX2-FP-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm111166; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm7 = ymm11[0,1,2,3,4,5],ymm7[6,7]1167; AVX2-FP-NEXT: vpshufb %xmm2, %xmm5, %xmm111168; AVX2-FP-NEXT: vpshufb %xmm2, %xmm4, %xmm21169; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm11[0],xmm2[1],xmm11[1]1170; AVX2-FP-NEXT: vpshufb %xmm9, %xmm3, %xmm111171; AVX2-FP-NEXT: vpshufb %xmm9, %xmm1, %xmm91172; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]1173; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm2 = xmm9[0,1],xmm2[2,3]1174; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm7[4,5,6,7]1175; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,1,2,3]1176; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm7 = xmm6[0,1,2,0,4,5,6,7]1177; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,1,2,3]1178; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm9 = xmm8[0,1,2,0,4,5,6,7]1179; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1]1180; AVX2-FP-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm71181; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm9 = xmm10[3,1,2,3]1182; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm10 = xmm9[2,0,2,3,4,5,6,7]1183; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm11 = xmm12[3,1,2,3]1184; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm12 = xmm11[2,0,2,3,4,5,6,7]1185; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1]1186; AVX2-FP-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm101187; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm7 = ymm10[0,1,2,3,4,5],ymm7[6,7]1188; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]1189; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm10 = xmm5[0,1,2,0,4,5,6,7]1190; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]1191; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm12 = xmm4[0,1,2,0,4,5,6,7]1192; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1]1193; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]1194; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm12 = xmm3[2,0,2,3,4,5,6,7]1195; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]1196; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm13 = xmm1[2,0,2,3,4,5,6,7]1197; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]1198; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm10 = xmm12[0,1],xmm10[2,3]1199; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm7 = ymm10[0,1,2,3],ymm7[4,5,6,7]1200; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,3,1,4,5,6,7]1201; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,1,3,1,4,5,6,7]1202; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm8[0],xmm6[0],xmm8[1],xmm6[1]1203; AVX2-FP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm61204; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm8 = xmm9[3,1,2,3,4,5,6,7]1205; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm9 = xmm11[3,1,2,3,4,5,6,7]1206; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]1207; AVX2-FP-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm81208; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm6 = ymm8[0,1,2,3,4,5],ymm6[6,7]1209; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]1210; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]1211; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]1212; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7]1213; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]1214; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]1215; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3]1216; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm6[4,5,6,7]1217; AVX2-FP-NEXT: vmovdqa %ymm0, (%rsi)1218; AVX2-FP-NEXT: vmovdqa %ymm2, (%rdx)1219; AVX2-FP-NEXT: vmovdqa %ymm7, (%rcx)1220; AVX2-FP-NEXT: vmovdqa %ymm1, (%r8)1221; AVX2-FP-NEXT: vzeroupper1222; AVX2-FP-NEXT: retq1223;1224; AVX2-FCP-LABEL: load_i16_stride4_vf16:1225; AVX2-FCP: # %bb.0:1226; AVX2-FCP-NEXT: vmovdqa 64(%rdi), %ymm11227; AVX2-FCP-NEXT: vmovdqa 96(%rdi), %ymm21228; AVX2-FCP-NEXT: vpxor %xmm0, %xmm0, %xmm01229; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm3 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]1230; AVX2-FCP-NEXT: vextracti128 $1, %ymm3, %xmm41231; AVX2-FCP-NEXT: vpackusdw %xmm4, %xmm3, %xmm31232; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]1233; AVX2-FCP-NEXT: vextracti128 $1, %ymm0, %xmm41234; AVX2-FCP-NEXT: vpackusdw %xmm4, %xmm0, %xmm01235; AVX2-FCP-NEXT: vpackusdw %xmm3, %xmm0, %xmm01236; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [0,2,2,3,0,2,4,6]1237; AVX2-FCP-NEXT: vpermd %ymm2, %ymm3, %ymm41238; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm5 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,16,17,20,21,24,25,28,29]1239; AVX2-FCP-NEXT: vpshufb %ymm5, %ymm4, %ymm61240; AVX2-FCP-NEXT: vpermd %ymm1, %ymm3, %ymm31241; AVX2-FCP-NEXT: vpshufb %ymm5, %ymm3, %ymm71242; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3,4,5],ymm6[6,7]1243; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm6[4,5,6,7]1244; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm61245; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm71246; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %xmm81247; AVX2-FCP-NEXT: vmovdqa 48(%rdi), %xmm91248; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm10 = [0,1,2,3,2,3,10,11,8,9,10,11,12,13,14,15]1249; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm9, %xmm111250; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm8, %xmm101251; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1]1252; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm11 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]1253; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm7, %xmm121254; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm6, %xmm111255; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1]1256; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3]1257; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm11 = [2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]1258; AVX2-FCP-NEXT: vpshufb %ymm11, %ymm4, %ymm41259; AVX2-FCP-NEXT: vpshufb %ymm11, %ymm3, %ymm31260; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm4[6,7]1261; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm10[0,1,2,3],ymm3[4,5,6,7]1262; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [1,3,2,3,1,3,5,7]1263; AVX2-FCP-NEXT: vpermd %ymm2, %ymm4, %ymm21264; AVX2-FCP-NEXT: vpshufb %ymm5, %ymm2, %ymm101265; AVX2-FCP-NEXT: vpermd %ymm1, %ymm4, %ymm11266; AVX2-FCP-NEXT: vpshufb %ymm5, %ymm1, %ymm41267; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],ymm10[6,7]1268; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm5 = xmm9[3,1,2,3]1269; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm9 = xmm5[0,1,2,0,4,5,6,7]1270; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[3,1,2,3]1271; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm10 = xmm8[0,1,2,0,4,5,6,7]1272; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]1273; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,1,2,3]1274; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm10 = xmm7[2,0,2,3,4,5,6,7]1275; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,1,2,3]1276; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm12 = xmm6[2,0,2,3,4,5,6,7]1277; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1]1278; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]1279; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3],ymm4[4,5,6,7]1280; AVX2-FCP-NEXT: vpshufb %ymm11, %ymm2, %ymm21281; AVX2-FCP-NEXT: vpshufb %ymm11, %ymm1, %ymm11282; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]1283; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm5[0,1,3,1,4,5,6,7]1284; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm5 = xmm8[0,1,3,1,4,5,6,7]1285; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]1286; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm5 = xmm7[3,1,2,3,4,5,6,7]1287; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[3,1,2,3,4,5,6,7]1288; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]1289; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]1290; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]1291; AVX2-FCP-NEXT: vmovdqa %ymm0, (%rsi)1292; AVX2-FCP-NEXT: vmovdqa %ymm3, (%rdx)1293; AVX2-FCP-NEXT: vmovdqa %ymm4, (%rcx)1294; AVX2-FCP-NEXT: vmovdqa %ymm1, (%r8)1295; AVX2-FCP-NEXT: vzeroupper1296; AVX2-FCP-NEXT: retq1297;1298; AVX512-LABEL: load_i16_stride4_vf16:1299; AVX512: # %bb.0:1300; AVX512-NEXT: vmovdqa 64(%rdi), %ymm11301; AVX512-NEXT: vpmovqw %ymm1, %xmm01302; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm01303; AVX512-NEXT: vmovdqa 112(%rdi), %xmm21304; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]1305; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm3[0,1,0,2,4,5,6,7]1306; AVX512-NEXT: vmovdqa 96(%rdi), %xmm51307; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[0,2,2,3]1308; AVX512-NEXT: vpshuflw {{.*#+}} xmm7 = xmm6[0,1,0,2,4,5,6,7]1309; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm7[0],xmm4[0],xmm7[1],xmm4[1]1310; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm41311; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm4[6,7]1312; AVX512-NEXT: vmovdqa64 (%rdi), %zmm41313; AVX512-NEXT: vpmovqw %zmm4, %xmm71314; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm0[4,5,6,7]1315; AVX512-NEXT: vpsrlq $16, %ymm1, %ymm71316; AVX512-NEXT: vpmovqw %ymm7, %xmm71317; AVX512-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm71318; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]1319; AVX512-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]1320; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]1321; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm31322; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm7[0,1,2,3,4,5],ymm3[6,7]1323; AVX512-NEXT: vpsrlq $16, %zmm4, %zmm61324; AVX512-NEXT: vpmovqw %zmm6, %xmm61325; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3],ymm3[4,5,6,7]1326; AVX512-NEXT: vpsrlq $32, %ymm1, %ymm61327; AVX512-NEXT: vpmovqw %ymm6, %xmm61328; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm61329; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]1330; AVX512-NEXT: vpshuflw {{.*#+}} xmm7 = xmm2[0,1,2,0,4,5,6,7]1331; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]1332; AVX512-NEXT: vpshuflw {{.*#+}} xmm8 = xmm5[0,1,2,0,4,5,6,7]1333; AVX512-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]1334; AVX512-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm71335; AVX512-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],ymm7[6,7]1336; AVX512-NEXT: vpsrlq $32, %zmm4, %zmm71337; AVX512-NEXT: vpmovqw %zmm7, %xmm71338; AVX512-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7]1339; AVX512-NEXT: vpsrlq $48, %ymm1, %ymm11340; AVX512-NEXT: vpmovqw %ymm1, %xmm11341; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm11342; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7]1343; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]1344; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]1345; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm21346; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]1347; AVX512-NEXT: vpsrlq $48, %zmm4, %zmm21348; AVX512-NEXT: vpmovqw %zmm2, %xmm21349; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]1350; AVX512-NEXT: vmovdqa %ymm0, (%rsi)1351; AVX512-NEXT: vmovdqa %ymm3, (%rdx)1352; AVX512-NEXT: vmovdqa %ymm6, (%rcx)1353; AVX512-NEXT: vmovdqa %ymm1, (%r8)1354; AVX512-NEXT: vzeroupper1355; AVX512-NEXT: retq1356;1357; AVX512-FCP-LABEL: load_i16_stride4_vf16:1358; AVX512-FCP: # %bb.0:1359; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [16,17,20,21,0,0,0,0,24,25,28,29,0,0,0,0,16,17,20,21,0,0,0,0,24,25,28,29,0,0,0,0]1360; AVX512-FCP-NEXT: # ymm0 = mem[0,1,0,1]1361; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,2,2,3,0,2,4,6]1362; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %ymm21363; AVX512-FCP-NEXT: vpermd %ymm2, %ymm1, %ymm31364; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm3, %ymm41365; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm5 = [0,1,4,5,u,u,u,u,8,9,12,13,12,13,14,15,16,17,20,21,u,u,u,u,24,25,28,29,u,u,u,u]1366; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm61367; AVX512-FCP-NEXT: vpermd %ymm6, %ymm1, %ymm11368; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm1, %ymm71369; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm8 = [0,2,2,3,4,6,12,14]1370; AVX512-FCP-NEXT: vpermt2d %ymm4, %ymm8, %ymm71371; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm41372; AVX512-FCP-NEXT: vpmovqw %zmm4, %xmm91373; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm9[0,1,2,3],ymm7[4,5,6,7]1374; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm9 = [2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]1375; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm3, %ymm31376; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm1, %ymm11377; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm3[6,7]1378; AVX512-FCP-NEXT: vpsrlq $16, %zmm4, %zmm31379; AVX512-FCP-NEXT: vpmovqw %zmm3, %xmm31380; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]1381; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [1,3,2,3,1,3,5,7]1382; AVX512-FCP-NEXT: vpermd %ymm2, %ymm3, %ymm21383; AVX512-FCP-NEXT: vpshufb %ymm0, %ymm2, %ymm01384; AVX512-FCP-NEXT: vpermd %ymm6, %ymm3, %ymm31385; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm3, %ymm51386; AVX512-FCP-NEXT: vpermt2d %ymm0, %ymm8, %ymm51387; AVX512-FCP-NEXT: vpsrlq $32, %zmm4, %zmm01388; AVX512-FCP-NEXT: vpmovqw %zmm0, %xmm01389; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]1390; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm2, %ymm21391; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm3, %ymm31392; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]1393; AVX512-FCP-NEXT: vpsrlq $48, %zmm4, %zmm31394; AVX512-FCP-NEXT: vpmovqw %zmm3, %xmm31395; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]1396; AVX512-FCP-NEXT: vmovdqa %ymm7, (%rsi)1397; AVX512-FCP-NEXT: vmovdqa %ymm1, (%rdx)1398; AVX512-FCP-NEXT: vmovdqa %ymm0, (%rcx)1399; AVX512-FCP-NEXT: vmovdqa %ymm2, (%r8)1400; AVX512-FCP-NEXT: vzeroupper1401; AVX512-FCP-NEXT: retq1402;1403; AVX512DQ-LABEL: load_i16_stride4_vf16:1404; AVX512DQ: # %bb.0:1405; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm11406; AVX512DQ-NEXT: vpmovqw %ymm1, %xmm01407; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm01408; AVX512DQ-NEXT: vmovdqa 112(%rdi), %xmm21409; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]1410; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm3[0,1,0,2,4,5,6,7]1411; AVX512DQ-NEXT: vmovdqa 96(%rdi), %xmm51412; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[0,2,2,3]1413; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm7 = xmm6[0,1,0,2,4,5,6,7]1414; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm7[0],xmm4[0],xmm7[1],xmm4[1]1415; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm41416; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm4[6,7]1417; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm41418; AVX512DQ-NEXT: vpmovqw %zmm4, %xmm71419; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm0[4,5,6,7]1420; AVX512DQ-NEXT: vpsrlq $16, %ymm1, %ymm71421; AVX512DQ-NEXT: vpmovqw %ymm7, %xmm71422; AVX512DQ-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm71423; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]1424; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]1425; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]1426; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm31427; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm7[0,1,2,3,4,5],ymm3[6,7]1428; AVX512DQ-NEXT: vpsrlq $16, %zmm4, %zmm61429; AVX512DQ-NEXT: vpmovqw %zmm6, %xmm61430; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3],ymm3[4,5,6,7]1431; AVX512DQ-NEXT: vpsrlq $32, %ymm1, %ymm61432; AVX512DQ-NEXT: vpmovqw %ymm6, %xmm61433; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm61434; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]1435; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm7 = xmm2[0,1,2,0,4,5,6,7]1436; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]1437; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm8 = xmm5[0,1,2,0,4,5,6,7]1438; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]1439; AVX512DQ-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm71440; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],ymm7[6,7]1441; AVX512DQ-NEXT: vpsrlq $32, %zmm4, %zmm71442; AVX512DQ-NEXT: vpmovqw %zmm7, %xmm71443; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7]1444; AVX512DQ-NEXT: vpsrlq $48, %ymm1, %ymm11445; AVX512DQ-NEXT: vpmovqw %ymm1, %xmm11446; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm11447; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7]1448; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]1449; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]1450; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm21451; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]1452; AVX512DQ-NEXT: vpsrlq $48, %zmm4, %zmm21453; AVX512DQ-NEXT: vpmovqw %zmm2, %xmm21454; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]1455; AVX512DQ-NEXT: vmovdqa %ymm0, (%rsi)1456; AVX512DQ-NEXT: vmovdqa %ymm3, (%rdx)1457; AVX512DQ-NEXT: vmovdqa %ymm6, (%rcx)1458; AVX512DQ-NEXT: vmovdqa %ymm1, (%r8)1459; AVX512DQ-NEXT: vzeroupper1460; AVX512DQ-NEXT: retq1461;1462; AVX512DQ-FCP-LABEL: load_i16_stride4_vf16:1463; AVX512DQ-FCP: # %bb.0:1464; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [16,17,20,21,0,0,0,0,24,25,28,29,0,0,0,0,16,17,20,21,0,0,0,0,24,25,28,29,0,0,0,0]1465; AVX512DQ-FCP-NEXT: # ymm0 = mem[0,1,0,1]1466; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,2,2,3,0,2,4,6]1467; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %ymm21468; AVX512DQ-FCP-NEXT: vpermd %ymm2, %ymm1, %ymm31469; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm3, %ymm41470; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm5 = [0,1,4,5,u,u,u,u,8,9,12,13,12,13,14,15,16,17,20,21,u,u,u,u,24,25,28,29,u,u,u,u]1471; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm61472; AVX512DQ-FCP-NEXT: vpermd %ymm6, %ymm1, %ymm11473; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm1, %ymm71474; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm8 = [0,2,2,3,4,6,12,14]1475; AVX512DQ-FCP-NEXT: vpermt2d %ymm4, %ymm8, %ymm71476; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm41477; AVX512DQ-FCP-NEXT: vpmovqw %zmm4, %xmm91478; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm9[0,1,2,3],ymm7[4,5,6,7]1479; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm9 = [2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]1480; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm3, %ymm31481; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm1, %ymm11482; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm3[6,7]1483; AVX512DQ-FCP-NEXT: vpsrlq $16, %zmm4, %zmm31484; AVX512DQ-FCP-NEXT: vpmovqw %zmm3, %xmm31485; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]1486; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [1,3,2,3,1,3,5,7]1487; AVX512DQ-FCP-NEXT: vpermd %ymm2, %ymm3, %ymm21488; AVX512DQ-FCP-NEXT: vpshufb %ymm0, %ymm2, %ymm01489; AVX512DQ-FCP-NEXT: vpermd %ymm6, %ymm3, %ymm31490; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm3, %ymm51491; AVX512DQ-FCP-NEXT: vpermt2d %ymm0, %ymm8, %ymm51492; AVX512DQ-FCP-NEXT: vpsrlq $32, %zmm4, %zmm01493; AVX512DQ-FCP-NEXT: vpmovqw %zmm0, %xmm01494; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]1495; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm2, %ymm21496; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm3, %ymm31497; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]1498; AVX512DQ-FCP-NEXT: vpsrlq $48, %zmm4, %zmm31499; AVX512DQ-FCP-NEXT: vpmovqw %zmm3, %xmm31500; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]1501; AVX512DQ-FCP-NEXT: vmovdqa %ymm7, (%rsi)1502; AVX512DQ-FCP-NEXT: vmovdqa %ymm1, (%rdx)1503; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, (%rcx)1504; AVX512DQ-FCP-NEXT: vmovdqa %ymm2, (%r8)1505; AVX512DQ-FCP-NEXT: vzeroupper1506; AVX512DQ-FCP-NEXT: retq1507;1508; AVX512BW-LABEL: load_i16_stride4_vf16:1509; AVX512BW: # %bb.0:1510; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm0 = [0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60]1511; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm11512; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm21513; AVX512BW-NEXT: vpermi2w %zmm2, %zmm1, %zmm01514; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm3 = [1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61]1515; AVX512BW-NEXT: vpermi2w %zmm2, %zmm1, %zmm31516; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm4 = [2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62]1517; AVX512BW-NEXT: vpermi2w %zmm2, %zmm1, %zmm41518; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm5 = [3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63]1519; AVX512BW-NEXT: vpermi2w %zmm2, %zmm1, %zmm51520; AVX512BW-NEXT: vmovdqa %ymm0, (%rsi)1521; AVX512BW-NEXT: vmovdqa %ymm3, (%rdx)1522; AVX512BW-NEXT: vmovdqa %ymm4, (%rcx)1523; AVX512BW-NEXT: vmovdqa %ymm5, (%r8)1524; AVX512BW-NEXT: vzeroupper1525; AVX512BW-NEXT: retq1526;1527; AVX512BW-FCP-LABEL: load_i16_stride4_vf16:1528; AVX512BW-FCP: # %bb.0:1529; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm0 = [0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60]1530; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm11531; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm21532; AVX512BW-FCP-NEXT: vpermi2w %zmm2, %zmm1, %zmm01533; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm3 = [1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61]1534; AVX512BW-FCP-NEXT: vpermi2w %zmm2, %zmm1, %zmm31535; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm4 = [2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62]1536; AVX512BW-FCP-NEXT: vpermi2w %zmm2, %zmm1, %zmm41537; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm5 = [3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63]1538; AVX512BW-FCP-NEXT: vpermi2w %zmm2, %zmm1, %zmm51539; AVX512BW-FCP-NEXT: vmovdqa %ymm0, (%rsi)1540; AVX512BW-FCP-NEXT: vmovdqa %ymm3, (%rdx)1541; AVX512BW-FCP-NEXT: vmovdqa %ymm4, (%rcx)1542; AVX512BW-FCP-NEXT: vmovdqa %ymm5, (%r8)1543; AVX512BW-FCP-NEXT: vzeroupper1544; AVX512BW-FCP-NEXT: retq1545;1546; AVX512DQ-BW-LABEL: load_i16_stride4_vf16:1547; AVX512DQ-BW: # %bb.0:1548; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm0 = [0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60]1549; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm11550; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %zmm21551; AVX512DQ-BW-NEXT: vpermi2w %zmm2, %zmm1, %zmm01552; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm3 = [1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61]1553; AVX512DQ-BW-NEXT: vpermi2w %zmm2, %zmm1, %zmm31554; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm4 = [2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62]1555; AVX512DQ-BW-NEXT: vpermi2w %zmm2, %zmm1, %zmm41556; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm5 = [3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63]1557; AVX512DQ-BW-NEXT: vpermi2w %zmm2, %zmm1, %zmm51558; AVX512DQ-BW-NEXT: vmovdqa %ymm0, (%rsi)1559; AVX512DQ-BW-NEXT: vmovdqa %ymm3, (%rdx)1560; AVX512DQ-BW-NEXT: vmovdqa %ymm4, (%rcx)1561; AVX512DQ-BW-NEXT: vmovdqa %ymm5, (%r8)1562; AVX512DQ-BW-NEXT: vzeroupper1563; AVX512DQ-BW-NEXT: retq1564;1565; AVX512DQ-BW-FCP-LABEL: load_i16_stride4_vf16:1566; AVX512DQ-BW-FCP: # %bb.0:1567; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm0 = [0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60]1568; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm11569; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm21570; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm2, %zmm1, %zmm01571; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm3 = [1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61]1572; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm2, %zmm1, %zmm31573; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm4 = [2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62]1574; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm2, %zmm1, %zmm41575; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm5 = [3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63]1576; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm2, %zmm1, %zmm51577; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm0, (%rsi)1578; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm3, (%rdx)1579; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm4, (%rcx)1580; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm5, (%r8)1581; AVX512DQ-BW-FCP-NEXT: vzeroupper1582; AVX512DQ-BW-FCP-NEXT: retq1583 %wide.vec = load <64 x i16>, ptr %in.vec, align 641584 %strided.vec0 = shufflevector <64 x i16> %wide.vec, <64 x i16> poison, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>1585 %strided.vec1 = shufflevector <64 x i16> %wide.vec, <64 x i16> poison, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>1586 %strided.vec2 = shufflevector <64 x i16> %wide.vec, <64 x i16> poison, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>1587 %strided.vec3 = shufflevector <64 x i16> %wide.vec, <64 x i16> poison, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>1588 store <16 x i16> %strided.vec0, ptr %out.vec0, align 641589 store <16 x i16> %strided.vec1, ptr %out.vec1, align 641590 store <16 x i16> %strided.vec2, ptr %out.vec2, align 641591 store <16 x i16> %strided.vec3, ptr %out.vec3, align 641592 ret void1593}1594 1595define void @load_i16_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3) nounwind {1596; SSE-LABEL: load_i16_stride4_vf32:1597; SSE: # %bb.0:1598; SSE-NEXT: subq $248, %rsp1599; SSE-NEXT: movdqa 160(%rdi), %xmm31600; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1601; SSE-NEXT: movdqa 128(%rdi), %xmm41602; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1603; SSE-NEXT: movdqa 144(%rdi), %xmm51604; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1605; SSE-NEXT: movdqa 96(%rdi), %xmm21606; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1607; SSE-NEXT: movdqa 112(%rdi), %xmm61608; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1609; SSE-NEXT: movdqa 64(%rdi), %xmm11610; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1611; SSE-NEXT: movdqa 80(%rdi), %xmm01612; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1613; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]1614; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1615; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]1616; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]1617; SSE-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill1618; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]1619; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]1620; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,2,2,3]1621; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1622; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]1623; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm2[0,2,2,3]1624; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm13[0,1,0,2,4,5,6,7]1625; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]1626; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]1627; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1628; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm5[0,2,2,3]1629; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm15[0,2,2,3,4,5,6,7]1630; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm4[0,2,2,3]1631; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm11[0,2,2,3,4,5,6,7]1632; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]1633; SSE-NEXT: movdqa 176(%rdi), %xmm01634; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1635; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm0[0,2,2,3]1636; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm10[0,1,0,2,4,5,6,7]1637; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm3[0,2,2,3]1638; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm9[0,1,0,2,4,5,6,7]1639; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]1640; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]1641; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1642; SSE-NEXT: movdqa (%rdi), %xmm11643; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1644; SSE-NEXT: movdqa 16(%rdi), %xmm01645; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1646; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm0[0,2,2,3]1647; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm8[0,2,2,3,4,5,6,7]1648; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm1[0,2,2,3]1649; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm12[0,2,2,3,4,5,6,7]1650; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]1651; SSE-NEXT: movdqa 32(%rdi), %xmm21652; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1653; SSE-NEXT: movdqa 48(%rdi), %xmm01654; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1655; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,2,2,3]1656; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm7[0,1,0,2,4,5,6,7]1657; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm2[0,2,2,3]1658; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm6[0,1,0,2,4,5,6,7]1659; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]1660; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]1661; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1662; SSE-NEXT: movdqa 192(%rdi), %xmm01663; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1664; SSE-NEXT: movdqa 208(%rdi), %xmm11665; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1666; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,2,2,3]1667; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm5[0,2,2,3,4,5,6,7]1668; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,2,2,3]1669; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm4[0,2,2,3,4,5,6,7]1670; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]1671; SSE-NEXT: movdqa 224(%rdi), %xmm21672; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1673; SSE-NEXT: movdqa 240(%rdi), %xmm11674; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1675; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,2,2,3]1676; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm3[0,1,0,2,4,5,6,7]1677; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]1678; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm2[0,1,0,2,4,5,6,7]1679; SSE-NEXT: punpckldq {{.*#+}} xmm14 = xmm14[0],xmm1[0],xmm14[1],xmm1[1]1680; SSE-NEXT: movsd {{.*#+}} xmm14 = xmm0[0],xmm14[1]1681; SSE-NEXT: movapd %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1682; SSE-NEXT: pshuflw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload1683; SSE-NEXT: # xmm0 = mem[1,3,2,3,4,5,6,7]1684; SSE-NEXT: pshuflw $237, (%rsp), %xmm1 # 16-byte Folded Reload1685; SSE-NEXT: # xmm1 = mem[1,3,2,3,4,5,6,7]1686; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]1687; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload1688; SSE-NEXT: # xmm0 = mem[0,1,1,3,4,5,6,7]1689; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm13[0,1,1,3,4,5,6,7]1690; SSE-NEXT: punpckldq {{.*#+}} xmm13 = xmm13[0],xmm0[0],xmm13[1],xmm0[1]1691; SSE-NEXT: movsd {{.*#+}} xmm13 = xmm1[0],xmm13[1]1692; SSE-NEXT: movapd %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1693; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm15[1,3,2,3,4,5,6,7]1694; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm11[1,3,2,3,4,5,6,7]1695; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]1696; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm10[0,1,1,3,4,5,6,7]1697; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]1698; SSE-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm0[0],xmm9[1],xmm0[1]1699; SSE-NEXT: movsd {{.*#+}} xmm9 = xmm1[0],xmm9[1]1700; SSE-NEXT: movapd %xmm9, (%rsp) # 16-byte Spill1701; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm8[1,3,2,3,4,5,6,7]1702; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm12[1,3,2,3,4,5,6,7]1703; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]1704; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm7[0,1,1,3,4,5,6,7]1705; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm6[0,1,1,3,4,5,6,7]1706; SSE-NEXT: punpckldq {{.*#+}} xmm13 = xmm13[0],xmm0[0],xmm13[1],xmm0[1]1707; SSE-NEXT: movsd {{.*#+}} xmm13 = xmm1[0],xmm13[1]1708; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm5[1,3,2,3,4,5,6,7]1709; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm4[1,3,2,3,4,5,6,7]1710; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]1711; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm3[0,1,1,3,4,5,6,7]1712; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm2[0,1,1,3,4,5,6,7]1713; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]1714; SSE-NEXT: movsd {{.*#+}} xmm15 = xmm1[0],xmm15[1]1715; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload1716; SSE-NEXT: # xmm0 = mem[3,1,2,3]1717; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1718; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload1719; SSE-NEXT: # xmm1 = mem[3,1,2,3]1720; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1721; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]1722; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]1723; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]1724; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload1725; SSE-NEXT: # xmm0 = mem[3,1,2,3]1726; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1727; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload1728; SSE-NEXT: # xmm9 = mem[3,1,2,3]1729; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]1730; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm9[0,1,2,0,4,5,6,7]1731; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]1732; SSE-NEXT: movsd {{.*#+}} xmm5 = xmm1[0],xmm5[1]1733; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload1734; SSE-NEXT: # xmm0 = mem[3,1,2,3]1735; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1736; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload1737; SSE-NEXT: # xmm1 = mem[3,1,2,3]1738; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1739; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]1740; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]1741; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]1742; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload1743; SSE-NEXT: # xmm0 = mem[3,1,2,3]1744; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1745; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload1746; SSE-NEXT: # xmm14 = mem[3,1,2,3]1747; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]1748; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm14[0,1,2,0,4,5,6,7]1749; SSE-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm0[0],xmm11[1],xmm0[1]1750; SSE-NEXT: movsd {{.*#+}} xmm11 = xmm1[0],xmm11[1]1751; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload1752; SSE-NEXT: # xmm0 = mem[3,1,2,3]1753; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1754; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload1755; SSE-NEXT: # xmm1 = mem[3,1,2,3]1756; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1757; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]1758; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm1[2,0,2,3,4,5,6,7]1759; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]1760; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload1761; SSE-NEXT: # xmm12 = mem[3,1,2,3]1762; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload1763; SSE-NEXT: # xmm10 = mem[3,1,2,3]1764; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm12[0,1,2,0,4,5,6,7]1765; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm10[0,1,2,0,4,5,6,7]1766; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]1767; SSE-NEXT: movsd {{.*#+}} xmm4 = xmm3[0],xmm4[1]1768; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload1769; SSE-NEXT: # xmm8 = mem[3,1,2,3]1770; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload1771; SSE-NEXT: # xmm7 = mem[3,1,2,3]1772; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm8[2,0,2,3,4,5,6,7]1773; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm7[2,0,2,3,4,5,6,7]1774; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]1775; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload1776; SSE-NEXT: # xmm6 = mem[3,1,2,3]1777; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload1778; SSE-NEXT: # xmm3 = mem[3,1,2,3]1779; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm6[0,1,2,0,4,5,6,7]1780; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm3[0,1,2,0,4,5,6,7]1781; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]1782; SSE-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]1783; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload1784; SSE-NEXT: # xmm0 = mem[3,1,2,3,4,5,6,7]1785; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload1786; SSE-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7]1787; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]1788; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload1789; SSE-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]1790; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[0,1,3,1,4,5,6,7]1791; SSE-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm0[0],xmm9[1],xmm0[1]1792; SSE-NEXT: movsd {{.*#+}} xmm9 = xmm2[0],xmm9[1]1793; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload1794; SSE-NEXT: # xmm0 = mem[3,1,2,3,4,5,6,7]1795; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload1796; SSE-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7]1797; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]1798; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload1799; SSE-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]1800; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm14[0,1,3,1,4,5,6,7]1801; SSE-NEXT: punpckldq {{.*#+}} xmm14 = xmm14[0],xmm0[0],xmm14[1],xmm0[1]1802; SSE-NEXT: movsd {{.*#+}} xmm14 = xmm2[0],xmm14[1]1803; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload1804; SSE-NEXT: # xmm0 = mem[3,1,2,3,4,5,6,7]1805; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload1806; SSE-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7]1807; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]1808; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm12[0,1,3,1,4,5,6,7]1809; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[0,1,3,1,4,5,6,7]1810; SSE-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm0[0],xmm10[1],xmm0[1]1811; SSE-NEXT: movsd {{.*#+}} xmm10 = xmm2[0],xmm10[1]1812; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm8[3,1,2,3,4,5,6,7]1813; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm7[3,1,2,3,4,5,6,7]1814; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]1815; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm6[0,1,3,1,4,5,6,7]1816; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]1817; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]1818; SSE-NEXT: movsd {{.*#+}} xmm3 = xmm2[0],xmm3[1]1819; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1820; SSE-NEXT: movaps %xmm0, 48(%rsi)1821; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1822; SSE-NEXT: movaps %xmm0, (%rsi)1823; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1824; SSE-NEXT: movaps %xmm0, 32(%rsi)1825; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1826; SSE-NEXT: movaps %xmm0, 16(%rsi)1827; SSE-NEXT: movapd %xmm15, 48(%rdx)1828; SSE-NEXT: movapd %xmm13, (%rdx)1829; SSE-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload1830; SSE-NEXT: movaps %xmm0, 32(%rdx)1831; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1832; SSE-NEXT: movaps %xmm0, 16(%rdx)1833; SSE-NEXT: movapd %xmm1, 48(%rcx)1834; SSE-NEXT: movapd %xmm4, 32(%rcx)1835; SSE-NEXT: movapd %xmm11, 16(%rcx)1836; SSE-NEXT: movapd %xmm5, (%rcx)1837; SSE-NEXT: movapd %xmm3, 48(%r8)1838; SSE-NEXT: movapd %xmm10, 32(%r8)1839; SSE-NEXT: movapd %xmm14, 16(%r8)1840; SSE-NEXT: movapd %xmm9, (%r8)1841; SSE-NEXT: addq $248, %rsp1842; SSE-NEXT: retq1843;1844; AVX-LABEL: load_i16_stride4_vf32:1845; AVX: # %bb.0:1846; AVX-NEXT: subq $280, %rsp # imm = 0x1181847; AVX-NEXT: vpxor %xmm6, %xmm6, %xmm61848; AVX-NEXT: vmovdqa 112(%rdi), %xmm41849; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0],xmm6[1,2,3],xmm4[4],xmm6[5,6,7]1850; AVX-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1851; AVX-NEXT: vmovdqa 96(%rdi), %xmm111852; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm11[0],xmm6[1,2,3],xmm11[4],xmm6[5,6,7]1853; AVX-NEXT: vpackusdw %xmm2, %xmm3, %xmm21854; AVX-NEXT: vmovdqa 80(%rdi), %xmm31855; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm3[0],xmm6[1,2,3],xmm3[4],xmm6[5,6,7]1856; AVX-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1857; AVX-NEXT: vmovdqa 64(%rdi), %xmm01858; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1859; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm0[0],xmm6[1,2,3],xmm0[4],xmm6[5,6,7]1860; AVX-NEXT: vpackusdw %xmm5, %xmm7, %xmm51861; AVX-NEXT: vpackusdw %xmm2, %xmm5, %xmm21862; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1863; AVX-NEXT: vmovdqa (%rdi), %xmm101864; AVX-NEXT: vmovdqa 16(%rdi), %xmm01865; AVX-NEXT: vmovdqa 32(%rdi), %xmm11866; AVX-NEXT: vmovdqa 48(%rdi), %xmm21867; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1868; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm2[0],xmm6[1,2,3],xmm2[4],xmm6[5,6,7]1869; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm1[0],xmm6[1,2,3],xmm1[4],xmm6[5,6,7]1870; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1871; AVX-NEXT: vpackusdw %xmm7, %xmm8, %xmm71872; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm0[0],xmm6[1,2,3],xmm0[4],xmm6[5,6,7]1873; AVX-NEXT: vmovdqa %xmm0, %xmm21874; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1875; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm10[0],xmm6[1,2,3],xmm10[4],xmm6[5,6,7]1876; AVX-NEXT: vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1877; AVX-NEXT: vpackusdw %xmm8, %xmm9, %xmm81878; AVX-NEXT: vpackusdw %xmm7, %xmm8, %xmm71879; AVX-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1880; AVX-NEXT: vmovdqa 240(%rdi), %xmm01881; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1882; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm0[0],xmm6[1,2,3],xmm0[4],xmm6[5,6,7]1883; AVX-NEXT: vmovdqa 224(%rdi), %xmm121884; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm12[0],xmm6[1,2,3],xmm12[4],xmm6[5,6,7]1885; AVX-NEXT: vmovdqa %xmm12, (%rsp) # 16-byte Spill1886; AVX-NEXT: vpackusdw %xmm7, %xmm8, %xmm71887; AVX-NEXT: vmovdqa 208(%rdi), %xmm51888; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm5[0],xmm6[1,2,3],xmm5[4],xmm6[5,6,7]1889; AVX-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1890; AVX-NEXT: vmovdqa 192(%rdi), %xmm01891; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm0[0],xmm6[1,2,3],xmm0[4],xmm6[5,6,7]1892; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1893; AVX-NEXT: vpackusdw %xmm8, %xmm9, %xmm81894; AVX-NEXT: vpackusdw %xmm7, %xmm8, %xmm71895; AVX-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1896; AVX-NEXT: vmovdqa 176(%rdi), %xmm141897; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm14[0],xmm6[1,2,3],xmm14[4],xmm6[5,6,7]1898; AVX-NEXT: vmovdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1899; AVX-NEXT: vmovdqa 160(%rdi), %xmm131900; AVX-NEXT: vpblendw {{.*#+}} xmm15 = xmm13[0],xmm6[1,2,3],xmm13[4],xmm6[5,6,7]1901; AVX-NEXT: vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1902; AVX-NEXT: vpackusdw %xmm7, %xmm15, %xmm71903; AVX-NEXT: vmovdqa 144(%rdi), %xmm91904; AVX-NEXT: vpblendw {{.*#+}} xmm15 = xmm9[0],xmm6[1,2,3],xmm9[4],xmm6[5,6,7]1905; AVX-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1906; AVX-NEXT: vmovdqa 128(%rdi), %xmm81907; AVX-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1908; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm8[0],xmm6[1,2,3],xmm8[4],xmm6[5,6,7]1909; AVX-NEXT: vpackusdw %xmm15, %xmm6, %xmm61910; AVX-NEXT: vpackusdw %xmm7, %xmm6, %xmm61911; AVX-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1912; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[0,2,2,3]1913; AVX-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]1914; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm11[0,2,2,3]1915; AVX-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7]1916; AVX-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]1917; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm3[0,2,2,3]1918; AVX-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[1,3,2,3,4,5,6,7]1919; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload1920; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm3[0,2,2,3]1921; AVX-NEXT: vpshuflw {{.*#+}} xmm15 = xmm15[1,3,2,3,4,5,6,7]1922; AVX-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm15[0],xmm7[0],xmm15[1],xmm7[1]1923; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0,1,2,3],xmm6[4,5,6,7]1924; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload1925; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[0,2,2,3]1926; AVX-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7]1927; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm1[0,2,2,3]1928; AVX-NEXT: vpshuflw {{.*#+}} xmm15 = xmm15[0,1,1,3,4,5,6,7]1929; AVX-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm15[0],xmm7[0],xmm15[1],xmm7[1]1930; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm2[0,2,2,3]1931; AVX-NEXT: vpshuflw {{.*#+}} xmm15 = xmm15[1,3,2,3,4,5,6,7]1932; AVX-NEXT: vpshufd {{.*#+}} xmm8 = xmm10[0,2,2,3]1933; AVX-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[1,3,2,3,4,5,6,7]1934; AVX-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm15[0],xmm8[1],xmm15[1]1935; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm8[0,1,2,3],xmm7[4,5,6,7]1936; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm61937; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7]1938; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill1939; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload1940; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm10[0,2,2,3]1941; AVX-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]1942; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm12[0,2,2,3]1943; AVX-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7]1944; AVX-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]1945; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[0,2,2,3]1946; AVX-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[1,3,2,3,4,5,6,7]1947; AVX-NEXT: vpshufd {{.*#+}} xmm8 = xmm0[0,2,2,3]1948; AVX-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[1,3,2,3,4,5,6,7]1949; AVX-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]1950; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0,1,2,3],xmm6[4,5,6,7]1951; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm14[0,2,2,3]1952; AVX-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7]1953; AVX-NEXT: vpshufd {{.*#+}} xmm8 = xmm13[0,2,2,3]1954; AVX-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,1,1,3,4,5,6,7]1955; AVX-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]1956; AVX-NEXT: vpshufd {{.*#+}} xmm8 = xmm9[0,2,2,3]1957; AVX-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[1,3,2,3,4,5,6,7]1958; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload1959; AVX-NEXT: # xmm15 = mem[0,2,2,3]1960; AVX-NEXT: vpshuflw {{.*#+}} xmm15 = xmm15[1,3,2,3,4,5,6,7]1961; AVX-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm15[0],xmm8[0],xmm15[1],xmm8[1]1962; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm8[0,1,2,3],xmm7[4,5,6,7]1963; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm61964; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm6[4,5,6,7]1965; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill1966; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload1967; AVX-NEXT: # xmm0 = mem[3,1,2,3]1968; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1969; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm11[3,1,2,3]1970; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1971; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]1972; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]1973; AVX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]1974; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload1975; AVX-NEXT: # xmm1 = mem[3,1,2,3]1976; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1977; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[3,1,2,3]1978; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1979; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]1980; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[2,0,2,3,4,5,6,7]1981; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]1982; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]1983; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[3,1,2,3]1984; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload1985; AVX-NEXT: # xmm6 = mem[3,1,2,3]1986; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm7[0,1,2,0,4,5,6,7]1987; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm6[0,1,2,0,4,5,6,7]1988; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]1989; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload1990; AVX-NEXT: # xmm12 = mem[3,1,2,3]1991; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload1992; AVX-NEXT: # xmm5 = mem[3,1,2,3]1993; AVX-NEXT: vpshuflw {{.*#+}} xmm8 = xmm12[2,0,2,3,4,5,6,7]1994; AVX-NEXT: vpshuflw {{.*#+}} xmm9 = xmm5[2,0,2,3,4,5,6,7]1995; AVX-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]1996; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm8[0,1,2,3],xmm1[4,5,6,7]1997; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm01998; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]1999; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2000; AVX-NEXT: vpshufd {{.*#+}} xmm13 = xmm10[3,1,2,3]2001; AVX-NEXT: vpshufd $231, (%rsp), %xmm14 # 16-byte Folded Reload2002; AVX-NEXT: # xmm14 = mem[3,1,2,3]2003; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm13[0,1,2,0,4,5,6,7]2004; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm14[0,1,2,0,4,5,6,7]2005; AVX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]2006; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload2007; AVX-NEXT: # xmm10 = mem[3,1,2,3]2008; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload2009; AVX-NEXT: # xmm11 = mem[3,1,2,3]2010; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm10[2,0,2,3,4,5,6,7]2011; AVX-NEXT: vpshuflw {{.*#+}} xmm8 = xmm11[2,0,2,3,4,5,6,7]2012; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm8[0],xmm1[0],xmm8[1],xmm1[1]2013; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0,1,2,3],xmm0[4,5,6,7]2014; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload2015; AVX-NEXT: # xmm8 = mem[3,1,2,3]2016; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload2017; AVX-NEXT: # xmm9 = mem[3,1,2,3]2018; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm8[0,1,2,0,4,5,6,7]2019; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm9[0,1,2,0,4,5,6,7]2020; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]2021; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload2022; AVX-NEXT: # xmm4 = mem[3,1,2,3]2023; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload2024; AVX-NEXT: # xmm2 = mem[3,1,2,3]2025; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm4[2,0,2,3,4,5,6,7]2026; AVX-NEXT: vpshuflw {{.*#+}} xmm15 = xmm2[2,0,2,3,4,5,6,7]2027; AVX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]2028; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]2029; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm12030; AVX-NEXT: vblendps {{.*#+}} ymm15 = ymm0[0,1,2,3],ymm1[4,5,6,7]2031; AVX-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload2032; AVX-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]2033; AVX-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload2034; AVX-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]2035; AVX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]2036; AVX-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload2037; AVX-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]2038; AVX-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload2039; AVX-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]2040; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]2041; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]2042; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm7[0,1,3,1,4,5,6,7]2043; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm6[0,1,3,1,4,5,6,7]2044; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]2045; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm12[3,1,2,3,4,5,6,7]2046; AVX-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[3,1,2,3,4,5,6,7]2047; AVX-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]2048; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]2049; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm02050; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]2051; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm13[0,1,3,1,4,5,6,7]2052; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm14[0,1,3,1,4,5,6,7]2053; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]2054; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm10[3,1,2,3,4,5,6,7]2055; AVX-NEXT: vpshuflw {{.*#+}} xmm5 = xmm11[3,1,2,3,4,5,6,7]2056; AVX-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]2057; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]2058; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm8[0,1,3,1,4,5,6,7]2059; AVX-NEXT: vpshuflw {{.*#+}} xmm5 = xmm9[0,1,3,1,4,5,6,7]2060; AVX-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]2061; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7]2062; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]2063; AVX-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]2064; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4,5,6,7]2065; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm12066; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]2067; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload2068; AVX-NEXT: vmovaps %xmm2, 32(%rsi)2069; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload2070; AVX-NEXT: vmovaps %xmm2, 48(%rsi)2071; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload2072; AVX-NEXT: vmovaps %xmm2, (%rsi)2073; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload2074; AVX-NEXT: vmovaps %xmm2, 16(%rsi)2075; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload2076; AVX-NEXT: vmovaps %ymm2, 32(%rdx)2077; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload2078; AVX-NEXT: vmovaps %ymm2, (%rdx)2079; AVX-NEXT: vmovaps %ymm15, 32(%rcx)2080; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload2081; AVX-NEXT: vmovaps %ymm2, (%rcx)2082; AVX-NEXT: vmovaps %ymm1, 32(%r8)2083; AVX-NEXT: vmovaps %ymm0, (%r8)2084; AVX-NEXT: addq $280, %rsp # imm = 0x1182085; AVX-NEXT: vzeroupper2086; AVX-NEXT: retq2087;2088; AVX2-LABEL: load_i16_stride4_vf32:2089; AVX2: # %bb.0:2090; AVX2-NEXT: subq $168, %rsp2091; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm02092; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]2093; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm22094; AVX2-NEXT: vpackusdw %xmm2, %xmm1, %xmm12095; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]2096; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm32097; AVX2-NEXT: vpackusdw %xmm3, %xmm2, %xmm22098; AVX2-NEXT: vpackusdw %xmm1, %xmm2, %xmm12099; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]2100; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm32101; AVX2-NEXT: vpackusdw %xmm3, %xmm2, %xmm22102; AVX2-NEXT: vpackusdw %xmm2, %xmm2, %xmm22103; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm22104; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]2105; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm42106; AVX2-NEXT: vpackusdw %xmm4, %xmm3, %xmm32107; AVX2-NEXT: vpackusdw %xmm3, %xmm3, %xmm32108; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm32109; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]2110; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]2111; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2112; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]2113; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm22114; AVX2-NEXT: vpackusdw %xmm2, %xmm1, %xmm12115; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]2116; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm32117; AVX2-NEXT: vpackusdw %xmm3, %xmm2, %xmm22118; AVX2-NEXT: vpackusdw %xmm1, %xmm2, %xmm12119; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]2120; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm32121; AVX2-NEXT: vpackusdw %xmm3, %xmm2, %xmm22122; AVX2-NEXT: vpackusdw %xmm2, %xmm2, %xmm22123; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm22124; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]2125; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm32126; AVX2-NEXT: vpackusdw %xmm3, %xmm0, %xmm02127; AVX2-NEXT: vpackusdw %xmm0, %xmm0, %xmm02128; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm02129; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]2130; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]2131; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2132; AVX2-NEXT: vmovdqa (%rdi), %xmm12133; AVX2-NEXT: vmovdqa %xmm1, (%rsp) # 16-byte Spill2134; AVX2-NEXT: vmovdqa 16(%rdi), %xmm52135; AVX2-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2136; AVX2-NEXT: vmovdqa 32(%rdi), %xmm62137; AVX2-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2138; AVX2-NEXT: vmovdqa 48(%rdi), %xmm72139; AVX2-NEXT: vmovdqa 112(%rdi), %xmm92140; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm9[0,2,2,3]2141; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]2142; AVX2-NEXT: vmovdqa 96(%rdi), %xmm132143; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm13[0,2,2,3]2144; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7]2145; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]2146; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm02147; AVX2-NEXT: vmovdqa 80(%rdi), %xmm142148; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm14[0,2,2,3]2149; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[1,3,2,3,4,5,6,7]2150; AVX2-NEXT: vmovdqa 64(%rdi), %xmm22151; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[0,2,2,3]2152; AVX2-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[1,3,2,3,4,5,6,7]2153; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]2154; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm32155; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]2156; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm7[0,2,2,3]2157; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]2158; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm6[0,2,2,3]2159; AVX2-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7]2160; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]2161; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm5[0,2,2,3]2162; AVX2-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[1,3,2,3,4,5,6,7]2163; AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm1[0,2,2,3]2164; AVX2-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[1,3,2,3,4,5,6,7]2165; AVX2-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm8[0],xmm4[0],xmm8[1],xmm4[1]2166; AVX2-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]2167; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]2168; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2169; AVX2-NEXT: vmovdqa 240(%rdi), %xmm02170; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2171; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]2172; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]2173; AVX2-NEXT: vmovdqa 224(%rdi), %xmm102174; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm10[0,2,2,3]2175; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]2176; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]2177; AVX2-NEXT: vmovdqa 208(%rdi), %xmm112178; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm11[0,2,2,3]2179; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,3,2,3,4,5,6,7]2180; AVX2-NEXT: vmovdqa 192(%rdi), %xmm152181; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm15[0,2,2,3]2182; AVX2-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[1,3,2,3,4,5,6,7]2183; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]2184; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm02185; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm32186; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]2187; AVX2-NEXT: vmovdqa 176(%rdi), %xmm122188; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm12[0,2,2,3]2189; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]2190; AVX2-NEXT: vmovdqa 160(%rdi), %xmm42191; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm4[0,2,2,3]2192; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]2193; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]2194; AVX2-NEXT: vmovdqa 144(%rdi), %xmm32195; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[0,2,2,3]2196; AVX2-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7]2197; AVX2-NEXT: vmovdqa 128(%rdi), %xmm62198; AVX2-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2199; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[0,2,2,3]2200; AVX2-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[1,3,2,3,4,5,6,7]2201; AVX2-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]2202; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm5[0,1],xmm1[2,3]2203; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]2204; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2205; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm9[3,1,2,3]2206; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2207; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm13[3,1,2,3]2208; AVX2-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2209; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]2210; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]2211; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]2212; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm14[3,1,2,3]2213; AVX2-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2214; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]2215; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2216; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]2217; AVX2-NEXT: vpshuflw {{.*#+}} xmm5 = xmm2[2,0,2,3,4,5,6,7]2218; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]2219; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm02220; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm12221; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3,4,5],ymm0[6,7]2222; AVX2-NEXT: vpshufd {{.*#+}} xmm13 = xmm7[3,1,2,3]2223; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload2224; AVX2-NEXT: # xmm9 = mem[3,1,2,3]2225; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm13[0,1,2,0,4,5,6,7]2226; AVX2-NEXT: vpshuflw {{.*#+}} xmm5 = xmm9[0,1,2,0,4,5,6,7]2227; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]2228; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload2229; AVX2-NEXT: # xmm7 = mem[3,1,2,3]2230; AVX2-NEXT: vpshufd $231, (%rsp), %xmm6 # 16-byte Folded Reload2231; AVX2-NEXT: # xmm6 = mem[3,1,2,3]2232; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm7[2,0,2,3,4,5,6,7]2233; AVX2-NEXT: vpshuflw {{.*#+}} xmm8 = xmm6[2,0,2,3,4,5,6,7]2234; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm8[0],xmm0[0],xmm8[1],xmm0[1]2235; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]2236; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]2237; AVX2-NEXT: vmovdqu %ymm0, (%rsp) # 32-byte Spill2238; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload2239; AVX2-NEXT: # xmm8 = mem[3,1,2,3]2240; AVX2-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[3,1,2,3]2241; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm8[0,1,2,0,4,5,6,7]2242; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm10[0,1,2,0,4,5,6,7]2243; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]2244; AVX2-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[3,1,2,3]2245; AVX2-NEXT: vpshufd {{.*#+}} xmm15 = xmm15[3,1,2,3]2246; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm11[2,0,2,3,4,5,6,7]2247; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm15[2,0,2,3,4,5,6,7]2248; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]2249; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm02250; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm12251; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm0[6,7]2252; AVX2-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[3,1,2,3]2253; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[3,1,2,3]2254; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm12[0,1,2,0,4,5,6,7]2255; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm5[0,1,2,0,4,5,6,7]2256; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]2257; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[3,1,2,3]2258; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload2259; AVX2-NEXT: # xmm3 = mem[3,1,2,3]2260; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm4[2,0,2,3,4,5,6,7]2261; AVX2-NEXT: vpshuflw {{.*#+}} xmm14 = xmm3[2,0,2,3,4,5,6,7]2262; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1]2263; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]2264; AVX2-NEXT: vpblendd {{.*#+}} ymm14 = ymm0[0,1,2,3],ymm1[4,5,6,7]2265; AVX2-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload2266; AVX2-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]2267; AVX2-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload2268; AVX2-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]2269; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]2270; AVX2-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload2271; AVX2-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]2272; AVX2-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload2273; AVX2-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7]2274; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]2275; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm02276; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm12277; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]2278; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm13[0,1,3,1,4,5,6,7]2279; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm9[0,1,3,1,4,5,6,7]2280; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]2281; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm7[3,1,2,3,4,5,6,7]2282; AVX2-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[3,1,2,3,4,5,6,7]2283; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1]2284; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]2285; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]2286; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm8[0,1,3,1,4,5,6,7]2287; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm10[0,1,3,1,4,5,6,7]2288; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]2289; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm11[3,1,2,3,4,5,6,7]2290; AVX2-NEXT: vpshuflw {{.*#+}} xmm6 = xmm15[3,1,2,3,4,5,6,7]2291; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1]2292; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm12293; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm22294; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]2295; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm12[0,1,3,1,4,5,6,7]2296; AVX2-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]2297; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]2298; AVX2-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7]2299; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7]2300; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]2301; AVX2-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]2302; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]2303; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload2304; AVX2-NEXT: vmovaps %ymm2, 32(%rsi)2305; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload2306; AVX2-NEXT: vmovaps %ymm2, (%rsi)2307; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload2308; AVX2-NEXT: vmovaps %ymm2, 32(%rdx)2309; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload2310; AVX2-NEXT: vmovaps %ymm2, (%rdx)2311; AVX2-NEXT: vmovdqa %ymm14, 32(%rcx)2312; AVX2-NEXT: vmovups (%rsp), %ymm2 # 32-byte Reload2313; AVX2-NEXT: vmovaps %ymm2, (%rcx)2314; AVX2-NEXT: vmovdqa %ymm1, 32(%r8)2315; AVX2-NEXT: vmovdqa %ymm0, (%r8)2316; AVX2-NEXT: addq $168, %rsp2317; AVX2-NEXT: vzeroupper2318; AVX2-NEXT: retq2319;2320; AVX2-FP-LABEL: load_i16_stride4_vf32:2321; AVX2-FP: # %bb.0:2322; AVX2-FP-NEXT: subq $184, %rsp2323; AVX2-FP-NEXT: vpxor %xmm0, %xmm0, %xmm02324; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]2325; AVX2-FP-NEXT: vextracti128 $1, %ymm1, %xmm22326; AVX2-FP-NEXT: vpackusdw %xmm2, %xmm1, %xmm12327; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]2328; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm32329; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm2, %xmm22330; AVX2-FP-NEXT: vpackusdw %xmm1, %xmm2, %xmm12331; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]2332; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm32333; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm2, %xmm22334; AVX2-FP-NEXT: vpackusdw %xmm2, %xmm2, %xmm22335; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm22336; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]2337; AVX2-FP-NEXT: vextracti128 $1, %ymm3, %xmm42338; AVX2-FP-NEXT: vpackusdw %xmm4, %xmm3, %xmm32339; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm3, %xmm32340; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm32341; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]2342; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]2343; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2344; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]2345; AVX2-FP-NEXT: vextracti128 $1, %ymm1, %xmm22346; AVX2-FP-NEXT: vpackusdw %xmm2, %xmm1, %xmm12347; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]2348; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm32349; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm2, %xmm22350; AVX2-FP-NEXT: vpackusdw %xmm1, %xmm2, %xmm12351; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]2352; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm32353; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm2, %xmm22354; AVX2-FP-NEXT: vpackusdw %xmm2, %xmm2, %xmm22355; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm22356; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]2357; AVX2-FP-NEXT: vextracti128 $1, %ymm0, %xmm32358; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm0, %xmm02359; AVX2-FP-NEXT: vpackusdw %xmm0, %xmm0, %xmm02360; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm02361; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]2362; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]2363; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2364; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm132365; AVX2-FP-NEXT: vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2366; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm122367; AVX2-FP-NEXT: vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2368; AVX2-FP-NEXT: vmovdqa 32(%rdi), %xmm142369; AVX2-FP-NEXT: vmovdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2370; AVX2-FP-NEXT: vmovdqa 48(%rdi), %xmm72371; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm10 = [0,1,2,3,2,3,10,11,8,9,10,11,12,13,14,15]2372; AVX2-FP-NEXT: vmovdqa 112(%rdi), %xmm22373; AVX2-FP-NEXT: vpshufb %xmm10, %xmm2, %xmm02374; AVX2-FP-NEXT: vmovdqa 96(%rdi), %xmm42375; AVX2-FP-NEXT: vpshufb %xmm10, %xmm4, %xmm12376; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]2377; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm32378; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm11 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]2379; AVX2-FP-NEXT: vmovdqa 80(%rdi), %xmm52380; AVX2-FP-NEXT: vpshufb %xmm11, %xmm5, %xmm82381; AVX2-FP-NEXT: vmovdqa 64(%rdi), %xmm62382; AVX2-FP-NEXT: vpshufb %xmm11, %xmm6, %xmm92383; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]2384; AVX2-FP-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm82385; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm8[0,1,2,3,4,5],ymm3[6,7]2386; AVX2-FP-NEXT: vpshufb %xmm10, %xmm7, %xmm82387; AVX2-FP-NEXT: vpshufb %xmm10, %xmm14, %xmm92388; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]2389; AVX2-FP-NEXT: vpshufb %xmm11, %xmm12, %xmm92390; AVX2-FP-NEXT: vpshufb %xmm11, %xmm13, %xmm122391; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm12[0],xmm9[0],xmm12[1],xmm9[1]2392; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]2393; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm3[4,5,6,7]2394; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2395; AVX2-FP-NEXT: vmovdqa 240(%rdi), %xmm142396; AVX2-FP-NEXT: vpshufb %xmm10, %xmm14, %xmm92397; AVX2-FP-NEXT: vmovdqa 224(%rdi), %xmm82398; AVX2-FP-NEXT: vpshufb %xmm10, %xmm8, %xmm122399; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm12[0],xmm9[0],xmm12[1],xmm9[1]2400; AVX2-FP-NEXT: vmovdqa 208(%rdi), %xmm02401; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2402; AVX2-FP-NEXT: vpshufb %xmm11, %xmm0, %xmm122403; AVX2-FP-NEXT: vmovdqa 192(%rdi), %xmm152404; AVX2-FP-NEXT: vpshufb %xmm11, %xmm15, %xmm132405; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]2406; AVX2-FP-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm92407; AVX2-FP-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm122408; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3,4,5],ymm9[6,7]2409; AVX2-FP-NEXT: vmovdqa 160(%rdi), %xmm02410; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2411; AVX2-FP-NEXT: vmovdqa 176(%rdi), %xmm132412; AVX2-FP-NEXT: vpshufb %xmm10, %xmm13, %xmm122413; AVX2-FP-NEXT: vpshufb %xmm10, %xmm0, %xmm102414; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm12[0],xmm10[1],xmm12[1]2415; AVX2-FP-NEXT: vmovdqa 128(%rdi), %xmm32416; AVX2-FP-NEXT: vmovdqa %xmm3, (%rsp) # 16-byte Spill2417; AVX2-FP-NEXT: vmovdqa 144(%rdi), %xmm122418; AVX2-FP-NEXT: vpshufb %xmm11, %xmm12, %xmm02419; AVX2-FP-NEXT: vpshufb %xmm11, %xmm3, %xmm112420; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm11[0],xmm0[0],xmm11[1],xmm0[1]2421; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm10[2,3]2422; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]2423; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2424; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm0 = xmm2[3,1,2,3]2425; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2426; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm1 = xmm4[3,1,2,3]2427; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2428; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]2429; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]2430; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]2431; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm1 = xmm5[3,1,2,3]2432; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2433; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm2 = xmm6[3,1,2,3]2434; AVX2-FP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2435; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]2436; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]2437; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]2438; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm02439; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm12440; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm1[0,1,2,3,4,5],ymm0[6,7]2441; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,1,2,3]2442; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload2443; AVX2-FP-NEXT: # xmm6 = mem[3,1,2,3]2444; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm7[0,1,2,0,4,5,6,7]2445; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm6[0,1,2,0,4,5,6,7]2446; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]2447; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload2448; AVX2-FP-NEXT: # xmm5 = mem[3,1,2,3]2449; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload2450; AVX2-FP-NEXT: # xmm4 = mem[3,1,2,3]2451; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm5[2,0,2,3,4,5,6,7]2452; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm9 = xmm4[2,0,2,3,4,5,6,7]2453; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm9[0],xmm0[0],xmm9[1],xmm0[1]2454; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]2455; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]2456; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2457; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm2 = xmm14[3,1,2,3]2458; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm3 = xmm8[3,1,2,3]2459; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm2[0,1,2,0,4,5,6,7]2460; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm8 = xmm3[0,1,2,0,4,5,6,7]2461; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm8[0],xmm0[0],xmm8[1],xmm0[1]2462; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload2463; AVX2-FP-NEXT: # xmm8 = mem[3,1,2,3]2464; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm14 = xmm15[3,1,2,3]2465; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm9 = xmm8[2,0,2,3,4,5,6,7]2466; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm15 = xmm14[2,0,2,3,4,5,6,7]2467; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1]2468; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm02469; AVX2-FP-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm92470; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm9[0,1,2,3,4,5],ymm0[6,7]2471; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[3,1,2,3]2472; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload2473; AVX2-FP-NEXT: # xmm9 = mem[3,1,2,3]2474; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm15 = xmm13[0,1,2,0,4,5,6,7]2475; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm11 = xmm9[0,1,2,0,4,5,6,7]2476; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm15[0],xmm11[1],xmm15[1]2477; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm15 = xmm12[3,1,2,3]2478; AVX2-FP-NEXT: vpshufd $231, (%rsp), %xmm12 # 16-byte Folded Reload2479; AVX2-FP-NEXT: # xmm12 = mem[3,1,2,3]2480; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm15[2,0,2,3,4,5,6,7]2481; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm10 = xmm12[2,0,2,3,4,5,6,7]2482; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm10[0],xmm0[0],xmm10[1],xmm0[1]2483; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm11[2,3]2484; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]2485; AVX2-FP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload2486; AVX2-FP-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]2487; AVX2-FP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload2488; AVX2-FP-NEXT: # xmm10 = mem[0,1,3,1,4,5,6,7]2489; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm10[0],xmm1[0],xmm10[1],xmm1[1]2490; AVX2-FP-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload2491; AVX2-FP-NEXT: # xmm10 = mem[3,1,2,3,4,5,6,7]2492; AVX2-FP-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload2493; AVX2-FP-NEXT: # xmm11 = mem[3,1,2,3,4,5,6,7]2494; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]2495; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm12496; AVX2-FP-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm102497; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm10[0,1,2,3,4,5],ymm1[6,7]2498; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,3,1,4,5,6,7]2499; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,3,1,4,5,6,7]2500; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]2501; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[3,1,2,3,4,5,6,7]2502; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7]2503; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]2504; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0,1],xmm6[2,3]2505; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]2506; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7]2507; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]2508; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]2509; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm8[3,1,2,3,4,5,6,7]2510; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm4 = xmm14[3,1,2,3,4,5,6,7]2511; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]2512; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm22513; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm32514; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]2515; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm13[0,1,3,1,4,5,6,7]2516; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm4 = xmm9[0,1,3,1,4,5,6,7]2517; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]2518; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm4 = xmm15[3,1,2,3,4,5,6,7]2519; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm5 = xmm12[3,1,2,3,4,5,6,7]2520; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]2521; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]2522; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]2523; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload2524; AVX2-FP-NEXT: vmovaps %ymm3, 32(%rsi)2525; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload2526; AVX2-FP-NEXT: vmovaps %ymm3, (%rsi)2527; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload2528; AVX2-FP-NEXT: vmovaps %ymm3, 32(%rdx)2529; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload2530; AVX2-FP-NEXT: vmovaps %ymm3, (%rdx)2531; AVX2-FP-NEXT: vmovdqa %ymm0, 32(%rcx)2532; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2533; AVX2-FP-NEXT: vmovaps %ymm0, (%rcx)2534; AVX2-FP-NEXT: vmovdqa %ymm2, 32(%r8)2535; AVX2-FP-NEXT: vmovdqa %ymm1, (%r8)2536; AVX2-FP-NEXT: addq $184, %rsp2537; AVX2-FP-NEXT: vzeroupper2538; AVX2-FP-NEXT: retq2539;2540; AVX2-FCP-LABEL: load_i16_stride4_vf32:2541; AVX2-FCP: # %bb.0:2542; AVX2-FCP-NEXT: subq $104, %rsp2543; AVX2-FCP-NEXT: vmovdqa 192(%rdi), %ymm32544; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2545; AVX2-FCP-NEXT: vmovdqa 224(%rdi), %ymm62546; AVX2-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2547; AVX2-FCP-NEXT: vmovdqa 64(%rdi), %ymm92548; AVX2-FCP-NEXT: vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2549; AVX2-FCP-NEXT: vmovdqa 96(%rdi), %ymm72550; AVX2-FCP-NEXT: vpxor %xmm0, %xmm0, %xmm02551; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]2552; AVX2-FCP-NEXT: vextracti128 $1, %ymm1, %xmm22553; AVX2-FCP-NEXT: vpackusdw %xmm2, %xmm1, %xmm12554; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]2555; AVX2-FCP-NEXT: vextracti128 $1, %ymm2, %xmm42556; AVX2-FCP-NEXT: vpackusdw %xmm4, %xmm2, %xmm22557; AVX2-FCP-NEXT: vpackusdw %xmm1, %xmm2, %xmm12558; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,2,2,3,0,2,4,6]2559; AVX2-FCP-NEXT: vpermd %ymm7, %ymm2, %ymm52560; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,16,17,20,21,24,25,28,29]2561; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm5, %ymm82562; AVX2-FCP-NEXT: vpermd %ymm9, %ymm2, %ymm102563; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm10, %ymm92564; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4,5],ymm8[6,7]2565; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm8[4,5,6,7]2566; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2567; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]2568; AVX2-FCP-NEXT: vextracti128 $1, %ymm1, %xmm82569; AVX2-FCP-NEXT: vpackusdw %xmm8, %xmm1, %xmm12570; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]2571; AVX2-FCP-NEXT: vextracti128 $1, %ymm0, %xmm82572; AVX2-FCP-NEXT: vpackusdw %xmm8, %xmm0, %xmm02573; AVX2-FCP-NEXT: vpackusdw %xmm1, %xmm0, %xmm02574; AVX2-FCP-NEXT: vpermd %ymm6, %ymm2, %ymm92575; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm9, %ymm12576; AVX2-FCP-NEXT: vpermd %ymm3, %ymm2, %ymm62577; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm6, %ymm82578; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm8[0,1,2,3,4,5],ymm1[6,7]2579; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]2580; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2581; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %xmm142582; AVX2-FCP-NEXT: vmovdqa 48(%rdi), %xmm32583; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm12 = [0,1,2,3,2,3,10,11,8,9,10,11,12,13,14,15]2584; AVX2-FCP-NEXT: vpshufb %xmm12, %xmm3, %xmm02585; AVX2-FCP-NEXT: vpshufb %xmm12, %xmm14, %xmm82586; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1]2587; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm132588; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm22589; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm1 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]2590; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm2, %xmm42591; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm13, %xmm112592; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm11[0],xmm4[0],xmm11[1],xmm4[1]2593; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0,1],xmm8[2,3]2594; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm8 = [2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]2595; AVX2-FCP-NEXT: vpshufb %ymm8, %ymm5, %ymm52596; AVX2-FCP-NEXT: vpshufb %ymm8, %ymm10, %ymm102597; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5],ymm5[6,7]2598; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm5[4,5,6,7]2599; AVX2-FCP-NEXT: vmovdqu %ymm0, (%rsp) # 32-byte Spill2600; AVX2-FCP-NEXT: vmovdqa 160(%rdi), %xmm112601; AVX2-FCP-NEXT: vmovdqa 176(%rdi), %xmm152602; AVX2-FCP-NEXT: vpshufb %xmm12, %xmm15, %xmm42603; AVX2-FCP-NEXT: vpshufb %xmm12, %xmm11, %xmm102604; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1]2605; AVX2-FCP-NEXT: vmovdqa 128(%rdi), %xmm102606; AVX2-FCP-NEXT: vmovdqa 144(%rdi), %xmm122607; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm12, %xmm02608; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm10, %xmm12609; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]2610; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm4[2,3]2611; AVX2-FCP-NEXT: vpshufb %ymm8, %ymm9, %ymm12612; AVX2-FCP-NEXT: vpshufb %ymm8, %ymm6, %ymm42613; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]2614; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]2615; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2616; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm6 = [1,3,2,3,1,3,5,7]2617; AVX2-FCP-NEXT: vpermd %ymm7, %ymm6, %ymm42618; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm7 # 32-byte Folded Reload2619; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm8 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,16,17,20,21,24,25,28,29]2620; AVX2-FCP-NEXT: vpshufb %ymm8, %ymm4, %ymm02621; AVX2-FCP-NEXT: vpshufb %ymm8, %ymm7, %ymm12622; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]2623; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[3,1,2,3]2624; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm14 = xmm14[3,1,2,3]2625; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm1[0,1,2,0,4,5,6,7]2626; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm5 = xmm14[0,1,2,0,4,5,6,7]2627; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]2628; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]2629; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[3,1,2,3]2630; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm5 = xmm2[2,0,2,3,4,5,6,7]2631; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm9 = xmm13[2,0,2,3,4,5,6,7]2632; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1]2633; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm3 = xmm5[0,1],xmm3[2,3]2634; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]2635; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2636; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm3 # 32-byte Folded Reload2637; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm5 # 32-byte Folded Reload2638; AVX2-FCP-NEXT: vpshufb %ymm8, %ymm3, %ymm62639; AVX2-FCP-NEXT: vpshufb %ymm8, %ymm5, %ymm92640; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm9[0,1,2,3,4,5],ymm6[6,7]2641; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm9 = xmm15[3,1,2,3]2642; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[3,1,2,3]2643; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm15 = xmm9[0,1,2,0,4,5,6,7]2644; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm11[0,1,2,0,4,5,6,7]2645; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm15[0],xmm0[1],xmm15[1]2646; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[3,1,2,3]2647; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[3,1,2,3]2648; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm15 = xmm12[2,0,2,3,4,5,6,7]2649; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm8 = xmm10[2,0,2,3,4,5,6,7]2650; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm15[0],xmm8[1],xmm15[1]2651; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm8[0,1],xmm0[2,3]2652; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm6[4,5,6,7]2653; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm8 = [2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]2654; AVX2-FCP-NEXT: vpshufb %ymm8, %ymm4, %ymm42655; AVX2-FCP-NEXT: vpshufb %ymm8, %ymm7, %ymm62656; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm6[0,1,2,3,4,5],ymm4[6,7]2657; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7]2658; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm6 = xmm14[0,1,3,1,4,5,6,7]2659; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]2660; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]2661; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm6 = xmm13[3,1,2,3,4,5,6,7]2662; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1]2663; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]2664; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm4[4,5,6,7]2665; AVX2-FCP-NEXT: vpshufb %ymm8, %ymm3, %ymm22666; AVX2-FCP-NEXT: vpshufb %ymm8, %ymm5, %ymm32667; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]2668; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm9[0,1,3,1,4,5,6,7]2669; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm4 = xmm11[0,1,3,1,4,5,6,7]2670; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]2671; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm4 = xmm12[3,1,2,3,4,5,6,7]2672; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm5 = xmm10[3,1,2,3,4,5,6,7]2673; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]2674; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]2675; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]2676; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload2677; AVX2-FCP-NEXT: vmovaps %ymm3, 32(%rsi)2678; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload2679; AVX2-FCP-NEXT: vmovaps %ymm3, (%rsi)2680; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload2681; AVX2-FCP-NEXT: vmovaps %ymm3, 32(%rdx)2682; AVX2-FCP-NEXT: vmovups (%rsp), %ymm3 # 32-byte Reload2683; AVX2-FCP-NEXT: vmovaps %ymm3, (%rdx)2684; AVX2-FCP-NEXT: vmovdqa %ymm0, 32(%rcx)2685; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2686; AVX2-FCP-NEXT: vmovaps %ymm0, (%rcx)2687; AVX2-FCP-NEXT: vmovdqa %ymm2, 32(%r8)2688; AVX2-FCP-NEXT: vmovdqa %ymm1, (%r8)2689; AVX2-FCP-NEXT: addq $104, %rsp2690; AVX2-FCP-NEXT: vzeroupper2691; AVX2-FCP-NEXT: retq2692;2693; AVX512-LABEL: load_i16_stride4_vf32:2694; AVX512: # %bb.0:2695; AVX512-NEXT: vmovdqa64 (%rdi), %zmm02696; AVX512-NEXT: vmovdqa64 128(%rdi), %zmm12697; AVX512-NEXT: vmovdqa 192(%rdi), %ymm22698; AVX512-NEXT: vpmovqw %ymm2, %xmm32699; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm32700; AVX512-NEXT: vmovdqa 240(%rdi), %xmm52701; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[0,2,2,3]2702; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm6[0,1,0,2,4,5,6,7]2703; AVX512-NEXT: vmovdqa 224(%rdi), %xmm72704; AVX512-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[0,2,2,3]2705; AVX512-NEXT: vpshuflw {{.*#+}} xmm9 = xmm8[0,1,0,2,4,5,6,7]2706; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1]2707; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm42708; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm4[6,7]2709; AVX512-NEXT: vpmovqw %zmm1, %xmm42710; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm3[4,5,6,7]2711; AVX512-NEXT: vmovdqa 64(%rdi), %ymm32712; AVX512-NEXT: vpmovqw %ymm3, %xmm92713; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm92714; AVX512-NEXT: vmovdqa 112(%rdi), %xmm102715; AVX512-NEXT: vpshufd {{.*#+}} xmm11 = xmm10[0,2,2,3]2716; AVX512-NEXT: vpshuflw {{.*#+}} xmm12 = xmm11[0,1,0,2,4,5,6,7]2717; AVX512-NEXT: vmovdqa 96(%rdi), %xmm132718; AVX512-NEXT: vpshufd {{.*#+}} xmm14 = xmm13[0,2,2,3]2719; AVX512-NEXT: vpshuflw {{.*#+}} xmm15 = xmm14[0,1,0,2,4,5,6,7]2720; AVX512-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm15[0],xmm12[0],xmm15[1],xmm12[1]2721; AVX512-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm122722; AVX512-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm12[6,7]2723; AVX512-NEXT: vpmovqw %zmm0, %xmm122724; AVX512-NEXT: vpblendd {{.*#+}} ymm9 = ymm12[0,1,2,3],ymm9[4,5,6,7]2725; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm4 = zmm9[0,1,2,3],zmm4[0,1,2,3]2726; AVX512-NEXT: vpsrlq $16, %ymm2, %ymm92727; AVX512-NEXT: vpmovqw %ymm9, %xmm92728; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm92729; AVX512-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]2730; AVX512-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,1,1,3,4,5,6,7]2731; AVX512-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm8[0],xmm6[0],xmm8[1],xmm6[1]2732; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm62733; AVX512-NEXT: vpblendd {{.*#+}} ymm6 = ymm9[0,1,2,3,4,5],ymm6[6,7]2734; AVX512-NEXT: vpsrlq $16, %zmm1, %zmm82735; AVX512-NEXT: vpmovqw %zmm8, %xmm82736; AVX512-NEXT: vpblendd {{.*#+}} ymm6 = ymm8[0,1,2,3],ymm6[4,5,6,7]2737; AVX512-NEXT: vpsrlq $16, %ymm3, %ymm82738; AVX512-NEXT: vpmovqw %ymm8, %xmm82739; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm82740; AVX512-NEXT: vpshuflw {{.*#+}} xmm9 = xmm11[0,1,1,3,4,5,6,7]2741; AVX512-NEXT: vpshuflw {{.*#+}} xmm11 = xmm14[0,1,1,3,4,5,6,7]2742; AVX512-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]2743; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm92744; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm9[6,7]2745; AVX512-NEXT: vpsrlq $16, %zmm0, %zmm92746; AVX512-NEXT: vpmovqw %zmm9, %xmm92747; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]2748; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm6 = zmm8[0,1,2,3],zmm6[0,1,2,3]2749; AVX512-NEXT: vpsrlq $32, %ymm2, %ymm82750; AVX512-NEXT: vpmovqw %ymm8, %xmm82751; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm82752; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]2753; AVX512-NEXT: vpshuflw {{.*#+}} xmm9 = xmm5[0,1,2,0,4,5,6,7]2754; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,1,2,3]2755; AVX512-NEXT: vpshuflw {{.*#+}} xmm11 = xmm7[0,1,2,0,4,5,6,7]2756; AVX512-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]2757; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm92758; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm9[6,7]2759; AVX512-NEXT: vpsrlq $32, %zmm1, %zmm92760; AVX512-NEXT: vpmovqw %zmm9, %xmm92761; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]2762; AVX512-NEXT: vpsrlq $32, %ymm3, %ymm92763; AVX512-NEXT: vpmovqw %ymm9, %xmm92764; AVX512-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm92765; AVX512-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[3,1,2,3]2766; AVX512-NEXT: vpshuflw {{.*#+}} xmm11 = xmm10[0,1,2,0,4,5,6,7]2767; AVX512-NEXT: vpshufd {{.*#+}} xmm12 = xmm13[3,1,2,3]2768; AVX512-NEXT: vpshuflw {{.*#+}} xmm13 = xmm12[0,1,2,0,4,5,6,7]2769; AVX512-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1]2770; AVX512-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm112771; AVX512-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm11[6,7]2772; AVX512-NEXT: vpsrlq $32, %zmm0, %zmm112773; AVX512-NEXT: vpmovqw %zmm11, %xmm112774; AVX512-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3],ymm9[4,5,6,7]2775; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm8 = zmm9[0,1,2,3],zmm8[0,1,2,3]2776; AVX512-NEXT: vpsrlq $48, %ymm2, %ymm22777; AVX512-NEXT: vpmovqw %ymm2, %xmm22778; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm22779; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]2780; AVX512-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,3,1,4,5,6,7]2781; AVX512-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]2782; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm52783; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]2784; AVX512-NEXT: vpsrlq $48, %zmm1, %zmm12785; AVX512-NEXT: vpmovqw %zmm1, %xmm12786; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]2787; AVX512-NEXT: vpsrlq $48, %ymm3, %ymm22788; AVX512-NEXT: vpmovqw %ymm2, %xmm22789; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm22790; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm10[0,1,3,1,4,5,6,7]2791; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm12[0,1,3,1,4,5,6,7]2792; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]2793; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm32794; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]2795; AVX512-NEXT: vpsrlq $48, %zmm0, %zmm02796; AVX512-NEXT: vpmovqw %zmm0, %xmm02797; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]2798; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[0,1,2,3]2799; AVX512-NEXT: vmovdqa64 %zmm4, (%rsi)2800; AVX512-NEXT: vmovdqa64 %zmm6, (%rdx)2801; AVX512-NEXT: vmovdqa64 %zmm8, (%rcx)2802; AVX512-NEXT: vmovdqa64 %zmm0, (%r8)2803; AVX512-NEXT: vzeroupper2804; AVX512-NEXT: retq2805;2806; AVX512-FCP-LABEL: load_i16_stride4_vf32:2807; AVX512-FCP: # %bb.0:2808; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm02809; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %zmm12810; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [16,17,20,21,0,0,0,0,24,25,28,29,0,0,0,0,16,17,20,21,0,0,0,0,24,25,28,29,0,0,0,0]2811; AVX512-FCP-NEXT: # ymm2 = mem[0,1,0,1]2812; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [0,2,2,3,0,2,4,6]2813; AVX512-FCP-NEXT: vmovdqa 224(%rdi), %ymm52814; AVX512-FCP-NEXT: vpermd %ymm5, %ymm4, %ymm62815; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm6, %ymm72816; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,4,5,u,u,u,u,8,9,12,13,12,13,14,15,16,17,20,21,u,u,u,u,24,25,28,29,u,u,u,u]2817; AVX512-FCP-NEXT: vmovdqa 192(%rdi), %ymm82818; AVX512-FCP-NEXT: vpermd %ymm8, %ymm4, %ymm92819; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm9, %ymm102820; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm11 = [0,2,2,3,4,6,12,14]2821; AVX512-FCP-NEXT: vpermt2d %ymm7, %ymm11, %ymm102822; AVX512-FCP-NEXT: vpmovqw %zmm1, %xmm72823; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm10[4,5,6,7]2824; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %ymm102825; AVX512-FCP-NEXT: vpermd %ymm10, %ymm4, %ymm122826; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm12, %ymm132827; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm142828; AVX512-FCP-NEXT: vpermd %ymm14, %ymm4, %ymm152829; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm15, %ymm42830; AVX512-FCP-NEXT: vpermt2d %ymm13, %ymm11, %ymm42831; AVX512-FCP-NEXT: vpmovqw %zmm0, %xmm132832; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm13[0,1,2,3],ymm4[4,5,6,7]2833; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm7 = zmm4[0,1,2,3],zmm7[0,1,2,3]2834; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]2835; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm6, %ymm62836; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm9, %ymm92837; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm9[0,1,2,3,4,5],ymm6[6,7]2838; AVX512-FCP-NEXT: vpsrlq $16, %zmm1, %zmm92839; AVX512-FCP-NEXT: vpmovqw %zmm9, %xmm92840; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm9[0,1,2,3],ymm6[4,5,6,7]2841; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm12, %ymm92842; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm15, %ymm122843; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm12[0,1,2,3,4,5],ymm9[6,7]2844; AVX512-FCP-NEXT: vpsrlq $16, %zmm0, %zmm122845; AVX512-FCP-NEXT: vpmovqw %zmm12, %xmm122846; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm12[0,1,2,3],ymm9[4,5,6,7]2847; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm6 = zmm9[0,1,2,3],zmm6[0,1,2,3]2848; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm9 = [1,3,2,3,1,3,5,7]2849; AVX512-FCP-NEXT: vpermd %ymm5, %ymm9, %ymm52850; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm5, %ymm122851; AVX512-FCP-NEXT: vpermd %ymm8, %ymm9, %ymm82852; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm8, %ymm132853; AVX512-FCP-NEXT: vpermt2d %ymm12, %ymm11, %ymm132854; AVX512-FCP-NEXT: vpsrlq $32, %zmm1, %zmm122855; AVX512-FCP-NEXT: vpmovqw %zmm12, %xmm122856; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3],ymm13[4,5,6,7]2857; AVX512-FCP-NEXT: vpermd %ymm10, %ymm9, %ymm102858; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm10, %ymm22859; AVX512-FCP-NEXT: vpermd %ymm14, %ymm9, %ymm92860; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm9, %ymm32861; AVX512-FCP-NEXT: vpermt2d %ymm2, %ymm11, %ymm32862; AVX512-FCP-NEXT: vpsrlq $32, %zmm0, %zmm22863; AVX512-FCP-NEXT: vpmovqw %zmm2, %xmm22864; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]2865; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,2,3],zmm12[0,1,2,3]2866; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm5, %ymm32867; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm8, %ymm52868; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3,4,5],ymm3[6,7]2869; AVX512-FCP-NEXT: vpsrlq $48, %zmm1, %zmm12870; AVX512-FCP-NEXT: vpmovqw %zmm1, %xmm12871; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]2872; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm10, %ymm32873; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm9, %ymm42874; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]2875; AVX512-FCP-NEXT: vpsrlq $48, %zmm0, %zmm02876; AVX512-FCP-NEXT: vpmovqw %zmm0, %xmm02877; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]2878; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[0,1,2,3]2879; AVX512-FCP-NEXT: vmovdqa64 %zmm7, (%rsi)2880; AVX512-FCP-NEXT: vmovdqa64 %zmm6, (%rdx)2881; AVX512-FCP-NEXT: vmovdqa64 %zmm2, (%rcx)2882; AVX512-FCP-NEXT: vmovdqa64 %zmm0, (%r8)2883; AVX512-FCP-NEXT: vzeroupper2884; AVX512-FCP-NEXT: retq2885;2886; AVX512DQ-LABEL: load_i16_stride4_vf32:2887; AVX512DQ: # %bb.0:2888; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm02889; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %zmm12890; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm22891; AVX512DQ-NEXT: vpmovqw %ymm2, %xmm32892; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm32893; AVX512DQ-NEXT: vmovdqa 240(%rdi), %xmm52894; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[0,2,2,3]2895; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm6[0,1,0,2,4,5,6,7]2896; AVX512DQ-NEXT: vmovdqa 224(%rdi), %xmm72897; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[0,2,2,3]2898; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm8[0,1,0,2,4,5,6,7]2899; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1]2900; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm42901; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm4[6,7]2902; AVX512DQ-NEXT: vpmovqw %zmm1, %xmm42903; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm3[4,5,6,7]2904; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm32905; AVX512DQ-NEXT: vpmovqw %ymm3, %xmm92906; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm92907; AVX512DQ-NEXT: vmovdqa 112(%rdi), %xmm102908; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm11 = xmm10[0,2,2,3]2909; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm12 = xmm11[0,1,0,2,4,5,6,7]2910; AVX512DQ-NEXT: vmovdqa 96(%rdi), %xmm132911; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm14 = xmm13[0,2,2,3]2912; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm15 = xmm14[0,1,0,2,4,5,6,7]2913; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm15[0],xmm12[0],xmm15[1],xmm12[1]2914; AVX512DQ-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm122915; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm12[6,7]2916; AVX512DQ-NEXT: vpmovqw %zmm0, %xmm122917; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm9 = ymm12[0,1,2,3],ymm9[4,5,6,7]2918; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm4 = zmm9[0,1,2,3],zmm4[0,1,2,3]2919; AVX512DQ-NEXT: vpsrlq $16, %ymm2, %ymm92920; AVX512DQ-NEXT: vpmovqw %ymm9, %xmm92921; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm92922; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]2923; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,1,1,3,4,5,6,7]2924; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm8[0],xmm6[0],xmm8[1],xmm6[1]2925; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm62926; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm6 = ymm9[0,1,2,3,4,5],ymm6[6,7]2927; AVX512DQ-NEXT: vpsrlq $16, %zmm1, %zmm82928; AVX512DQ-NEXT: vpmovqw %zmm8, %xmm82929; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm6 = ymm8[0,1,2,3],ymm6[4,5,6,7]2930; AVX512DQ-NEXT: vpsrlq $16, %ymm3, %ymm82931; AVX512DQ-NEXT: vpmovqw %ymm8, %xmm82932; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm82933; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm11[0,1,1,3,4,5,6,7]2934; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm11 = xmm14[0,1,1,3,4,5,6,7]2935; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]2936; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm92937; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm9[6,7]2938; AVX512DQ-NEXT: vpsrlq $16, %zmm0, %zmm92939; AVX512DQ-NEXT: vpmovqw %zmm9, %xmm92940; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]2941; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm6 = zmm8[0,1,2,3],zmm6[0,1,2,3]2942; AVX512DQ-NEXT: vpsrlq $32, %ymm2, %ymm82943; AVX512DQ-NEXT: vpmovqw %ymm8, %xmm82944; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm82945; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]2946; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm5[0,1,2,0,4,5,6,7]2947; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,1,2,3]2948; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm11 = xmm7[0,1,2,0,4,5,6,7]2949; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]2950; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm92951; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm9[6,7]2952; AVX512DQ-NEXT: vpsrlq $32, %zmm1, %zmm92953; AVX512DQ-NEXT: vpmovqw %zmm9, %xmm92954; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]2955; AVX512DQ-NEXT: vpsrlq $32, %ymm3, %ymm92956; AVX512DQ-NEXT: vpmovqw %ymm9, %xmm92957; AVX512DQ-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm92958; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[3,1,2,3]2959; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm11 = xmm10[0,1,2,0,4,5,6,7]2960; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm12 = xmm13[3,1,2,3]2961; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm13 = xmm12[0,1,2,0,4,5,6,7]2962; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1]2963; AVX512DQ-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm112964; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm11[6,7]2965; AVX512DQ-NEXT: vpsrlq $32, %zmm0, %zmm112966; AVX512DQ-NEXT: vpmovqw %zmm11, %xmm112967; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3],ymm9[4,5,6,7]2968; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm8 = zmm9[0,1,2,3],zmm8[0,1,2,3]2969; AVX512DQ-NEXT: vpsrlq $48, %ymm2, %ymm22970; AVX512DQ-NEXT: vpmovqw %ymm2, %xmm22971; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm22972; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]2973; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,3,1,4,5,6,7]2974; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]2975; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm52976; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]2977; AVX512DQ-NEXT: vpsrlq $48, %zmm1, %zmm12978; AVX512DQ-NEXT: vpmovqw %zmm1, %xmm12979; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]2980; AVX512DQ-NEXT: vpsrlq $48, %ymm3, %ymm22981; AVX512DQ-NEXT: vpmovqw %ymm2, %xmm22982; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm22983; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm10[0,1,3,1,4,5,6,7]2984; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm12[0,1,3,1,4,5,6,7]2985; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]2986; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm32987; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]2988; AVX512DQ-NEXT: vpsrlq $48, %zmm0, %zmm02989; AVX512DQ-NEXT: vpmovqw %zmm0, %xmm02990; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]2991; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[0,1,2,3]2992; AVX512DQ-NEXT: vmovdqa64 %zmm4, (%rsi)2993; AVX512DQ-NEXT: vmovdqa64 %zmm6, (%rdx)2994; AVX512DQ-NEXT: vmovdqa64 %zmm8, (%rcx)2995; AVX512DQ-NEXT: vmovdqa64 %zmm0, (%r8)2996; AVX512DQ-NEXT: vzeroupper2997; AVX512DQ-NEXT: retq2998;2999; AVX512DQ-FCP-LABEL: load_i16_stride4_vf32:3000; AVX512DQ-FCP: # %bb.0:3001; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm03002; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %zmm13003; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [16,17,20,21,0,0,0,0,24,25,28,29,0,0,0,0,16,17,20,21,0,0,0,0,24,25,28,29,0,0,0,0]3004; AVX512DQ-FCP-NEXT: # ymm2 = mem[0,1,0,1]3005; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [0,2,2,3,0,2,4,6]3006; AVX512DQ-FCP-NEXT: vmovdqa 224(%rdi), %ymm53007; AVX512DQ-FCP-NEXT: vpermd %ymm5, %ymm4, %ymm63008; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm6, %ymm73009; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,4,5,u,u,u,u,8,9,12,13,12,13,14,15,16,17,20,21,u,u,u,u,24,25,28,29,u,u,u,u]3010; AVX512DQ-FCP-NEXT: vmovdqa 192(%rdi), %ymm83011; AVX512DQ-FCP-NEXT: vpermd %ymm8, %ymm4, %ymm93012; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm9, %ymm103013; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm11 = [0,2,2,3,4,6,12,14]3014; AVX512DQ-FCP-NEXT: vpermt2d %ymm7, %ymm11, %ymm103015; AVX512DQ-FCP-NEXT: vpmovqw %zmm1, %xmm73016; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm10[4,5,6,7]3017; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %ymm103018; AVX512DQ-FCP-NEXT: vpermd %ymm10, %ymm4, %ymm123019; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm12, %ymm133020; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm143021; AVX512DQ-FCP-NEXT: vpermd %ymm14, %ymm4, %ymm153022; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm15, %ymm43023; AVX512DQ-FCP-NEXT: vpermt2d %ymm13, %ymm11, %ymm43024; AVX512DQ-FCP-NEXT: vpmovqw %zmm0, %xmm133025; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm13[0,1,2,3],ymm4[4,5,6,7]3026; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm7 = zmm4[0,1,2,3],zmm7[0,1,2,3]3027; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]3028; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm6, %ymm63029; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm9, %ymm93030; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm9[0,1,2,3,4,5],ymm6[6,7]3031; AVX512DQ-FCP-NEXT: vpsrlq $16, %zmm1, %zmm93032; AVX512DQ-FCP-NEXT: vpmovqw %zmm9, %xmm93033; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm9[0,1,2,3],ymm6[4,5,6,7]3034; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm12, %ymm93035; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm15, %ymm123036; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm12[0,1,2,3,4,5],ymm9[6,7]3037; AVX512DQ-FCP-NEXT: vpsrlq $16, %zmm0, %zmm123038; AVX512DQ-FCP-NEXT: vpmovqw %zmm12, %xmm123039; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm12[0,1,2,3],ymm9[4,5,6,7]3040; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm6 = zmm9[0,1,2,3],zmm6[0,1,2,3]3041; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm9 = [1,3,2,3,1,3,5,7]3042; AVX512DQ-FCP-NEXT: vpermd %ymm5, %ymm9, %ymm53043; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm5, %ymm123044; AVX512DQ-FCP-NEXT: vpermd %ymm8, %ymm9, %ymm83045; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm8, %ymm133046; AVX512DQ-FCP-NEXT: vpermt2d %ymm12, %ymm11, %ymm133047; AVX512DQ-FCP-NEXT: vpsrlq $32, %zmm1, %zmm123048; AVX512DQ-FCP-NEXT: vpmovqw %zmm12, %xmm123049; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3],ymm13[4,5,6,7]3050; AVX512DQ-FCP-NEXT: vpermd %ymm10, %ymm9, %ymm103051; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm10, %ymm23052; AVX512DQ-FCP-NEXT: vpermd %ymm14, %ymm9, %ymm93053; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm9, %ymm33054; AVX512DQ-FCP-NEXT: vpermt2d %ymm2, %ymm11, %ymm33055; AVX512DQ-FCP-NEXT: vpsrlq $32, %zmm0, %zmm23056; AVX512DQ-FCP-NEXT: vpmovqw %zmm2, %xmm23057; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]3058; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,2,3],zmm12[0,1,2,3]3059; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm5, %ymm33060; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm8, %ymm53061; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3,4,5],ymm3[6,7]3062; AVX512DQ-FCP-NEXT: vpsrlq $48, %zmm1, %zmm13063; AVX512DQ-FCP-NEXT: vpmovqw %zmm1, %xmm13064; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]3065; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm10, %ymm33066; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm9, %ymm43067; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]3068; AVX512DQ-FCP-NEXT: vpsrlq $48, %zmm0, %zmm03069; AVX512DQ-FCP-NEXT: vpmovqw %zmm0, %xmm03070; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]3071; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[0,1,2,3]3072; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm7, (%rsi)3073; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm6, (%rdx)3074; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, (%rcx)3075; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, (%r8)3076; AVX512DQ-FCP-NEXT: vzeroupper3077; AVX512DQ-FCP-NEXT: retq3078;3079; AVX512BW-LABEL: load_i16_stride4_vf32:3080; AVX512BW: # %bb.0:3081; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm03082; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm13083; AVX512BW-NEXT: vmovdqa64 128(%rdi), %zmm23084; AVX512BW-NEXT: vmovdqa64 192(%rdi), %zmm33085; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm4 = [0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60]3086; AVX512BW-NEXT: # zmm4 = mem[0,1,2,3,0,1,2,3]3087; AVX512BW-NEXT: vmovdqa64 %zmm2, %zmm53088; AVX512BW-NEXT: vpermt2w %zmm3, %zmm4, %zmm53089; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm43090; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm4 = zmm4[0,1,2,3],zmm5[4,5,6,7]3091; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm5 = [1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61,1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61]3092; AVX512BW-NEXT: # zmm5 = mem[0,1,2,3,0,1,2,3]3093; AVX512BW-NEXT: vmovdqa64 %zmm2, %zmm63094; AVX512BW-NEXT: vpermt2w %zmm3, %zmm5, %zmm63095; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm53096; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm5 = zmm5[0,1,2,3],zmm6[4,5,6,7]3097; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm6 = [2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62,2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62]3098; AVX512BW-NEXT: # zmm6 = mem[0,1,2,3,0,1,2,3]3099; AVX512BW-NEXT: vmovdqa64 %zmm2, %zmm73100; AVX512BW-NEXT: vpermt2w %zmm3, %zmm6, %zmm73101; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm63102; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm6 = zmm6[0,1,2,3],zmm7[4,5,6,7]3103; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63,3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63]3104; AVX512BW-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]3105; AVX512BW-NEXT: vpermt2w %zmm3, %zmm7, %zmm23106; AVX512BW-NEXT: vpermt2w %zmm1, %zmm7, %zmm03107; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm2[4,5,6,7]3108; AVX512BW-NEXT: vmovdqa64 %zmm4, (%rsi)3109; AVX512BW-NEXT: vmovdqa64 %zmm5, (%rdx)3110; AVX512BW-NEXT: vmovdqa64 %zmm6, (%rcx)3111; AVX512BW-NEXT: vmovdqa64 %zmm0, (%r8)3112; AVX512BW-NEXT: vzeroupper3113; AVX512BW-NEXT: retq3114;3115; AVX512BW-FCP-LABEL: load_i16_stride4_vf32:3116; AVX512BW-FCP: # %bb.0:3117; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm03118; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm13119; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %zmm23120; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %zmm33121; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm4 = [0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60]3122; AVX512BW-FCP-NEXT: # zmm4 = mem[0,1,2,3,0,1,2,3]3123; AVX512BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm53124; AVX512BW-FCP-NEXT: vpermt2w %zmm3, %zmm4, %zmm53125; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm43126; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm4 = zmm4[0,1,2,3],zmm5[4,5,6,7]3127; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm5 = [1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61,1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61]3128; AVX512BW-FCP-NEXT: # zmm5 = mem[0,1,2,3,0,1,2,3]3129; AVX512BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm63130; AVX512BW-FCP-NEXT: vpermt2w %zmm3, %zmm5, %zmm63131; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm53132; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm5 = zmm5[0,1,2,3],zmm6[4,5,6,7]3133; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm6 = [2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62,2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62]3134; AVX512BW-FCP-NEXT: # zmm6 = mem[0,1,2,3,0,1,2,3]3135; AVX512BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm73136; AVX512BW-FCP-NEXT: vpermt2w %zmm3, %zmm6, %zmm73137; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm63138; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm6 = zmm6[0,1,2,3],zmm7[4,5,6,7]3139; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63,3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63]3140; AVX512BW-FCP-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]3141; AVX512BW-FCP-NEXT: vpermt2w %zmm3, %zmm7, %zmm23142; AVX512BW-FCP-NEXT: vpermt2w %zmm1, %zmm7, %zmm03143; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm2[4,5,6,7]3144; AVX512BW-FCP-NEXT: vmovdqa64 %zmm4, (%rsi)3145; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, (%rdx)3146; AVX512BW-FCP-NEXT: vmovdqa64 %zmm6, (%rcx)3147; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, (%r8)3148; AVX512BW-FCP-NEXT: vzeroupper3149; AVX512BW-FCP-NEXT: retq3150;3151; AVX512DQ-BW-LABEL: load_i16_stride4_vf32:3152; AVX512DQ-BW: # %bb.0:3153; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm03154; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %zmm13155; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %zmm23156; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %zmm33157; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm4 = [0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60]3158; AVX512DQ-BW-NEXT: # zmm4 = mem[0,1,2,3,0,1,2,3]3159; AVX512DQ-BW-NEXT: vmovdqa64 %zmm2, %zmm53160; AVX512DQ-BW-NEXT: vpermt2w %zmm3, %zmm4, %zmm53161; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm43162; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm4 = zmm4[0,1,2,3],zmm5[4,5,6,7]3163; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm5 = [1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61,1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61]3164; AVX512DQ-BW-NEXT: # zmm5 = mem[0,1,2,3,0,1,2,3]3165; AVX512DQ-BW-NEXT: vmovdqa64 %zmm2, %zmm63166; AVX512DQ-BW-NEXT: vpermt2w %zmm3, %zmm5, %zmm63167; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm53168; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm5 = zmm5[0,1,2,3],zmm6[4,5,6,7]3169; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm6 = [2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62,2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62]3170; AVX512DQ-BW-NEXT: # zmm6 = mem[0,1,2,3,0,1,2,3]3171; AVX512DQ-BW-NEXT: vmovdqa64 %zmm2, %zmm73172; AVX512DQ-BW-NEXT: vpermt2w %zmm3, %zmm6, %zmm73173; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm63174; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm6 = zmm6[0,1,2,3],zmm7[4,5,6,7]3175; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63,3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63]3176; AVX512DQ-BW-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]3177; AVX512DQ-BW-NEXT: vpermt2w %zmm3, %zmm7, %zmm23178; AVX512DQ-BW-NEXT: vpermt2w %zmm1, %zmm7, %zmm03179; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm2[4,5,6,7]3180; AVX512DQ-BW-NEXT: vmovdqa64 %zmm4, (%rsi)3181; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, (%rdx)3182; AVX512DQ-BW-NEXT: vmovdqa64 %zmm6, (%rcx)3183; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, (%r8)3184; AVX512DQ-BW-NEXT: vzeroupper3185; AVX512DQ-BW-NEXT: retq3186;3187; AVX512DQ-BW-FCP-LABEL: load_i16_stride4_vf32:3188; AVX512DQ-BW-FCP: # %bb.0:3189; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm03190; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm13191; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %zmm23192; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %zmm33193; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm4 = [0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60]3194; AVX512DQ-BW-FCP-NEXT: # zmm4 = mem[0,1,2,3,0,1,2,3]3195; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm53196; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm3, %zmm4, %zmm53197; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm43198; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm4 = zmm4[0,1,2,3],zmm5[4,5,6,7]3199; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm5 = [1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61,1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61]3200; AVX512DQ-BW-FCP-NEXT: # zmm5 = mem[0,1,2,3,0,1,2,3]3201; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm63202; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm3, %zmm5, %zmm63203; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm53204; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm5 = zmm5[0,1,2,3],zmm6[4,5,6,7]3205; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm6 = [2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62,2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62]3206; AVX512DQ-BW-FCP-NEXT: # zmm6 = mem[0,1,2,3,0,1,2,3]3207; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm73208; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm3, %zmm6, %zmm73209; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm63210; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm6 = zmm6[0,1,2,3],zmm7[4,5,6,7]3211; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63,3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63]3212; AVX512DQ-BW-FCP-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3]3213; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm3, %zmm7, %zmm23214; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm1, %zmm7, %zmm03215; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm2[4,5,6,7]3216; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm4, (%rsi)3217; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, (%rdx)3218; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm6, (%rcx)3219; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, (%r8)3220; AVX512DQ-BW-FCP-NEXT: vzeroupper3221; AVX512DQ-BW-FCP-NEXT: retq3222 %wide.vec = load <128 x i16>, ptr %in.vec, align 643223 %strided.vec0 = shufflevector <128 x i16> %wide.vec, <128 x i16> poison, <32 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60, i32 64, i32 68, i32 72, i32 76, i32 80, i32 84, i32 88, i32 92, i32 96, i32 100, i32 104, i32 108, i32 112, i32 116, i32 120, i32 124>3224 %strided.vec1 = shufflevector <128 x i16> %wide.vec, <128 x i16> poison, <32 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61, i32 65, i32 69, i32 73, i32 77, i32 81, i32 85, i32 89, i32 93, i32 97, i32 101, i32 105, i32 109, i32 113, i32 117, i32 121, i32 125>3225 %strided.vec2 = shufflevector <128 x i16> %wide.vec, <128 x i16> poison, <32 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62, i32 66, i32 70, i32 74, i32 78, i32 82, i32 86, i32 90, i32 94, i32 98, i32 102, i32 106, i32 110, i32 114, i32 118, i32 122, i32 126>3226 %strided.vec3 = shufflevector <128 x i16> %wide.vec, <128 x i16> poison, <32 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63, i32 67, i32 71, i32 75, i32 79, i32 83, i32 87, i32 91, i32 95, i32 99, i32 103, i32 107, i32 111, i32 115, i32 119, i32 123, i32 127>3227 store <32 x i16> %strided.vec0, ptr %out.vec0, align 643228 store <32 x i16> %strided.vec1, ptr %out.vec1, align 643229 store <32 x i16> %strided.vec2, ptr %out.vec2, align 643230 store <32 x i16> %strided.vec3, ptr %out.vec3, align 643231 ret void3232}3233 3234define void @load_i16_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3) nounwind {3235; SSE-LABEL: load_i16_stride4_vf64:3236; SSE: # %bb.0:3237; SSE-NEXT: subq $824, %rsp # imm = 0x3383238; SSE-NEXT: movdqa 352(%rdi), %xmm33239; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3240; SSE-NEXT: movdqa 320(%rdi), %xmm43241; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3242; SSE-NEXT: movdqa 336(%rdi), %xmm53243; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3244; SSE-NEXT: movdqa 96(%rdi), %xmm23245; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3246; SSE-NEXT: movdqa 112(%rdi), %xmm63247; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3248; SSE-NEXT: movdqa 64(%rdi), %xmm13249; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3250; SSE-NEXT: movdqa 80(%rdi), %xmm03251; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3252; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]3253; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3254; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]3255; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]3256; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3257; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]3258; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]3259; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,2,2,3]3260; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3261; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]3262; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]3263; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3264; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,2,4,5,6,7]3265; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]3266; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]3267; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3268; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,2,2,3]3269; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3270; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]3271; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm4[0,2,2,3]3272; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3273; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]3274; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]3275; SSE-NEXT: movdqa 368(%rdi), %xmm03276; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3277; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]3278; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3279; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]3280; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]3281; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3282; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,2,4,5,6,7]3283; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]3284; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]3285; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3286; SSE-NEXT: movdqa (%rdi), %xmm13287; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3288; SSE-NEXT: movdqa 16(%rdi), %xmm03289; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3290; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]3291; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3292; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]3293; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]3294; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3295; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]3296; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]3297; SSE-NEXT: movdqa 32(%rdi), %xmm23298; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3299; SSE-NEXT: movdqa 48(%rdi), %xmm03300; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3301; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]3302; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3303; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]3304; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]3305; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3306; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,2,4,5,6,7]3307; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]3308; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]3309; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3310; SSE-NEXT: movdqa 256(%rdi), %xmm13311; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3312; SSE-NEXT: movdqa 272(%rdi), %xmm03313; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3314; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]3315; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3316; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]3317; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]3318; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3319; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]3320; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]3321; SSE-NEXT: movdqa 288(%rdi), %xmm23322; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3323; SSE-NEXT: movdqa 304(%rdi), %xmm03324; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3325; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]3326; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3327; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]3328; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]3329; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3330; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,2,4,5,6,7]3331; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]3332; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]3333; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3334; SSE-NEXT: movdqa 192(%rdi), %xmm13335; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3336; SSE-NEXT: movdqa 208(%rdi), %xmm03337; SSE-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill3338; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]3339; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3340; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]3341; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]3342; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3343; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]3344; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]3345; SSE-NEXT: movdqa 224(%rdi), %xmm23346; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3347; SSE-NEXT: movdqa 240(%rdi), %xmm03348; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3349; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]3350; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3351; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]3352; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm2[0,2,2,3]3353; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm14[0,1,0,2,4,5,6,7]3354; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]3355; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]3356; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3357; SSE-NEXT: movdqa 448(%rdi), %xmm13358; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3359; SSE-NEXT: movdqa 464(%rdi), %xmm03360; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3361; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm0[0,2,2,3]3362; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm13[0,2,2,3,4,5,6,7]3363; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm1[0,2,2,3]3364; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm12[0,2,2,3,4,5,6,7]3365; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]3366; SSE-NEXT: movdqa 480(%rdi), %xmm23367; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3368; SSE-NEXT: movdqa 496(%rdi), %xmm03369; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3370; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm0[0,2,2,3]3371; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm11[0,1,0,2,4,5,6,7]3372; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm2[0,2,2,3]3373; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm10[0,1,0,2,4,5,6,7]3374; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]3375; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]3376; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3377; SSE-NEXT: movdqa 128(%rdi), %xmm13378; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3379; SSE-NEXT: movdqa 144(%rdi), %xmm03380; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3381; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm0[0,2,2,3]3382; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm9[0,2,2,3,4,5,6,7]3383; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm1[0,2,2,3]3384; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm8[0,2,2,3,4,5,6,7]3385; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]3386; SSE-NEXT: movdqa 160(%rdi), %xmm23387; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3388; SSE-NEXT: movdqa 176(%rdi), %xmm03389; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3390; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,2,2,3]3391; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm7[0,1,0,2,4,5,6,7]3392; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm2[0,2,2,3]3393; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm6[0,1,0,2,4,5,6,7]3394; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]3395; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]3396; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3397; SSE-NEXT: movdqa 384(%rdi), %xmm03398; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3399; SSE-NEXT: movdqa 400(%rdi), %xmm13400; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3401; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,2,2,3]3402; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm5[0,2,2,3,4,5,6,7]3403; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,2,2,3]3404; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm4[0,2,2,3,4,5,6,7]3405; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]3406; SSE-NEXT: movdqa 416(%rdi), %xmm23407; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3408; SSE-NEXT: movdqa 432(%rdi), %xmm13409; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3410; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,2,2,3]3411; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm3[0,1,0,2,4,5,6,7]3412; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]3413; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm2[0,1,0,2,4,5,6,7]3414; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm1[0],xmm15[1],xmm1[1]3415; SSE-NEXT: movsd {{.*#+}} xmm15 = xmm0[0],xmm15[1]3416; SSE-NEXT: movapd %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3417; SSE-NEXT: pshuflw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload3418; SSE-NEXT: # xmm0 = mem[1,3,2,3,4,5,6,7]3419; SSE-NEXT: pshuflw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3420; SSE-NEXT: # xmm1 = mem[1,3,2,3,4,5,6,7]3421; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]3422; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload3423; SSE-NEXT: # xmm0 = mem[0,1,1,3,4,5,6,7]3424; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload3425; SSE-NEXT: # xmm15 = mem[0,1,1,3,4,5,6,7]3426; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]3427; SSE-NEXT: movsd {{.*#+}} xmm15 = xmm1[0],xmm15[1]3428; SSE-NEXT: movapd %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3429; SSE-NEXT: pshuflw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload3430; SSE-NEXT: # xmm0 = mem[1,3,2,3,4,5,6,7]3431; SSE-NEXT: pshuflw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3432; SSE-NEXT: # xmm1 = mem[1,3,2,3,4,5,6,7]3433; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]3434; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload3435; SSE-NEXT: # xmm0 = mem[0,1,1,3,4,5,6,7]3436; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload3437; SSE-NEXT: # xmm15 = mem[0,1,1,3,4,5,6,7]3438; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]3439; SSE-NEXT: movsd {{.*#+}} xmm15 = xmm1[0],xmm15[1]3440; SSE-NEXT: movapd %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3441; SSE-NEXT: pshuflw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload3442; SSE-NEXT: # xmm0 = mem[1,3,2,3,4,5,6,7]3443; SSE-NEXT: pshuflw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3444; SSE-NEXT: # xmm1 = mem[1,3,2,3,4,5,6,7]3445; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]3446; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload3447; SSE-NEXT: # xmm0 = mem[0,1,1,3,4,5,6,7]3448; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload3449; SSE-NEXT: # xmm15 = mem[0,1,1,3,4,5,6,7]3450; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]3451; SSE-NEXT: movsd {{.*#+}} xmm15 = xmm1[0],xmm15[1]3452; SSE-NEXT: movapd %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3453; SSE-NEXT: pshuflw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload3454; SSE-NEXT: # xmm0 = mem[1,3,2,3,4,5,6,7]3455; SSE-NEXT: pshuflw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3456; SSE-NEXT: # xmm1 = mem[1,3,2,3,4,5,6,7]3457; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]3458; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload3459; SSE-NEXT: # xmm0 = mem[0,1,1,3,4,5,6,7]3460; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload3461; SSE-NEXT: # xmm15 = mem[0,1,1,3,4,5,6,7]3462; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]3463; SSE-NEXT: movsd {{.*#+}} xmm15 = xmm1[0],xmm15[1]3464; SSE-NEXT: movapd %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3465; SSE-NEXT: pshuflw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload3466; SSE-NEXT: # xmm0 = mem[1,3,2,3,4,5,6,7]3467; SSE-NEXT: pshuflw $237, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3468; SSE-NEXT: # xmm1 = mem[1,3,2,3,4,5,6,7]3469; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]3470; SSE-NEXT: pshuflw $212, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload3471; SSE-NEXT: # xmm0 = mem[0,1,1,3,4,5,6,7]3472; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm14[0,1,1,3,4,5,6,7]3473; SSE-NEXT: punpckldq {{.*#+}} xmm14 = xmm14[0],xmm0[0],xmm14[1],xmm0[1]3474; SSE-NEXT: movsd {{.*#+}} xmm14 = xmm1[0],xmm14[1]3475; SSE-NEXT: movapd %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3476; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm13[1,3,2,3,4,5,6,7]3477; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm12[1,3,2,3,4,5,6,7]3478; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]3479; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm11[0,1,1,3,4,5,6,7]3480; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]3481; SSE-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm0[0],xmm10[1],xmm0[1]3482; SSE-NEXT: movsd {{.*#+}} xmm10 = xmm1[0],xmm10[1]3483; SSE-NEXT: movapd %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3484; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm9[1,3,2,3,4,5,6,7]3485; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm8[1,3,2,3,4,5,6,7]3486; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]3487; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm7[0,1,1,3,4,5,6,7]3488; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]3489; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1]3490; SSE-NEXT: movsd {{.*#+}} xmm6 = xmm1[0],xmm6[1]3491; SSE-NEXT: movapd %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3492; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm5[1,3,2,3,4,5,6,7]3493; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm4[1,3,2,3,4,5,6,7]3494; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]3495; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm3[0,1,1,3,4,5,6,7]3496; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7]3497; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]3498; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]3499; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3500; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload3501; SSE-NEXT: # xmm0 = mem[3,1,2,3]3502; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3503; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3504; SSE-NEXT: # xmm1 = mem[3,1,2,3]3505; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3506; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]3507; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]3508; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]3509; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload3510; SSE-NEXT: # xmm0 = mem[3,1,2,3]3511; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3512; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload3513; SSE-NEXT: # xmm2 = mem[3,1,2,3]3514; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3515; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]3516; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]3517; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]3518; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]3519; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3520; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload3521; SSE-NEXT: # xmm0 = mem[3,1,2,3]3522; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3523; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3524; SSE-NEXT: # xmm1 = mem[3,1,2,3]3525; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3526; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]3527; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]3528; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]3529; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload3530; SSE-NEXT: # xmm0 = mem[3,1,2,3]3531; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3532; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload3533; SSE-NEXT: # xmm2 = mem[3,1,2,3]3534; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3535; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]3536; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]3537; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]3538; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm1[0],xmm2[1]3539; SSE-NEXT: movapd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3540; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload3541; SSE-NEXT: # xmm0 = mem[3,1,2,3]3542; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3543; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3544; SSE-NEXT: # xmm1 = mem[3,1,2,3]3545; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3546; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]3547; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm1[2,0,2,3,4,5,6,7]3548; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]3549; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload3550; SSE-NEXT: # xmm13 = mem[3,1,2,3]3551; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload3552; SSE-NEXT: # xmm8 = mem[3,1,2,3]3553; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm13[0,1,2,0,4,5,6,7]3554; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm8[0,1,2,0,4,5,6,7]3555; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]3556; SSE-NEXT: movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]3557; SSE-NEXT: movapd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3558; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload3559; SSE-NEXT: # xmm0 = mem[3,1,2,3]3560; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3561; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3562; SSE-NEXT: # xmm1 = mem[3,1,2,3]3563; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3564; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]3565; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm1[2,0,2,3,4,5,6,7]3566; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]3567; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload3568; SSE-NEXT: # xmm0 = mem[3,1,2,3]3569; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3570; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3571; SSE-NEXT: # xmm1 = mem[3,1,2,3]3572; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3573; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]3574; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm1[0,1,2,0,4,5,6,7]3575; SSE-NEXT: punpckldq {{.*#+}} xmm14 = xmm14[0],xmm0[0],xmm14[1],xmm0[1]3576; SSE-NEXT: movsd {{.*#+}} xmm14 = xmm2[0],xmm14[1]3577; SSE-NEXT: pshufd $231, (%rsp), %xmm0 # 16-byte Folded Reload3578; SSE-NEXT: # xmm0 = mem[3,1,2,3]3579; SSE-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill3580; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3581; SSE-NEXT: # xmm1 = mem[3,1,2,3]3582; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3583; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]3584; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm1[2,0,2,3,4,5,6,7]3585; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]3586; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload3587; SSE-NEXT: # xmm15 = mem[3,1,2,3]3588; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload3589; SSE-NEXT: # xmm12 = mem[3,1,2,3]3590; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm15[0,1,2,0,4,5,6,7]3591; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm12[0,1,2,0,4,5,6,7]3592; SSE-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm0[0],xmm10[1],xmm0[1]3593; SSE-NEXT: movsd {{.*#+}} xmm10 = xmm2[0],xmm10[1]3594; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload3595; SSE-NEXT: # xmm0 = mem[3,1,2,3]3596; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3597; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3598; SSE-NEXT: # xmm1 = mem[3,1,2,3]3599; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3600; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]3601; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm1[2,0,2,3,4,5,6,7]3602; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]3603; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload3604; SSE-NEXT: # xmm0 = mem[3,1,2,3]3605; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3606; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3607; SSE-NEXT: # xmm1 = mem[3,1,2,3]3608; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3609; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]3610; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm1[0,1,2,0,4,5,6,7]3611; SSE-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm0[0],xmm9[1],xmm0[1]3612; SSE-NEXT: movsd {{.*#+}} xmm9 = xmm2[0],xmm9[1]3613; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload3614; SSE-NEXT: # xmm7 = mem[3,1,2,3]3615; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload3616; SSE-NEXT: # xmm6 = mem[3,1,2,3]3617; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm7[2,0,2,3,4,5,6,7]3618; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm6[2,0,2,3,4,5,6,7]3619; SSE-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm0[0],xmm11[1],xmm0[1]3620; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload3621; SSE-NEXT: # xmm4 = mem[3,1,2,3]3622; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload3623; SSE-NEXT: # xmm3 = mem[3,1,2,3]3624; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm4[0,1,2,0,4,5,6,7]3625; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm3[0,1,2,0,4,5,6,7]3626; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]3627; SSE-NEXT: movsd {{.*#+}} xmm5 = xmm11[0],xmm5[1]3628; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload3629; SSE-NEXT: # xmm0 = mem[3,1,2,3]3630; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3631; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload3632; SSE-NEXT: # xmm2 = mem[3,1,2,3]3633; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3634; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm0[2,0,2,3,4,5,6,7]3635; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm2[2,0,2,3,4,5,6,7]3636; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]3637; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3638; SSE-NEXT: # xmm1 = mem[3,1,2,3]3639; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3640; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload3641; SSE-NEXT: # xmm11 = mem[3,1,2,3]3642; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]3643; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm11[0,1,2,0,4,5,6,7]3644; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]3645; SSE-NEXT: movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]3646; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload3647; SSE-NEXT: # xmm0 = mem[3,1,2,3,4,5,6,7]3648; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3649; SSE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]3650; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]3651; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm13[0,1,3,1,4,5,6,7]3652; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm8[0,1,3,1,4,5,6,7]3653; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm13[0],xmm0[1],xmm13[1]3654; SSE-NEXT: movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]3655; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3656; SSE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]3657; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload3658; SSE-NEXT: # xmm13 = mem[3,1,2,3,4,5,6,7]3659; SSE-NEXT: punpckldq {{.*#+}} xmm13 = xmm13[0],xmm1[0],xmm13[1],xmm1[1]3660; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3661; SSE-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]3662; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload3663; SSE-NEXT: # xmm8 = mem[0,1,3,1,4,5,6,7]3664; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm1[0],xmm8[1],xmm1[1]3665; SSE-NEXT: movsd {{.*#+}} xmm8 = xmm13[0],xmm8[1]3666; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm7[3,1,2,3,4,5,6,7]3667; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[3,1,2,3,4,5,6,7]3668; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]3669; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm4[0,1,3,1,4,5,6,7]3670; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm3[0,1,3,1,4,5,6,7]3671; SSE-NEXT: punpckldq {{.*#+}} xmm13 = xmm13[0],xmm1[0],xmm13[1],xmm1[1]3672; SSE-NEXT: movsd {{.*#+}} xmm13 = xmm6[0],xmm13[1]3673; SSE-NEXT: pshuflw $231, (%rsp), %xmm1 # 16-byte Folded Reload3674; SSE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]3675; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload3676; SSE-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]3677; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]3678; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm15[0,1,3,1,4,5,6,7]3679; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm12[0,1,3,1,4,5,6,7]3680; SSE-NEXT: punpckldq {{.*#+}} xmm12 = xmm12[0],xmm1[0],xmm12[1],xmm1[1]3681; SSE-NEXT: movsd {{.*#+}} xmm12 = xmm3[0],xmm12[1]3682; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3683; SSE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]3684; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload3685; SSE-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]3686; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]3687; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3688; SSE-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]3689; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload3690; SSE-NEXT: # xmm15 = mem[0,1,3,1,4,5,6,7]3691; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm1[0],xmm15[1],xmm1[1]3692; SSE-NEXT: movsd {{.*#+}} xmm15 = xmm3[0],xmm15[1]3693; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3694; SSE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]3695; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload3696; SSE-NEXT: # xmm4 = mem[3,1,2,3,4,5,6,7]3697; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]3698; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3699; SSE-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]3700; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload3701; SSE-NEXT: # xmm3 = mem[0,1,3,1,4,5,6,7]3702; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]3703; SSE-NEXT: movsd {{.*#+}} xmm3 = xmm4[0],xmm3[1]3704; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3705; SSE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]3706; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload3707; SSE-NEXT: # xmm6 = mem[3,1,2,3,4,5,6,7]3708; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]3709; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3710; SSE-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]3711; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm11[0,1,3,1,4,5,6,7]3712; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]3713; SSE-NEXT: movsd {{.*#+}} xmm4 = xmm6[0],xmm4[1]3714; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3715; SSE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]3716; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload3717; SSE-NEXT: # xmm6 = mem[3,1,2,3,4,5,6,7]3718; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]3719; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload3720; SSE-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]3721; SSE-NEXT: pshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload3722; SSE-NEXT: # xmm7 = mem[0,1,3,1,4,5,6,7]3723; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1]3724; SSE-NEXT: movsd {{.*#+}} xmm7 = xmm6[0],xmm7[1]3725; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3726; SSE-NEXT: movaps %xmm1, 96(%rsi)3727; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3728; SSE-NEXT: movaps %xmm1, 32(%rsi)3729; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3730; SSE-NEXT: movaps %xmm1, 112(%rsi)3731; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3732; SSE-NEXT: movaps %xmm1, 48(%rsi)3733; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3734; SSE-NEXT: movaps %xmm1, 64(%rsi)3735; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3736; SSE-NEXT: movaps %xmm1, (%rsi)3737; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3738; SSE-NEXT: movaps %xmm1, 80(%rsi)3739; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3740; SSE-NEXT: movaps %xmm1, 16(%rsi)3741; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3742; SSE-NEXT: movaps %xmm1, 96(%rdx)3743; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3744; SSE-NEXT: movaps %xmm1, 32(%rdx)3745; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3746; SSE-NEXT: movaps %xmm1, 112(%rdx)3747; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3748; SSE-NEXT: movaps %xmm1, 48(%rdx)3749; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3750; SSE-NEXT: movaps %xmm1, 64(%rdx)3751; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3752; SSE-NEXT: movaps %xmm1, (%rdx)3753; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3754; SSE-NEXT: movaps %xmm1, 80(%rdx)3755; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3756; SSE-NEXT: movaps %xmm1, 16(%rdx)3757; SSE-NEXT: movapd %xmm2, 96(%rcx)3758; SSE-NEXT: movapd %xmm5, 32(%rcx)3759; SSE-NEXT: movapd %xmm9, 112(%rcx)3760; SSE-NEXT: movapd %xmm10, 48(%rcx)3761; SSE-NEXT: movapd %xmm14, 64(%rcx)3762; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3763; SSE-NEXT: movaps %xmm1, (%rcx)3764; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3765; SSE-NEXT: movaps %xmm1, 80(%rcx)3766; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3767; SSE-NEXT: movaps %xmm1, 16(%rcx)3768; SSE-NEXT: movapd %xmm7, 112(%r8)3769; SSE-NEXT: movapd %xmm4, 96(%r8)3770; SSE-NEXT: movapd %xmm3, 80(%r8)3771; SSE-NEXT: movapd %xmm15, 64(%r8)3772; SSE-NEXT: movapd %xmm12, 48(%r8)3773; SSE-NEXT: movapd %xmm13, 32(%r8)3774; SSE-NEXT: movapd %xmm8, 16(%r8)3775; SSE-NEXT: movapd %xmm0, (%r8)3776; SSE-NEXT: addq $824, %rsp # imm = 0x3383777; SSE-NEXT: retq3778;3779; AVX-LABEL: load_i16_stride4_vf64:3780; AVX: # %bb.0:3781; AVX-NEXT: subq $776, %rsp # imm = 0x3083782; AVX-NEXT: vpxor %xmm10, %xmm10, %xmm103783; AVX-NEXT: vmovdqa 368(%rdi), %xmm83784; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm8[0],xmm10[1,2,3],xmm8[4],xmm10[5,6,7]3785; AVX-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3786; AVX-NEXT: vmovdqa 352(%rdi), %xmm13787; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3788; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm10[1,2,3],xmm1[4],xmm10[5,6,7]3789; AVX-NEXT: vpackusdw %xmm0, %xmm1, %xmm03790; AVX-NEXT: vmovdqa 336(%rdi), %xmm43791; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm4[0],xmm10[1,2,3],xmm4[4],xmm10[5,6,7]3792; AVX-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3793; AVX-NEXT: vmovdqa 320(%rdi), %xmm73794; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm7[0],xmm10[1,2,3],xmm7[4],xmm10[5,6,7]3795; AVX-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3796; AVX-NEXT: vpackusdw %xmm1, %xmm2, %xmm13797; AVX-NEXT: vpackusdw %xmm0, %xmm1, %xmm03798; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3799; AVX-NEXT: vmovdqa 304(%rdi), %xmm133800; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm13[0],xmm10[1,2,3],xmm13[4],xmm10[5,6,7]3801; AVX-NEXT: vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3802; AVX-NEXT: vmovdqa 288(%rdi), %xmm13803; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3804; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm10[1,2,3],xmm1[4],xmm10[5,6,7]3805; AVX-NEXT: vpackusdw %xmm0, %xmm1, %xmm03806; AVX-NEXT: vmovdqa 272(%rdi), %xmm143807; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm14[0],xmm10[1,2,3],xmm14[4],xmm10[5,6,7]3808; AVX-NEXT: vmovdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3809; AVX-NEXT: vmovdqa 256(%rdi), %xmm23810; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3811; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm10[1,2,3],xmm2[4],xmm10[5,6,7]3812; AVX-NEXT: vpackusdw %xmm1, %xmm2, %xmm13813; AVX-NEXT: vpackusdw %xmm0, %xmm1, %xmm03814; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3815; AVX-NEXT: vmovdqa 112(%rdi), %xmm03816; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3817; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm10[1,2,3],xmm0[4],xmm10[5,6,7]3818; AVX-NEXT: vmovdqa 96(%rdi), %xmm03819; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3820; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0],xmm10[1,2,3],xmm0[4],xmm10[5,6,7]3821; AVX-NEXT: vpackusdw %xmm2, %xmm3, %xmm23822; AVX-NEXT: vmovdqa 80(%rdi), %xmm33823; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm3[0],xmm10[1,2,3],xmm3[4],xmm10[5,6,7]3824; AVX-NEXT: vmovdqa 64(%rdi), %xmm03825; AVX-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill3826; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm0[0],xmm10[1,2,3],xmm0[4],xmm10[5,6,7]3827; AVX-NEXT: vpackusdw %xmm5, %xmm6, %xmm53828; AVX-NEXT: vpackusdw %xmm2, %xmm5, %xmm23829; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3830; AVX-NEXT: vmovdqa 32(%rdi), %xmm03831; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3832; AVX-NEXT: vmovdqa 48(%rdi), %xmm13833; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3834; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm1[0],xmm10[1,2,3],xmm1[4],xmm10[5,6,7]3835; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm0[0],xmm10[1,2,3],xmm0[4],xmm10[5,6,7]3836; AVX-NEXT: vpackusdw %xmm5, %xmm6, %xmm53837; AVX-NEXT: vmovdqa (%rdi), %xmm03838; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3839; AVX-NEXT: vmovdqa 16(%rdi), %xmm13840; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3841; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm1[0],xmm10[1,2,3],xmm1[4],xmm10[5,6,7]3842; AVX-NEXT: vpblendw {{.*#+}} xmm11 = xmm0[0],xmm10[1,2,3],xmm0[4],xmm10[5,6,7]3843; AVX-NEXT: vpackusdw %xmm9, %xmm11, %xmm93844; AVX-NEXT: vpackusdw %xmm5, %xmm9, %xmm53845; AVX-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3846; AVX-NEXT: vmovdqa 240(%rdi), %xmm03847; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3848; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm0[0],xmm10[1,2,3],xmm0[4],xmm10[5,6,7]3849; AVX-NEXT: vmovdqa 224(%rdi), %xmm153850; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm15[0],xmm10[1,2,3],xmm15[4],xmm10[5,6,7]3851; AVX-NEXT: vmovdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3852; AVX-NEXT: vpackusdw %xmm5, %xmm9, %xmm53853; AVX-NEXT: vmovdqa 208(%rdi), %xmm03854; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3855; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm0[0],xmm10[1,2,3],xmm0[4],xmm10[5,6,7]3856; AVX-NEXT: vmovdqa 192(%rdi), %xmm03857; AVX-NEXT: vpblendw {{.*#+}} xmm11 = xmm0[0],xmm10[1,2,3],xmm0[4],xmm10[5,6,7]3858; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3859; AVX-NEXT: vpackusdw %xmm9, %xmm11, %xmm93860; AVX-NEXT: vpackusdw %xmm5, %xmm9, %xmm53861; AVX-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3862; AVX-NEXT: vmovdqa 176(%rdi), %xmm23863; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3864; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm2[0],xmm10[1,2,3],xmm2[4],xmm10[5,6,7]3865; AVX-NEXT: vmovdqa 160(%rdi), %xmm23866; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3867; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm2[0],xmm10[1,2,3],xmm2[4],xmm10[5,6,7]3868; AVX-NEXT: vpackusdw %xmm5, %xmm9, %xmm53869; AVX-NEXT: vmovdqa 144(%rdi), %xmm93870; AVX-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3871; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0],xmm10[1,2,3],xmm9[4],xmm10[5,6,7]3872; AVX-NEXT: vmovdqa 128(%rdi), %xmm113873; AVX-NEXT: vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3874; AVX-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0],xmm10[1,2,3],xmm11[4],xmm10[5,6,7]3875; AVX-NEXT: vpackusdw %xmm9, %xmm11, %xmm93876; AVX-NEXT: vpackusdw %xmm5, %xmm9, %xmm53877; AVX-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3878; AVX-NEXT: vmovdqa 496(%rdi), %xmm13879; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3880; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm1[0],xmm10[1,2,3],xmm1[4],xmm10[5,6,7]3881; AVX-NEXT: vmovdqa 480(%rdi), %xmm13882; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3883; AVX-NEXT: vpblendw {{.*#+}} xmm11 = xmm1[0],xmm10[1,2,3],xmm1[4],xmm10[5,6,7]3884; AVX-NEXT: vpackusdw %xmm9, %xmm11, %xmm93885; AVX-NEXT: vmovdqa 464(%rdi), %xmm13886; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3887; AVX-NEXT: vpblendw {{.*#+}} xmm11 = xmm1[0],xmm10[1,2,3],xmm1[4],xmm10[5,6,7]3888; AVX-NEXT: vmovdqa 448(%rdi), %xmm13889; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3890; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm1[0],xmm10[1,2,3],xmm1[4],xmm10[5,6,7]3891; AVX-NEXT: vpackusdw %xmm11, %xmm12, %xmm113892; AVX-NEXT: vpackusdw %xmm9, %xmm11, %xmm93893; AVX-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3894; AVX-NEXT: vmovdqa 432(%rdi), %xmm23895; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3896; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm2[0],xmm10[1,2,3],xmm2[4],xmm10[5,6,7]3897; AVX-NEXT: vmovdqa 416(%rdi), %xmm13898; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3899; AVX-NEXT: vpblendw {{.*#+}} xmm11 = xmm1[0],xmm10[1,2,3],xmm1[4],xmm10[5,6,7]3900; AVX-NEXT: vpackusdw %xmm9, %xmm11, %xmm93901; AVX-NEXT: vmovdqa 400(%rdi), %xmm53902; AVX-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3903; AVX-NEXT: vpblendw {{.*#+}} xmm11 = xmm5[0],xmm10[1,2,3],xmm5[4],xmm10[5,6,7]3904; AVX-NEXT: vmovdqa 384(%rdi), %xmm23905; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3906; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm2[0],xmm10[1,2,3],xmm2[4],xmm10[5,6,7]3907; AVX-NEXT: vpackusdw %xmm11, %xmm10, %xmm103908; AVX-NEXT: vpackusdw %xmm9, %xmm10, %xmm93909; AVX-NEXT: vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3910; AVX-NEXT: vpshufd {{.*#+}} xmm9 = xmm8[0,2,2,3]3911; AVX-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]3912; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload3913; AVX-NEXT: # xmm10 = mem[0,2,2,3]3914; AVX-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]3915; AVX-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]3916; AVX-NEXT: vpshufd {{.*#+}} xmm10 = xmm4[0,2,2,3]3917; AVX-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[1,3,2,3,4,5,6,7]3918; AVX-NEXT: vpshufd {{.*#+}} xmm11 = xmm7[0,2,2,3]3919; AVX-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7]3920; AVX-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]3921; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm10[0,1,2,3],xmm9[4,5,6,7]3922; AVX-NEXT: vpshufd {{.*#+}} xmm10 = xmm13[0,2,2,3]3923; AVX-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]3924; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload3925; AVX-NEXT: # xmm11 = mem[0,2,2,3]3926; AVX-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[0,1,1,3,4,5,6,7]3927; AVX-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]3928; AVX-NEXT: vpshufd {{.*#+}} xmm11 = xmm14[0,2,2,3]3929; AVX-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7]3930; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload3931; AVX-NEXT: # xmm12 = mem[0,2,2,3]3932; AVX-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[1,3,2,3,4,5,6,7]3933; AVX-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]3934; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1,2,3],xmm10[4,5,6,7]3935; AVX-NEXT: vinsertf128 $1, %xmm9, %ymm0, %ymm93936; AVX-NEXT: vblendps {{.*#+}} ymm9 = ymm10[0,1,2,3],ymm9[4,5,6,7]3937; AVX-NEXT: vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3938; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload3939; AVX-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[0,2,2,3]3940; AVX-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]3941; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload3942; AVX-NEXT: vpshufd {{.*#+}} xmm10 = xmm2[0,2,2,3]3943; AVX-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]3944; AVX-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]3945; AVX-NEXT: vmovdqa %xmm3, %xmm43946; AVX-NEXT: vpshufd {{.*#+}} xmm10 = xmm3[0,2,2,3]3947; AVX-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[1,3,2,3,4,5,6,7]3948; AVX-NEXT: vmovdqa (%rsp), %xmm3 # 16-byte Reload3949; AVX-NEXT: vpshufd {{.*#+}} xmm11 = xmm3[0,2,2,3]3950; AVX-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7]3951; AVX-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]3952; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm10[0,1,2,3],xmm9[4,5,6,7]3953; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload3954; AVX-NEXT: vpshufd {{.*#+}} xmm10 = xmm6[0,2,2,3]3955; AVX-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]3956; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload3957; AVX-NEXT: vpshufd {{.*#+}} xmm11 = xmm5[0,2,2,3]3958; AVX-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[0,1,1,3,4,5,6,7]3959; AVX-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]3960; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload3961; AVX-NEXT: vpshufd {{.*#+}} xmm11 = xmm8[0,2,2,3]3962; AVX-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7]3963; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload3964; AVX-NEXT: vpshufd {{.*#+}} xmm12 = xmm7[0,2,2,3]3965; AVX-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[1,3,2,3,4,5,6,7]3966; AVX-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]3967; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1,2,3],xmm10[4,5,6,7]3968; AVX-NEXT: vinsertf128 $1, %xmm9, %ymm0, %ymm93969; AVX-NEXT: vblendps {{.*#+}} ymm9 = ymm10[0,1,2,3],ymm9[4,5,6,7]3970; AVX-NEXT: vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3971; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload3972; AVX-NEXT: # xmm9 = mem[0,2,2,3]3973; AVX-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]3974; AVX-NEXT: vpshufd {{.*#+}} xmm10 = xmm15[0,2,2,3]3975; AVX-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]3976; AVX-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]3977; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload3978; AVX-NEXT: # xmm10 = mem[0,2,2,3]3979; AVX-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[1,3,2,3,4,5,6,7]3980; AVX-NEXT: vpshufd {{.*#+}} xmm11 = xmm0[0,2,2,3]3981; AVX-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7]3982; AVX-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]3983; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm10[0,1,2,3],xmm9[4,5,6,7]3984; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload3985; AVX-NEXT: # xmm10 = mem[0,2,2,3]3986; AVX-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]3987; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload3988; AVX-NEXT: # xmm11 = mem[0,2,2,3]3989; AVX-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[0,1,1,3,4,5,6,7]3990; AVX-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]3991; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload3992; AVX-NEXT: # xmm11 = mem[0,2,2,3]3993; AVX-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7]3994; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload3995; AVX-NEXT: # xmm12 = mem[0,2,2,3]3996; AVX-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[1,3,2,3,4,5,6,7]3997; AVX-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]3998; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1,2,3],xmm10[4,5,6,7]3999; AVX-NEXT: vinsertf128 $1, %xmm9, %ymm0, %ymm94000; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm9[4,5,6,7]4001; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4002; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload4003; AVX-NEXT: vpshufd {{.*#+}} xmm9 = xmm13[0,2,2,3]4004; AVX-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]4005; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload4006; AVX-NEXT: vpshufd {{.*#+}} xmm10 = xmm14[0,2,2,3]4007; AVX-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]4008; AVX-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]4009; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload4010; AVX-NEXT: # xmm10 = mem[0,2,2,3]4011; AVX-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[1,3,2,3,4,5,6,7]4012; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload4013; AVX-NEXT: vpshufd {{.*#+}} xmm11 = xmm15[0,2,2,3]4014; AVX-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7]4015; AVX-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]4016; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm10[0,1,2,3],xmm9[4,5,6,7]4017; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload4018; AVX-NEXT: # xmm10 = mem[0,2,2,3]4019; AVX-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7]4020; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload4021; AVX-NEXT: # xmm11 = mem[0,2,2,3]4022; AVX-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[0,1,1,3,4,5,6,7]4023; AVX-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]4024; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload4025; AVX-NEXT: # xmm11 = mem[0,2,2,3]4026; AVX-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7]4027; AVX-NEXT: vpshufd $232, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload4028; AVX-NEXT: # xmm12 = mem[0,2,2,3]4029; AVX-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[1,3,2,3,4,5,6,7]4030; AVX-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]4031; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm11[0,1,2,3],xmm10[4,5,6,7]4032; AVX-NEXT: vinsertf128 $1, %xmm9, %ymm0, %ymm94033; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm9[4,5,6,7]4034; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4035; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm1[3,1,2,3]4036; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4037; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]4038; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4039; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]4040; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]4041; AVX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]4042; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm4[3,1,2,3]4043; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4044; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[3,1,2,3]4045; AVX-NEXT: vmovdqa %xmm2, (%rsp) # 16-byte Spill4046; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]4047; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[2,0,2,3,4,5,6,7]4048; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]4049; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]4050; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm6[3,1,2,3]4051; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4052; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm5[3,1,2,3]4053; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4054; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]4055; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]4056; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]4057; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm8[3,1,2,3]4058; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4059; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm7[3,1,2,3]4060; AVX-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4061; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]4062; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,0,2,3,4,5,6,7]4063; AVX-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]4064; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]4065; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm04066; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]4067; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4068; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload4069; AVX-NEXT: # xmm0 = mem[3,1,2,3]4070; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4071; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload4072; AVX-NEXT: # xmm1 = mem[3,1,2,3]4073; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4074; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]4075; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]4076; AVX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]4077; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload4078; AVX-NEXT: # xmm1 = mem[3,1,2,3]4079; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4080; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload4081; AVX-NEXT: # xmm2 = mem[3,1,2,3]4082; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4083; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]4084; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]4085; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]4086; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]4087; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload4088; AVX-NEXT: # xmm1 = mem[3,1,2,3]4089; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4090; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload4091; AVX-NEXT: # xmm2 = mem[3,1,2,3]4092; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4093; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]4094; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]4095; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]4096; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload4097; AVX-NEXT: # xmm12 = mem[3,1,2,3]4098; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload4099; AVX-NEXT: # xmm11 = mem[3,1,2,3]4100; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm12[2,0,2,3,4,5,6,7]4101; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm11[2,0,2,3,4,5,6,7]4102; AVX-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]4103; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]4104; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm04105; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]4106; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4107; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm13[3,1,2,3]4108; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4109; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm14[3,1,2,3]4110; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4111; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]4112; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm2[0,1,2,0,4,5,6,7]4113; AVX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]4114; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload4115; AVX-NEXT: # xmm1 = mem[3,1,2,3]4116; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4117; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm15[3,1,2,3]4118; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4119; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]4120; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]4121; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]4122; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]4123; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload4124; AVX-NEXT: # xmm1 = mem[3,1,2,3]4125; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4126; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload4127; AVX-NEXT: # xmm2 = mem[3,1,2,3]4128; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4129; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]4130; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]4131; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]4132; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload4133; AVX-NEXT: # xmm13 = mem[3,1,2,3]4134; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload4135; AVX-NEXT: # xmm14 = mem[3,1,2,3]4136; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm13[2,0,2,3,4,5,6,7]4137; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm14[2,0,2,3,4,5,6,7]4138; AVX-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]4139; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]4140; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm04141; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]4142; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4143; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload4144; AVX-NEXT: # xmm10 = mem[3,1,2,3]4145; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload4146; AVX-NEXT: # xmm9 = mem[3,1,2,3]4147; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm10[0,1,2,0,4,5,6,7]4148; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm9[0,1,2,0,4,5,6,7]4149; AVX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]4150; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload4151; AVX-NEXT: # xmm8 = mem[3,1,2,3]4152; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload4153; AVX-NEXT: # xmm7 = mem[3,1,2,3]4154; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm8[2,0,2,3,4,5,6,7]4155; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm7[2,0,2,3,4,5,6,7]4156; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]4157; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0,1,2,3],xmm0[4,5,6,7]4158; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload4159; AVX-NEXT: # xmm6 = mem[3,1,2,3]4160; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload4161; AVX-NEXT: # xmm5 = mem[3,1,2,3]4162; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm6[0,1,2,0,4,5,6,7]4163; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm5[0,1,2,0,4,5,6,7]4164; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]4165; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload4166; AVX-NEXT: # xmm4 = mem[3,1,2,3]4167; AVX-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload4168; AVX-NEXT: # xmm2 = mem[3,1,2,3]4169; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm4[2,0,2,3,4,5,6,7]4170; AVX-NEXT: vpshuflw {{.*#+}} xmm15 = xmm2[2,0,2,3,4,5,6,7]4171; AVX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]4172; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]4173; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm14174; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]4175; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4176; AVX-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload4177; AVX-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]4178; AVX-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload4179; AVX-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]4180; AVX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]4181; AVX-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload4182; AVX-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]4183; AVX-NEXT: vpshuflw $231, (%rsp), %xmm3 # 16-byte Folded Reload4184; AVX-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]4185; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]4186; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]4187; AVX-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload4188; AVX-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]4189; AVX-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload4190; AVX-NEXT: # xmm3 = mem[0,1,3,1,4,5,6,7]4191; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]4192; AVX-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload4193; AVX-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]4194; AVX-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload4195; AVX-NEXT: # xmm15 = mem[3,1,2,3,4,5,6,7]4196; AVX-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm15[0],xmm3[0],xmm15[1],xmm3[1]4197; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]4198; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm04199; AVX-NEXT: vblendps {{.*#+}} ymm15 = ymm1[0,1,2,3],ymm0[4,5,6,7]4200; AVX-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload4201; AVX-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]4202; AVX-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload4203; AVX-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]4204; AVX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]4205; AVX-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload4206; AVX-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]4207; AVX-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload4208; AVX-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]4209; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]4210; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]4211; AVX-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload4212; AVX-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]4213; AVX-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload4214; AVX-NEXT: # xmm3 = mem[0,1,3,1,4,5,6,7]4215; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]4216; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm12[3,1,2,3,4,5,6,7]4217; AVX-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[3,1,2,3,4,5,6,7]4218; AVX-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm11[0],xmm3[0],xmm11[1],xmm3[1]4219; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]4220; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm04221; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm1[0,1,2,3],ymm0[4,5,6,7]4222; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm10[0,1,3,1,4,5,6,7]4223; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm9[0,1,3,1,4,5,6,7]4224; AVX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]4225; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm8[3,1,2,3,4,5,6,7]4226; AVX-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[3,1,2,3,4,5,6,7]4227; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm7[0],xmm1[0],xmm7[1],xmm1[1]4228; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]4229; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm6[0,1,3,1,4,5,6,7]4230; AVX-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]4231; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]4232; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7]4233; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]4234; AVX-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]4235; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]4236; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm04237; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]4238; AVX-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload4239; AVX-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]4240; AVX-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload4241; AVX-NEXT: # xmm2 = mem[0,1,3,1,4,5,6,7]4242; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]4243; AVX-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload4244; AVX-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7]4245; AVX-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload4246; AVX-NEXT: # xmm4 = mem[3,1,2,3,4,5,6,7]4247; AVX-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]4248; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]4249; AVX-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload4250; AVX-NEXT: # xmm2 = mem[0,1,3,1,4,5,6,7]4251; AVX-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload4252; AVX-NEXT: # xmm4 = mem[0,1,3,1,4,5,6,7]4253; AVX-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]4254; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm13[3,1,2,3,4,5,6,7]4255; AVX-NEXT: vpshuflw {{.*#+}} xmm5 = xmm14[3,1,2,3,4,5,6,7]4256; AVX-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]4257; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3],xmm2[4,5,6,7]4258; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm14259; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]4260; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload4261; AVX-NEXT: vmovaps %xmm2, 96(%rsi)4262; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload4263; AVX-NEXT: vmovaps %xmm2, 112(%rsi)4264; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload4265; AVX-NEXT: vmovaps %xmm2, 32(%rsi)4266; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload4267; AVX-NEXT: vmovaps %xmm2, 48(%rsi)4268; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload4269; AVX-NEXT: vmovaps %xmm2, (%rsi)4270; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload4271; AVX-NEXT: vmovaps %xmm2, 16(%rsi)4272; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload4273; AVX-NEXT: vmovaps %xmm2, 64(%rsi)4274; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload4275; AVX-NEXT: vmovaps %xmm2, 80(%rsi)4276; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload4277; AVX-NEXT: vmovaps %ymm2, 96(%rdx)4278; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload4279; AVX-NEXT: vmovaps %ymm2, 32(%rdx)4280; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload4281; AVX-NEXT: vmovaps %ymm2, (%rdx)4282; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload4283; AVX-NEXT: vmovaps %ymm2, 64(%rdx)4284; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload4285; AVX-NEXT: vmovaps %ymm2, 32(%rcx)4286; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload4287; AVX-NEXT: vmovaps %ymm2, 96(%rcx)4288; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload4289; AVX-NEXT: vmovaps %ymm2, 64(%rcx)4290; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload4291; AVX-NEXT: vmovaps %ymm2, (%rcx)4292; AVX-NEXT: vmovaps %ymm1, 96(%r8)4293; AVX-NEXT: vmovaps %ymm0, 32(%r8)4294; AVX-NEXT: vmovaps %ymm3, 64(%r8)4295; AVX-NEXT: vmovaps %ymm15, (%r8)4296; AVX-NEXT: addq $776, %rsp # imm = 0x3084297; AVX-NEXT: vzeroupper4298; AVX-NEXT: retq4299;4300; AVX2-LABEL: load_i16_stride4_vf64:4301; AVX2: # %bb.0:4302; AVX2-NEXT: subq $696, %rsp # imm = 0x2B84303; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm04304; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4305; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm24306; AVX2-NEXT: vpackusdw %xmm2, %xmm1, %xmm14307; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4308; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm34309; AVX2-NEXT: vpackusdw %xmm3, %xmm2, %xmm24310; AVX2-NEXT: vpackusdw %xmm1, %xmm2, %xmm14311; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4312; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm34313; AVX2-NEXT: vpackusdw %xmm3, %xmm2, %xmm24314; AVX2-NEXT: vpackusdw %xmm2, %xmm2, %xmm24315; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm24316; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4317; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm44318; AVX2-NEXT: vpackusdw %xmm4, %xmm3, %xmm34319; AVX2-NEXT: vpackusdw %xmm3, %xmm3, %xmm34320; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm34321; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]4322; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]4323; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4324; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4325; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm24326; AVX2-NEXT: vpackusdw %xmm2, %xmm1, %xmm14327; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4328; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm34329; AVX2-NEXT: vpackusdw %xmm3, %xmm2, %xmm24330; AVX2-NEXT: vpackusdw %xmm1, %xmm2, %xmm14331; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4332; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm34333; AVX2-NEXT: vpackusdw %xmm3, %xmm2, %xmm24334; AVX2-NEXT: vpackusdw %xmm2, %xmm2, %xmm24335; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm24336; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4337; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm44338; AVX2-NEXT: vpackusdw %xmm4, %xmm3, %xmm34339; AVX2-NEXT: vpackusdw %xmm3, %xmm3, %xmm34340; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm34341; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]4342; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]4343; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4344; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4345; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm24346; AVX2-NEXT: vpackusdw %xmm2, %xmm1, %xmm14347; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4348; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm34349; AVX2-NEXT: vpackusdw %xmm3, %xmm2, %xmm24350; AVX2-NEXT: vpackusdw %xmm1, %xmm2, %xmm14351; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4352; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm34353; AVX2-NEXT: vpackusdw %xmm3, %xmm2, %xmm24354; AVX2-NEXT: vpackusdw %xmm2, %xmm2, %xmm24355; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm24356; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4357; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm44358; AVX2-NEXT: vpackusdw %xmm4, %xmm3, %xmm34359; AVX2-NEXT: vpackusdw %xmm3, %xmm3, %xmm34360; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm34361; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]4362; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]4363; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4364; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4365; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm24366; AVX2-NEXT: vpackusdw %xmm2, %xmm1, %xmm14367; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4368; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm34369; AVX2-NEXT: vpackusdw %xmm3, %xmm2, %xmm24370; AVX2-NEXT: vpackusdw %xmm1, %xmm2, %xmm14371; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4372; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm34373; AVX2-NEXT: vpackusdw %xmm3, %xmm2, %xmm24374; AVX2-NEXT: vpackusdw %xmm2, %xmm2, %xmm24375; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm24376; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4377; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm34378; AVX2-NEXT: vpackusdw %xmm3, %xmm0, %xmm04379; AVX2-NEXT: vpackusdw %xmm0, %xmm0, %xmm04380; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm04381; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]4382; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]4383; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4384; AVX2-NEXT: vmovdqa 256(%rdi), %xmm44385; AVX2-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4386; AVX2-NEXT: vmovdqa 272(%rdi), %xmm34387; AVX2-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4388; AVX2-NEXT: vmovdqa 288(%rdi), %xmm54389; AVX2-NEXT: vmovdqa %xmm5, (%rsp) # 16-byte Spill4390; AVX2-NEXT: vmovdqa 304(%rdi), %xmm64391; AVX2-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4392; AVX2-NEXT: vmovdqa 368(%rdi), %xmm04393; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4394; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]4395; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]4396; AVX2-NEXT: vmovdqa 352(%rdi), %xmm14397; AVX2-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4398; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]4399; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]4400; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]4401; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm04402; AVX2-NEXT: vmovdqa 336(%rdi), %xmm14403; AVX2-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4404; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]4405; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]4406; AVX2-NEXT: vmovdqa 320(%rdi), %xmm24407; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4408; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]4409; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]4410; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]4411; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm14412; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]4413; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm6[0,2,2,3]4414; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]4415; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm5[0,2,2,3]4416; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7]4417; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]4418; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]4419; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]4420; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]4421; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[1,3,2,3,4,5,6,7]4422; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]4423; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]4424; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]4425; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4426; AVX2-NEXT: vmovdqa 112(%rdi), %xmm04427; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4428; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]4429; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]4430; AVX2-NEXT: vmovdqa 96(%rdi), %xmm14431; AVX2-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4432; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]4433; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]4434; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]4435; AVX2-NEXT: vmovdqa 80(%rdi), %xmm14436; AVX2-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4437; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]4438; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]4439; AVX2-NEXT: vmovdqa 64(%rdi), %xmm24440; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4441; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]4442; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]4443; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]4444; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm04445; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm14446; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]4447; AVX2-NEXT: vmovdqa 48(%rdi), %xmm144448; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm14[0,2,2,3]4449; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]4450; AVX2-NEXT: vmovdqa 32(%rdi), %xmm84451; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm8[0,2,2,3]4452; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7]4453; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]4454; AVX2-NEXT: vmovdqa 16(%rdi), %xmm124455; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm12[0,2,2,3]4456; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]4457; AVX2-NEXT: vmovdqa (%rdi), %xmm94458; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm9[0,2,2,3]4459; AVX2-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7]4460; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]4461; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]4462; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]4463; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4464; AVX2-NEXT: vmovdqa 240(%rdi), %xmm04465; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4466; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]4467; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]4468; AVX2-NEXT: vmovdqa 224(%rdi), %xmm14469; AVX2-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4470; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]4471; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]4472; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]4473; AVX2-NEXT: vmovdqa 208(%rdi), %xmm14474; AVX2-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4475; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]4476; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]4477; AVX2-NEXT: vmovdqa 192(%rdi), %xmm24478; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4479; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]4480; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]4481; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]4482; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm04483; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm14484; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]4485; AVX2-NEXT: vmovdqa 176(%rdi), %xmm14486; AVX2-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4487; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]4488; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]4489; AVX2-NEXT: vmovdqa 160(%rdi), %xmm24490; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4491; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]4492; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7]4493; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]4494; AVX2-NEXT: vmovdqa 144(%rdi), %xmm24495; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4496; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]4497; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7]4498; AVX2-NEXT: vmovdqa 128(%rdi), %xmm34499; AVX2-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4500; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[0,2,2,3]4501; AVX2-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7]4502; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]4503; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]4504; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]4505; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4506; AVX2-NEXT: vmovdqa 496(%rdi), %xmm114507; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm11[0,2,2,3]4508; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]4509; AVX2-NEXT: vmovdqa 480(%rdi), %xmm134510; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm13[0,2,2,3]4511; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]4512; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]4513; AVX2-NEXT: vmovdqa 464(%rdi), %xmm154514; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm15[0,2,2,3]4515; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]4516; AVX2-NEXT: vmovdqa 448(%rdi), %xmm64517; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm6[0,2,2,3]4518; AVX2-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7]4519; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]4520; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm04521; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm14522; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm1[0,1,2,3,4,5],ymm0[6,7]4523; AVX2-NEXT: vmovdqa 432(%rdi), %xmm54524; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm5[0,2,2,3]4525; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]4526; AVX2-NEXT: vmovdqa 416(%rdi), %xmm44527; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[0,2,2,3]4528; AVX2-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7]4529; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm7[0],xmm1[0],xmm7[1],xmm1[1]4530; AVX2-NEXT: vmovdqa 400(%rdi), %xmm74531; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm7[0,2,2,3]4532; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm1[1,3,2,3,4,5,6,7]4533; AVX2-NEXT: vmovdqa 384(%rdi), %xmm14534; AVX2-NEXT: vpshufd {{.*#+}} xmm10 = xmm1[0,2,2,3]4535; AVX2-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[1,3,2,3,4,5,6,7]4536; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm10[0],xmm0[0],xmm10[1],xmm0[1]4537; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]4538; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]4539; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4540; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload4541; AVX2-NEXT: # xmm0 = mem[3,1,2,3]4542; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4543; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload4544; AVX2-NEXT: # xmm2 = mem[3,1,2,3]4545; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4546; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]4547; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]4548; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]4549; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload4550; AVX2-NEXT: # xmm2 = mem[3,1,2,3]4551; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4552; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload4553; AVX2-NEXT: # xmm3 = mem[3,1,2,3]4554; AVX2-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4555; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]4556; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,0,2,3,4,5,6,7]4557; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]4558; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm04559; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm24560; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]4561; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm14[3,1,2,3]4562; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4563; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm8[3,1,2,3]4564; AVX2-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4565; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]4566; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,2,0,4,5,6,7]4567; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]4568; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm12[3,1,2,3]4569; AVX2-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4570; AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm9[3,1,2,3]4571; AVX2-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4572; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,0,2,3,4,5,6,7]4573; AVX2-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[2,0,2,3,4,5,6,7]4574; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm8[0],xmm3[0],xmm8[1],xmm3[1]4575; AVX2-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]4576; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]4577; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4578; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload4579; AVX2-NEXT: # xmm0 = mem[3,1,2,3]4580; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4581; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload4582; AVX2-NEXT: # xmm2 = mem[3,1,2,3]4583; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4584; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]4585; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]4586; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]4587; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload4588; AVX2-NEXT: # xmm2 = mem[3,1,2,3]4589; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4590; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload4591; AVX2-NEXT: # xmm3 = mem[3,1,2,3]4592; AVX2-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4593; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]4594; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,0,2,3,4,5,6,7]4595; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]4596; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm04597; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm24598; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]4599; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload4600; AVX2-NEXT: # xmm2 = mem[3,1,2,3]4601; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4602; AVX2-NEXT: vpshufd $231, (%rsp), %xmm3 # 16-byte Folded Reload4603; AVX2-NEXT: # xmm3 = mem[3,1,2,3]4604; AVX2-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4605; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]4606; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,2,0,4,5,6,7]4607; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]4608; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload4609; AVX2-NEXT: # xmm12 = mem[3,1,2,3]4610; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload4611; AVX2-NEXT: # xmm10 = mem[3,1,2,3]4612; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm12[2,0,2,3,4,5,6,7]4613; AVX2-NEXT: vpshuflw {{.*#+}} xmm14 = xmm10[2,0,2,3,4,5,6,7]4614; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm14[0],xmm3[0],xmm14[1],xmm3[1]4615; AVX2-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]4616; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]4617; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4618; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm11[3,1,2,3]4619; AVX2-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill4620; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm13[3,1,2,3]4621; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4622; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]4623; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]4624; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]4625; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm15[3,1,2,3]4626; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4627; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm6[3,1,2,3]4628; AVX2-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4629; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]4630; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,0,2,3,4,5,6,7]4631; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]4632; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm04633; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm24634; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]4635; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm5[3,1,2,3]4636; AVX2-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4637; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[3,1,2,3]4638; AVX2-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4639; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]4640; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,2,0,4,5,6,7]4641; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]4642; AVX2-NEXT: vpshufd {{.*#+}} xmm13 = xmm7[3,1,2,3]4643; AVX2-NEXT: vpshufd {{.*#+}} xmm11 = xmm1[3,1,2,3]4644; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm13[2,0,2,3,4,5,6,7]4645; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm11[2,0,2,3,4,5,6,7]4646; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]4647; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3]4648; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]4649; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4650; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload4651; AVX2-NEXT: # xmm9 = mem[3,1,2,3]4652; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload4653; AVX2-NEXT: # xmm8 = mem[3,1,2,3]4654; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm9[0,1,2,0,4,5,6,7]4655; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm8[0,1,2,0,4,5,6,7]4656; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]4657; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload4658; AVX2-NEXT: # xmm7 = mem[3,1,2,3]4659; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload4660; AVX2-NEXT: # xmm6 = mem[3,1,2,3]4661; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm7[2,0,2,3,4,5,6,7]4662; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm6[2,0,2,3,4,5,6,7]4663; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]4664; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm04665; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm14666; AVX2-NEXT: vpblendd {{.*#+}} ymm14 = ymm1[0,1,2,3,4,5],ymm0[6,7]4667; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload4668; AVX2-NEXT: # xmm5 = mem[3,1,2,3]4669; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload4670; AVX2-NEXT: # xmm4 = mem[3,1,2,3]4671; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm5[0,1,2,0,4,5,6,7]4672; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm4[0,1,2,0,4,5,6,7]4673; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]4674; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload4675; AVX2-NEXT: # xmm3 = mem[3,1,2,3]4676; AVX2-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload4677; AVX2-NEXT: # xmm2 = mem[3,1,2,3]4678; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm3[2,0,2,3,4,5,6,7]4679; AVX2-NEXT: vpshuflw {{.*#+}} xmm15 = xmm2[2,0,2,3,4,5,6,7]4680; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]4681; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]4682; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm14[4,5,6,7]4683; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4684; AVX2-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload4685; AVX2-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]4686; AVX2-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload4687; AVX2-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]4688; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]4689; AVX2-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload4690; AVX2-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]4691; AVX2-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload4692; AVX2-NEXT: # xmm14 = mem[3,1,2,3,4,5,6,7]4693; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm14[0],xmm1[0],xmm14[1],xmm1[1]4694; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm04695; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm14696; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]4697; AVX2-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload4698; AVX2-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]4699; AVX2-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload4700; AVX2-NEXT: # xmm14 = mem[0,1,3,1,4,5,6,7]4701; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm14[0],xmm1[0],xmm14[1],xmm1[1]4702; AVX2-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload4703; AVX2-NEXT: # xmm14 = mem[3,1,2,3,4,5,6,7]4704; AVX2-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload4705; AVX2-NEXT: # xmm15 = mem[3,1,2,3,4,5,6,7]4706; AVX2-NEXT: vpunpckldq {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1]4707; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm14[0,1],xmm1[2,3]4708; AVX2-NEXT: vpblendd {{.*#+}} ymm15 = ymm1[0,1,2,3],ymm0[4,5,6,7]4709; AVX2-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload4710; AVX2-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]4711; AVX2-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload4712; AVX2-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]4713; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]4714; AVX2-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload4715; AVX2-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]4716; AVX2-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload4717; AVX2-NEXT: # xmm14 = mem[3,1,2,3,4,5,6,7]4718; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm14[0],xmm1[0],xmm14[1],xmm1[1]4719; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm04720; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm14721; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]4722; AVX2-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload4723; AVX2-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]4724; AVX2-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload4725; AVX2-NEXT: # xmm14 = mem[0,1,3,1,4,5,6,7]4726; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm14[0],xmm1[0],xmm14[1],xmm1[1]4727; AVX2-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[3,1,2,3,4,5,6,7]4728; AVX2-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[3,1,2,3,4,5,6,7]4729; AVX2-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm12[0],xmm10[1],xmm12[1]4730; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm10[0,1],xmm1[2,3]4731; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5,6,7]4732; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm9[0,1,3,1,4,5,6,7]4733; AVX2-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,1,3,1,4,5,6,7]4734; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm8[0],xmm0[0],xmm8[1],xmm0[1]4735; AVX2-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[3,1,2,3,4,5,6,7]4736; AVX2-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[3,1,2,3,4,5,6,7]4737; AVX2-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]4738; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm04739; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm64740; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1,2,3,4,5],ymm0[6,7]4741; AVX2-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]4742; AVX2-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]4743; AVX2-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]4744; AVX2-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7]4745; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]4746; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]4747; AVX2-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3]4748; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]4749; AVX2-NEXT: vpshuflw $116, (%rsp), %xmm2 # 16-byte Folded Reload4750; AVX2-NEXT: # xmm2 = mem[0,1,3,1,4,5,6,7]4751; AVX2-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload4752; AVX2-NEXT: # xmm3 = mem[0,1,3,1,4,5,6,7]4753; AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]4754; AVX2-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload4755; AVX2-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]4756; AVX2-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload4757; AVX2-NEXT: # xmm4 = mem[3,1,2,3,4,5,6,7]4758; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]4759; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm24760; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm34761; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]4762; AVX2-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload4763; AVX2-NEXT: # xmm3 = mem[0,1,3,1,4,5,6,7]4764; AVX2-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload4765; AVX2-NEXT: # xmm4 = mem[0,1,3,1,4,5,6,7]4766; AVX2-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]4767; AVX2-NEXT: vpshuflw {{.*#+}} xmm4 = xmm13[3,1,2,3,4,5,6,7]4768; AVX2-NEXT: vpshuflw {{.*#+}} xmm5 = xmm11[3,1,2,3,4,5,6,7]4769; AVX2-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]4770; AVX2-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]4771; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]4772; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload4773; AVX2-NEXT: vmovaps %ymm3, 32(%rsi)4774; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload4775; AVX2-NEXT: vmovaps %ymm3, 96(%rsi)4776; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload4777; AVX2-NEXT: vmovaps %ymm3, 64(%rsi)4778; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload4779; AVX2-NEXT: vmovaps %ymm3, (%rsi)4780; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload4781; AVX2-NEXT: vmovaps %ymm3, 96(%rdx)4782; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload4783; AVX2-NEXT: vmovaps %ymm3, 32(%rdx)4784; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload4785; AVX2-NEXT: vmovaps %ymm3, (%rdx)4786; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload4787; AVX2-NEXT: vmovaps %ymm3, 64(%rdx)4788; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload4789; AVX2-NEXT: vmovaps %ymm3, 32(%rcx)4790; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload4791; AVX2-NEXT: vmovaps %ymm3, 96(%rcx)4792; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload4793; AVX2-NEXT: vmovaps %ymm3, 64(%rcx)4794; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload4795; AVX2-NEXT: vmovaps %ymm3, (%rcx)4796; AVX2-NEXT: vmovdqa %ymm2, 96(%r8)4797; AVX2-NEXT: vmovdqa %ymm0, 32(%r8)4798; AVX2-NEXT: vmovdqa %ymm1, 64(%r8)4799; AVX2-NEXT: vmovdqa %ymm15, (%r8)4800; AVX2-NEXT: addq $696, %rsp # imm = 0x2B84801; AVX2-NEXT: vzeroupper4802; AVX2-NEXT: retq4803;4804; AVX2-FP-LABEL: load_i16_stride4_vf64:4805; AVX2-FP: # %bb.0:4806; AVX2-FP-NEXT: subq $712, %rsp # imm = 0x2C84807; AVX2-FP-NEXT: vpxor %xmm0, %xmm0, %xmm04808; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4809; AVX2-FP-NEXT: vextracti128 $1, %ymm1, %xmm24810; AVX2-FP-NEXT: vpackusdw %xmm2, %xmm1, %xmm14811; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4812; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm34813; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm2, %xmm24814; AVX2-FP-NEXT: vpackusdw %xmm1, %xmm2, %xmm14815; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4816; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm34817; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm2, %xmm24818; AVX2-FP-NEXT: vpackusdw %xmm2, %xmm2, %xmm24819; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm24820; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4821; AVX2-FP-NEXT: vextracti128 $1, %ymm3, %xmm44822; AVX2-FP-NEXT: vpackusdw %xmm4, %xmm3, %xmm34823; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm3, %xmm34824; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm34825; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]4826; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]4827; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4828; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4829; AVX2-FP-NEXT: vextracti128 $1, %ymm1, %xmm24830; AVX2-FP-NEXT: vpackusdw %xmm2, %xmm1, %xmm14831; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4832; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm34833; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm2, %xmm24834; AVX2-FP-NEXT: vpackusdw %xmm1, %xmm2, %xmm14835; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4836; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm34837; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm2, %xmm24838; AVX2-FP-NEXT: vpackusdw %xmm2, %xmm2, %xmm24839; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm24840; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4841; AVX2-FP-NEXT: vextracti128 $1, %ymm3, %xmm44842; AVX2-FP-NEXT: vpackusdw %xmm4, %xmm3, %xmm34843; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm3, %xmm34844; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm34845; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]4846; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]4847; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4848; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4849; AVX2-FP-NEXT: vextracti128 $1, %ymm1, %xmm24850; AVX2-FP-NEXT: vpackusdw %xmm2, %xmm1, %xmm14851; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4852; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm34853; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm2, %xmm24854; AVX2-FP-NEXT: vpackusdw %xmm1, %xmm2, %xmm14855; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4856; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm34857; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm2, %xmm24858; AVX2-FP-NEXT: vpackusdw %xmm2, %xmm2, %xmm24859; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm24860; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4861; AVX2-FP-NEXT: vextracti128 $1, %ymm3, %xmm44862; AVX2-FP-NEXT: vpackusdw %xmm4, %xmm3, %xmm34863; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm3, %xmm34864; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm34865; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]4866; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]4867; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4868; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4869; AVX2-FP-NEXT: vextracti128 $1, %ymm1, %xmm24870; AVX2-FP-NEXT: vpackusdw %xmm2, %xmm1, %xmm14871; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4872; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm34873; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm2, %xmm24874; AVX2-FP-NEXT: vpackusdw %xmm1, %xmm2, %xmm14875; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4876; AVX2-FP-NEXT: vextracti128 $1, %ymm2, %xmm34877; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm2, %xmm24878; AVX2-FP-NEXT: vpackusdw %xmm2, %xmm2, %xmm24879; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm24880; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm0[1,2,3],mem[4],ymm0[5,6,7],mem[8],ymm0[9,10,11],mem[12],ymm0[13,14,15]4881; AVX2-FP-NEXT: vextracti128 $1, %ymm0, %xmm34882; AVX2-FP-NEXT: vpackusdw %xmm3, %xmm0, %xmm04883; AVX2-FP-NEXT: vpackusdw %xmm0, %xmm0, %xmm04884; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm04885; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]4886; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]4887; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4888; AVX2-FP-NEXT: vmovdqa 256(%rdi), %xmm54889; AVX2-FP-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4890; AVX2-FP-NEXT: vmovdqa 288(%rdi), %xmm34891; AVX2-FP-NEXT: vmovdqa %xmm3, (%rsp) # 16-byte Spill4892; AVX2-FP-NEXT: vmovdqa 304(%rdi), %xmm74893; AVX2-FP-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4894; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,2,3,2,3,10,11,8,9,10,11,12,13,14,15]4895; AVX2-FP-NEXT: vmovdqa 368(%rdi), %xmm04896; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4897; AVX2-FP-NEXT: vpshufb %xmm4, %xmm0, %xmm04898; AVX2-FP-NEXT: vmovdqa 352(%rdi), %xmm14899; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4900; AVX2-FP-NEXT: vpshufb %xmm4, %xmm1, %xmm14901; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]4902; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm04903; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm6 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]4904; AVX2-FP-NEXT: vmovdqa 336(%rdi), %xmm14905; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4906; AVX2-FP-NEXT: vpshufb %xmm6, %xmm1, %xmm14907; AVX2-FP-NEXT: vmovdqa 320(%rdi), %xmm24908; AVX2-FP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4909; AVX2-FP-NEXT: vpshufb %xmm6, %xmm2, %xmm24910; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]4911; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm14912; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]4913; AVX2-FP-NEXT: vpshufb %xmm4, %xmm7, %xmm14914; AVX2-FP-NEXT: vpshufb %xmm4, %xmm3, %xmm24915; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]4916; AVX2-FP-NEXT: vmovdqa 272(%rdi), %xmm24917; AVX2-FP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4918; AVX2-FP-NEXT: vpshufb %xmm6, %xmm2, %xmm24919; AVX2-FP-NEXT: vpshufb %xmm6, %xmm5, %xmm34920; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]4921; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]4922; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]4923; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4924; AVX2-FP-NEXT: vmovdqa 112(%rdi), %xmm54925; AVX2-FP-NEXT: vpshufb %xmm4, %xmm5, %xmm24926; AVX2-FP-NEXT: vmovdqa 96(%rdi), %xmm114927; AVX2-FP-NEXT: vpshufb %xmm4, %xmm11, %xmm34928; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]4929; AVX2-FP-NEXT: vmovdqa 80(%rdi), %xmm124930; AVX2-FP-NEXT: vpshufb %xmm6, %xmm12, %xmm74931; AVX2-FP-NEXT: vmovdqa 64(%rdi), %xmm154932; AVX2-FP-NEXT: vpshufb %xmm6, %xmm15, %xmm84933; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]4934; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm34935; AVX2-FP-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm74936; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5],ymm3[6,7]4937; AVX2-FP-NEXT: vmovdqa 32(%rdi), %xmm04938; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4939; AVX2-FP-NEXT: vmovdqa 48(%rdi), %xmm34940; AVX2-FP-NEXT: vpshufb %xmm4, %xmm3, %xmm84941; AVX2-FP-NEXT: vpshufb %xmm4, %xmm0, %xmm94942; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]4943; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm04944; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4945; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm14946; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4947; AVX2-FP-NEXT: vpshufb %xmm6, %xmm1, %xmm94948; AVX2-FP-NEXT: vpshufb %xmm6, %xmm0, %xmm104949; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]4950; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]4951; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm7[4,5,6,7]4952; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4953; AVX2-FP-NEXT: vmovdqa 240(%rdi), %xmm04954; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4955; AVX2-FP-NEXT: vpshufb %xmm4, %xmm0, %xmm74956; AVX2-FP-NEXT: vmovdqa 224(%rdi), %xmm04957; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4958; AVX2-FP-NEXT: vpshufb %xmm4, %xmm0, %xmm84959; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]4960; AVX2-FP-NEXT: vmovdqa 208(%rdi), %xmm04961; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4962; AVX2-FP-NEXT: vpshufb %xmm6, %xmm0, %xmm84963; AVX2-FP-NEXT: vmovdqa 192(%rdi), %xmm04964; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4965; AVX2-FP-NEXT: vpshufb %xmm6, %xmm0, %xmm94966; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]4967; AVX2-FP-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm74968; AVX2-FP-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm84969; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3,4,5],ymm7[6,7]4970; AVX2-FP-NEXT: vmovdqa 160(%rdi), %xmm04971; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4972; AVX2-FP-NEXT: vmovdqa 176(%rdi), %xmm14973; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4974; AVX2-FP-NEXT: vpshufb %xmm4, %xmm1, %xmm84975; AVX2-FP-NEXT: vpshufb %xmm4, %xmm0, %xmm94976; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]4977; AVX2-FP-NEXT: vmovdqa 128(%rdi), %xmm04978; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4979; AVX2-FP-NEXT: vmovdqa 144(%rdi), %xmm14980; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4981; AVX2-FP-NEXT: vpshufb %xmm6, %xmm1, %xmm94982; AVX2-FP-NEXT: vpshufb %xmm6, %xmm0, %xmm104983; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]4984; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]4985; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm7[4,5,6,7]4986; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4987; AVX2-FP-NEXT: vmovdqa 496(%rdi), %xmm84988; AVX2-FP-NEXT: vpshufb %xmm4, %xmm8, %xmm94989; AVX2-FP-NEXT: vmovdqa 480(%rdi), %xmm74990; AVX2-FP-NEXT: vpshufb %xmm4, %xmm7, %xmm104991; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm13 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]4992; AVX2-FP-NEXT: vmovdqa 464(%rdi), %xmm104993; AVX2-FP-NEXT: vpshufb %xmm6, %xmm10, %xmm144994; AVX2-FP-NEXT: vmovdqa 448(%rdi), %xmm04995; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4996; AVX2-FP-NEXT: vpshufb %xmm6, %xmm0, %xmm94997; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm9[0],xmm14[0],xmm9[1],xmm14[1]4998; AVX2-FP-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm134999; AVX2-FP-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm95000; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm13[6,7]5001; AVX2-FP-NEXT: vmovdqa 416(%rdi), %xmm145002; AVX2-FP-NEXT: vmovdqa 432(%rdi), %xmm135003; AVX2-FP-NEXT: vpshufb %xmm4, %xmm13, %xmm05004; AVX2-FP-NEXT: vpshufb %xmm4, %xmm14, %xmm45005; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]5006; AVX2-FP-NEXT: vmovdqa 384(%rdi), %xmm05007; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5008; AVX2-FP-NEXT: vmovdqa 400(%rdi), %xmm45009; AVX2-FP-NEXT: vpshufb %xmm6, %xmm4, %xmm15010; AVX2-FP-NEXT: vpshufb %xmm6, %xmm0, %xmm65011; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]5012; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]5013; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm9[4,5,6,7]5014; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5015; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm0 = xmm5[3,1,2,3]5016; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5017; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm1 = xmm11[3,1,2,3]5018; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5019; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]5020; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]5021; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]5022; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm1 = xmm12[3,1,2,3]5023; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5024; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm2 = xmm15[3,1,2,3]5025; AVX2-FP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5026; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]5027; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]5028; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]5029; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm05030; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm15031; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]5032; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[3,1,2,3]5033; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5034; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload5035; AVX2-FP-NEXT: # xmm2 = mem[3,1,2,3]5036; AVX2-FP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5037; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]5038; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]5039; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]5040; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload5041; AVX2-FP-NEXT: # xmm2 = mem[3,1,2,3]5042; AVX2-FP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5043; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload5044; AVX2-FP-NEXT: # xmm3 = mem[3,1,2,3]5045; AVX2-FP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5046; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]5047; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[2,0,2,3,4,5,6,7]5048; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]5049; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]5050; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]5051; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5052; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload5053; AVX2-FP-NEXT: # xmm0 = mem[3,1,2,3]5054; AVX2-FP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5055; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload5056; AVX2-FP-NEXT: # xmm1 = mem[3,1,2,3]5057; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5058; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]5059; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]5060; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]5061; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload5062; AVX2-FP-NEXT: # xmm1 = mem[3,1,2,3]5063; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5064; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload5065; AVX2-FP-NEXT: # xmm2 = mem[3,1,2,3]5066; AVX2-FP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5067; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]5068; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]5069; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]5070; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm05071; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm15072; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]5073; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload5074; AVX2-FP-NEXT: # xmm1 = mem[3,1,2,3]5075; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5076; AVX2-FP-NEXT: vpshufd $231, (%rsp), %xmm15 # 16-byte Folded Reload5077; AVX2-FP-NEXT: # xmm15 = mem[3,1,2,3]5078; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]5079; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm15[0,1,2,0,4,5,6,7]5080; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]5081; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload5082; AVX2-FP-NEXT: # xmm12 = mem[3,1,2,3]5083; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload5084; AVX2-FP-NEXT: # xmm11 = mem[3,1,2,3]5085; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm12[2,0,2,3,4,5,6,7]5086; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm11[2,0,2,3,4,5,6,7]5087; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]5088; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]5089; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]5090; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5091; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm0 = xmm8[3,1,2,3]5092; AVX2-FP-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill5093; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm1 = xmm7[3,1,2,3]5094; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5095; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,0,4,5,6,7]5096; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]5097; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]5098; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm1 = xmm10[3,1,2,3]5099; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5100; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload5101; AVX2-FP-NEXT: # xmm2 = mem[3,1,2,3]5102; AVX2-FP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5103; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,0,2,3,4,5,6,7]5104; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]5105; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]5106; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm05107; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm15108; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]5109; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm1 = xmm13[3,1,2,3]5110; AVX2-FP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5111; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm2 = xmm14[3,1,2,3]5112; AVX2-FP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5113; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]5114; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]5115; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]5116; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm2 = xmm4[3,1,2,3]5117; AVX2-FP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5118; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload5119; AVX2-FP-NEXT: # xmm10 = mem[3,1,2,3]5120; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]5121; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm10[2,0,2,3,4,5,6,7]5122; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]5123; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]5124; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]5125; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5126; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload5127; AVX2-FP-NEXT: # xmm14 = mem[3,1,2,3]5128; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload5129; AVX2-FP-NEXT: # xmm9 = mem[3,1,2,3]5130; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm14[0,1,2,0,4,5,6,7]5131; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm9[0,1,2,0,4,5,6,7]5132; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]5133; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload5134; AVX2-FP-NEXT: # xmm8 = mem[3,1,2,3]5135; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload5136; AVX2-FP-NEXT: # xmm7 = mem[3,1,2,3]5137; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm8[2,0,2,3,4,5,6,7]5138; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm7[2,0,2,3,4,5,6,7]5139; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]5140; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm05141; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm15142; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm1[0,1,2,3,4,5],ymm0[6,7]5143; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload5144; AVX2-FP-NEXT: # xmm6 = mem[3,1,2,3]5145; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload5146; AVX2-FP-NEXT: # xmm5 = mem[3,1,2,3]5147; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm6[0,1,2,0,4,5,6,7]5148; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm5[0,1,2,0,4,5,6,7]5149; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]5150; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload5151; AVX2-FP-NEXT: # xmm4 = mem[3,1,2,3]5152; AVX2-FP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload5153; AVX2-FP-NEXT: # xmm2 = mem[3,1,2,3]5154; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm4[2,0,2,3,4,5,6,7]5155; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm13 = xmm2[2,0,2,3,4,5,6,7]5156; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm13[0],xmm0[0],xmm13[1],xmm0[1]5157; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]5158; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]5159; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5160; AVX2-FP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload5161; AVX2-FP-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]5162; AVX2-FP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload5163; AVX2-FP-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]5164; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]5165; AVX2-FP-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload5166; AVX2-FP-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]5167; AVX2-FP-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload5168; AVX2-FP-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]5169; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]5170; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm05171; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm15172; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]5173; AVX2-FP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload5174; AVX2-FP-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]5175; AVX2-FP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload5176; AVX2-FP-NEXT: # xmm3 = mem[0,1,3,1,4,5,6,7]5177; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]5178; AVX2-FP-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload5179; AVX2-FP-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]5180; AVX2-FP-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload5181; AVX2-FP-NEXT: # xmm13 = mem[3,1,2,3,4,5,6,7]5182; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm13[0],xmm3[0],xmm13[1],xmm3[1]5183; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]5184; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm13 = ymm1[0,1,2,3],ymm0[4,5,6,7]5185; AVX2-FP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload5186; AVX2-FP-NEXT: # xmm0 = mem[0,1,3,1,4,5,6,7]5187; AVX2-FP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload5188; AVX2-FP-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]5189; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]5190; AVX2-FP-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload5191; AVX2-FP-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7]5192; AVX2-FP-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload5193; AVX2-FP-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]5194; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]5195; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm05196; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm15197; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]5198; AVX2-FP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload5199; AVX2-FP-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]5200; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm15[0,1,3,1,4,5,6,7]5201; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]5202; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm12[3,1,2,3,4,5,6,7]5203; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[3,1,2,3,4,5,6,7]5204; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm11[0],xmm3[0],xmm11[1],xmm3[1]5205; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]5206; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm11 = ymm1[0,1,2,3],ymm0[4,5,6,7]5207; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm14[0,1,3,1,4,5,6,7]5208; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm9[0,1,3,1,4,5,6,7]5209; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]5210; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm8[3,1,2,3,4,5,6,7]5211; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm7[3,1,2,3,4,5,6,7]5212; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]5213; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm05214; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm15215; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]5216; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm6[0,1,3,1,4,5,6,7]5217; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm5[0,1,3,1,4,5,6,7]5218; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]5219; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm4[3,1,2,3,4,5,6,7]5220; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]5221; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]5222; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]5223; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]5224; AVX2-FP-NEXT: vpshuflw $116, (%rsp), %xmm1 # 16-byte Folded Reload5225; AVX2-FP-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]5226; AVX2-FP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload5227; AVX2-FP-NEXT: # xmm2 = mem[0,1,3,1,4,5,6,7]5228; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]5229; AVX2-FP-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload5230; AVX2-FP-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7]5231; AVX2-FP-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload5232; AVX2-FP-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]5233; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]5234; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm15235; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm25236; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]5237; AVX2-FP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload5238; AVX2-FP-NEXT: # xmm2 = mem[0,1,3,1,4,5,6,7]5239; AVX2-FP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload5240; AVX2-FP-NEXT: # xmm3 = mem[0,1,3,1,4,5,6,7]5241; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]5242; AVX2-FP-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload5243; AVX2-FP-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]5244; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm4 = xmm10[3,1,2,3,4,5,6,7]5245; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]5246; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]5247; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]5248; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload5249; AVX2-FP-NEXT: vmovaps %ymm2, 32(%rsi)5250; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload5251; AVX2-FP-NEXT: vmovaps %ymm2, 96(%rsi)5252; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload5253; AVX2-FP-NEXT: vmovaps %ymm2, 64(%rsi)5254; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload5255; AVX2-FP-NEXT: vmovaps %ymm2, (%rsi)5256; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload5257; AVX2-FP-NEXT: vmovaps %ymm2, 96(%rdx)5258; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload5259; AVX2-FP-NEXT: vmovaps %ymm2, 32(%rdx)5260; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload5261; AVX2-FP-NEXT: vmovaps %ymm2, (%rdx)5262; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload5263; AVX2-FP-NEXT: vmovaps %ymm2, 64(%rdx)5264; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload5265; AVX2-FP-NEXT: vmovaps %ymm2, 32(%rcx)5266; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload5267; AVX2-FP-NEXT: vmovaps %ymm2, 96(%rcx)5268; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload5269; AVX2-FP-NEXT: vmovaps %ymm2, 64(%rcx)5270; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload5271; AVX2-FP-NEXT: vmovaps %ymm2, (%rcx)5272; AVX2-FP-NEXT: vmovdqa %ymm1, 96(%r8)5273; AVX2-FP-NEXT: vmovdqa %ymm0, 32(%r8)5274; AVX2-FP-NEXT: vmovdqa %ymm11, 64(%r8)5275; AVX2-FP-NEXT: vmovdqa %ymm13, (%r8)5276; AVX2-FP-NEXT: addq $712, %rsp # imm = 0x2C85277; AVX2-FP-NEXT: vzeroupper5278; AVX2-FP-NEXT: retq5279;5280; AVX2-FCP-LABEL: load_i16_stride4_vf64:5281; AVX2-FCP: # %bb.0:5282; AVX2-FCP-NEXT: subq $680, %rsp # imm = 0x2A85283; AVX2-FCP-NEXT: vmovdqa 320(%rdi), %ymm75284; AVX2-FCP-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5285; AVX2-FCP-NEXT: vmovdqa 352(%rdi), %ymm65286; AVX2-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5287; AVX2-FCP-NEXT: vmovdqa 64(%rdi), %ymm55288; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5289; AVX2-FCP-NEXT: vmovdqa 96(%rdi), %ymm45290; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5291; AVX2-FCP-NEXT: vpxor %xmm1, %xmm1, %xmm15292; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm0 = mem[0],ymm1[1,2,3],mem[4],ymm1[5,6,7],mem[8],ymm1[9,10,11],mem[12],ymm1[13,14,15]5293; AVX2-FCP-NEXT: vextracti128 $1, %ymm0, %xmm25294; AVX2-FCP-NEXT: vpackusdw %xmm2, %xmm0, %xmm05295; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm1[1,2,3],mem[4],ymm1[5,6,7],mem[8],ymm1[9,10,11],mem[12],ymm1[13,14,15]5296; AVX2-FCP-NEXT: vextracti128 $1, %ymm2, %xmm35297; AVX2-FCP-NEXT: vpackusdw %xmm3, %xmm2, %xmm25298; AVX2-FCP-NEXT: vpackusdw %xmm0, %xmm2, %xmm25299; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,2,2,3,0,2,4,6]5300; AVX2-FCP-NEXT: vpermd %ymm4, %ymm0, %ymm35301; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5302; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,16,17,20,21,24,25,28,29]5303; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm3, %ymm35304; AVX2-FCP-NEXT: vpermd %ymm5, %ymm0, %ymm55305; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5306; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm5, %ymm55307; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3,4,5],ymm3[6,7]5308; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]5309; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5310; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm2 = mem[0],ymm1[1,2,3],mem[4],ymm1[5,6,7],mem[8],ymm1[9,10,11],mem[12],ymm1[13,14,15]5311; AVX2-FCP-NEXT: vextracti128 $1, %ymm2, %xmm35312; AVX2-FCP-NEXT: vpackusdw %xmm3, %xmm2, %xmm25313; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm3 = mem[0],ymm1[1,2,3],mem[4],ymm1[5,6,7],mem[8],ymm1[9,10,11],mem[12],ymm1[13,14,15]5314; AVX2-FCP-NEXT: vextracti128 $1, %ymm3, %xmm55315; AVX2-FCP-NEXT: vpackusdw %xmm5, %xmm3, %xmm35316; AVX2-FCP-NEXT: vpackusdw %xmm2, %xmm3, %xmm35317; AVX2-FCP-NEXT: vpermd %ymm6, %ymm0, %ymm25318; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm2, %ymm55319; AVX2-FCP-NEXT: vpermd %ymm7, %ymm0, %ymm65320; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm6, %ymm75321; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0,1,2,3,4,5],ymm5[6,7]5322; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm5[4,5,6,7]5323; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5324; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm3 = mem[0],ymm1[1,2,3],mem[4],ymm1[5,6,7],mem[8],ymm1[9,10,11],mem[12],ymm1[13,14,15]5325; AVX2-FCP-NEXT: vextracti128 $1, %ymm3, %xmm55326; AVX2-FCP-NEXT: vpackusdw %xmm5, %xmm3, %xmm35327; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm5 = mem[0],ymm1[1,2,3],mem[4],ymm1[5,6,7],mem[8],ymm1[9,10,11],mem[12],ymm1[13,14,15]5328; AVX2-FCP-NEXT: vextracti128 $1, %ymm5, %xmm75329; AVX2-FCP-NEXT: vpackusdw %xmm7, %xmm5, %xmm55330; AVX2-FCP-NEXT: vmovdqa 448(%rdi), %ymm75331; AVX2-FCP-NEXT: vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5332; AVX2-FCP-NEXT: vpackusdw %xmm3, %xmm5, %xmm35333; AVX2-FCP-NEXT: vmovdqa 480(%rdi), %ymm55334; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5335; AVX2-FCP-NEXT: vpermd %ymm5, %ymm0, %ymm55336; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5337; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm5, %ymm55338; AVX2-FCP-NEXT: vpermd %ymm7, %ymm0, %ymm145339; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm14, %ymm75340; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0,1,2,3,4,5],ymm5[6,7]5341; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm5[4,5,6,7]5342; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5343; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm3 = mem[0],ymm1[1,2,3],mem[4],ymm1[5,6,7],mem[8],ymm1[9,10,11],mem[12],ymm1[13,14,15]5344; AVX2-FCP-NEXT: vextracti128 $1, %ymm3, %xmm55345; AVX2-FCP-NEXT: vpackusdw %xmm5, %xmm3, %xmm35346; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm1 = mem[0],ymm1[1,2,3],mem[4],ymm1[5,6,7],mem[8],ymm1[9,10,11],mem[12],ymm1[13,14,15]5347; AVX2-FCP-NEXT: vextracti128 $1, %ymm1, %xmm55348; AVX2-FCP-NEXT: vpackusdw %xmm5, %xmm1, %xmm15349; AVX2-FCP-NEXT: vpackusdw %xmm3, %xmm1, %xmm15350; AVX2-FCP-NEXT: vmovdqa 192(%rdi), %ymm55351; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5352; AVX2-FCP-NEXT: vmovdqa 224(%rdi), %ymm35353; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5354; AVX2-FCP-NEXT: vpermd %ymm3, %ymm0, %ymm95355; AVX2-FCP-NEXT: vpermd %ymm5, %ymm0, %ymm35356; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm9, %ymm05357; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm3, %ymm55358; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3,4,5],ymm0[6,7]5359; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]5360; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5361; AVX2-FCP-NEXT: vmovdqa 288(%rdi), %xmm15362; AVX2-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5363; AVX2-FCP-NEXT: vmovdqa 304(%rdi), %xmm125364; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm5 = [0,1,2,3,2,3,10,11,8,9,10,11,12,13,14,15]5365; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm12, %xmm05366; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm1, %xmm15367; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]5368; AVX2-FCP-NEXT: vmovdqa 256(%rdi), %xmm105369; AVX2-FCP-NEXT: vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5370; AVX2-FCP-NEXT: vmovdqa 272(%rdi), %xmm75371; AVX2-FCP-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5372; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]5373; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm7, %xmm75374; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm10, %xmm85375; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]5376; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm7[0,1],xmm1[2,3]5377; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm10 = [2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]5378; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm2, %ymm25379; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm6, %ymm65380; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3,4,5],ymm2[6,7]5381; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]5382; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5383; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %xmm25384; AVX2-FCP-NEXT: vmovdqa 48(%rdi), %xmm15385; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm1, %xmm65386; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm2, %xmm75387; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]5388; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm75389; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm155390; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm15, %xmm115391; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm7, %xmm135392; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1]5393; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm11[0,1],xmm6[2,3]5394; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload5395; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm8, %ymm115396; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload5397; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm8, %ymm135398; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm13[0,1,2,3,4,5],ymm11[6,7]5399; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm11[4,5,6,7]5400; AVX2-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5401; AVX2-FCP-NEXT: vmovdqa 160(%rdi), %xmm85402; AVX2-FCP-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5403; AVX2-FCP-NEXT: vmovdqa 176(%rdi), %xmm65404; AVX2-FCP-NEXT: vmovdqa %xmm6, (%rsp) # 16-byte Spill5405; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm6, %xmm65406; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm8, %xmm115407; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm11[0],xmm6[0],xmm11[1],xmm6[1]5408; AVX2-FCP-NEXT: vmovdqa 128(%rdi), %xmm85409; AVX2-FCP-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5410; AVX2-FCP-NEXT: vmovdqa 144(%rdi), %xmm115411; AVX2-FCP-NEXT: vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5412; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm11, %xmm115413; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm8, %xmm135414; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1]5415; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm11[0,1],xmm6[2,3]5416; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm9, %ymm95417; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm3, %ymm35418; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm9[6,7]5419; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3],ymm3[4,5,6,7]5420; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5421; AVX2-FCP-NEXT: vmovdqa 416(%rdi), %xmm65422; AVX2-FCP-NEXT: vmovdqa 432(%rdi), %xmm135423; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm13, %xmm35424; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm6, %xmm55425; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]5426; AVX2-FCP-NEXT: vmovdqa 384(%rdi), %xmm55427; AVX2-FCP-NEXT: vmovdqa 400(%rdi), %xmm35428; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm115429; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm5, %xmm05430; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm11[0],xmm0[1],xmm11[1]5431; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm9[2,3]5432; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload5433; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm8, %ymm95434; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm14, %ymm115435; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3,4,5],ymm9[6,7]5436; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm9[4,5,6,7]5437; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5438; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [1,3,2,3,1,3,5,7]5439; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload5440; AVX2-FCP-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5441; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm11 # 32-byte Folded Reload5442; AVX2-FCP-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5443; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm8, %ymm95444; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm11, %ymm115445; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3,4,5],ymm9[6,7]5446; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]5447; AVX2-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5448; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]5449; AVX2-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5450; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]5451; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]5452; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]5453; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm2 = xmm15[3,1,2,3]5454; AVX2-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5455; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,1,2,3]5456; AVX2-FCP-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5457; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]5458; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[2,0,2,3,4,5,6,7]5459; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]5460; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]5461; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm9[4,5,6,7]5462; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5463; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload5464; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5465; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload5466; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5467; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm1, %ymm15468; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm2, %ymm25469; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]5470; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm2 = xmm12[3,1,2,3]5471; AVX2-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5472; AVX2-FCP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload5473; AVX2-FCP-NEXT: # xmm14 = mem[3,1,2,3]5474; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]5475; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm7 = xmm14[0,1,2,0,4,5,6,7]5476; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]5477; AVX2-FCP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload5478; AVX2-FCP-NEXT: # xmm11 = mem[3,1,2,3]5479; AVX2-FCP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload5480; AVX2-FCP-NEXT: # xmm9 = mem[3,1,2,3]5481; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm12 = xmm11[2,0,2,3,4,5,6,7]5482; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm15 = xmm9[2,0,2,3,4,5,6,7]5483; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm15[0],xmm12[0],xmm15[1],xmm12[1]5484; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm12[0,1],xmm2[2,3]5485; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]5486; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5487; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload5488; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5489; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload5490; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5491; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm1, %ymm15492; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm2, %ymm25493; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]5494; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm2 = xmm13[3,1,2,3]5495; AVX2-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5496; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,1,2,3]5497; AVX2-FCP-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5498; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]5499; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,2,0,4,5,6,7]5500; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1]5501; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm13 = xmm3[3,1,2,3]5502; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm12 = xmm5[3,1,2,3]5503; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm13[2,0,2,3,4,5,6,7]5504; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm5 = xmm12[2,0,2,3,4,5,6,7]5505; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]5506; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]5507; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]5508; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5509; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload5510; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm7 # 32-byte Folded Reload5511; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm8, %ymm05512; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm7, %ymm15513; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm1[0,1,2,3,4,5],ymm0[6,7]5514; AVX2-FCP-NEXT: vpshufd $231, (%rsp), %xmm6 # 16-byte Folded Reload5515; AVX2-FCP-NEXT: # xmm6 = mem[3,1,2,3]5516; AVX2-FCP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload5517; AVX2-FCP-NEXT: # xmm5 = mem[3,1,2,3]5518; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm6[0,1,2,0,4,5,6,7]5519; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm5[0,1,2,0,4,5,6,7]5520; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]5521; AVX2-FCP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload5522; AVX2-FCP-NEXT: # xmm4 = mem[3,1,2,3]5523; AVX2-FCP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload5524; AVX2-FCP-NEXT: # xmm2 = mem[3,1,2,3]5525; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm4[2,0,2,3,4,5,6,7]5526; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm15 = xmm2[2,0,2,3,4,5,6,7]5527; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]5528; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]5529; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]5530; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5531; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload5532; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm0, %ymm05533; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload5534; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm1, %ymm15535; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]5536; AVX2-FCP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload5537; AVX2-FCP-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]5538; AVX2-FCP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload5539; AVX2-FCP-NEXT: # xmm3 = mem[0,1,3,1,4,5,6,7]5540; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]5541; AVX2-FCP-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload5542; AVX2-FCP-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]5543; AVX2-FCP-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload5544; AVX2-FCP-NEXT: # xmm15 = mem[3,1,2,3,4,5,6,7]5545; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm15[0],xmm3[0],xmm15[1],xmm3[1]5546; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]5547; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm15 = ymm1[0,1,2,3],ymm0[4,5,6,7]5548; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload5549; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm0, %ymm05550; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload5551; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm1, %ymm15552; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]5553; AVX2-FCP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload5554; AVX2-FCP-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]5555; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm14[0,1,3,1,4,5,6,7]5556; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]5557; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm11[3,1,2,3,4,5,6,7]5558; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[3,1,2,3,4,5,6,7]5559; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm9[0],xmm3[0],xmm9[1],xmm3[1]5560; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]5561; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]5562; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm8, %ymm15563; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm7, %ymm35564; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]5565; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm6[0,1,3,1,4,5,6,7]5566; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]5567; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]5568; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7]5569; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]5570; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]5571; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3]5572; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]5573; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload5574; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm2, %ymm25575; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload5576; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm3, %ymm35577; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]5578; AVX2-FCP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload5579; AVX2-FCP-NEXT: # xmm3 = mem[0,1,3,1,4,5,6,7]5580; AVX2-FCP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload5581; AVX2-FCP-NEXT: # xmm4 = mem[0,1,3,1,4,5,6,7]5582; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]5583; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm4 = xmm13[3,1,2,3,4,5,6,7]5584; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm5 = xmm12[3,1,2,3,4,5,6,7]5585; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]5586; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]5587; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]5588; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload5589; AVX2-FCP-NEXT: vmovaps %ymm3, 32(%rsi)5590; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload5591; AVX2-FCP-NEXT: vmovaps %ymm3, 96(%rsi)5592; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload5593; AVX2-FCP-NEXT: vmovaps %ymm3, 64(%rsi)5594; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload5595; AVX2-FCP-NEXT: vmovaps %ymm3, (%rsi)5596; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload5597; AVX2-FCP-NEXT: vmovaps %ymm3, 96(%rdx)5598; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload5599; AVX2-FCP-NEXT: vmovaps %ymm3, 32(%rdx)5600; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload5601; AVX2-FCP-NEXT: vmovaps %ymm3, (%rdx)5602; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload5603; AVX2-FCP-NEXT: vmovaps %ymm3, 64(%rdx)5604; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload5605; AVX2-FCP-NEXT: vmovaps %ymm3, 32(%rcx)5606; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload5607; AVX2-FCP-NEXT: vmovaps %ymm3, 96(%rcx)5608; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload5609; AVX2-FCP-NEXT: vmovaps %ymm3, 64(%rcx)5610; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload5611; AVX2-FCP-NEXT: vmovaps %ymm3, (%rcx)5612; AVX2-FCP-NEXT: vmovdqa %ymm2, 96(%r8)5613; AVX2-FCP-NEXT: vmovdqa %ymm1, 32(%r8)5614; AVX2-FCP-NEXT: vmovdqa %ymm0, 64(%r8)5615; AVX2-FCP-NEXT: vmovdqa %ymm15, (%r8)5616; AVX2-FCP-NEXT: addq $680, %rsp # imm = 0x2A85617; AVX2-FCP-NEXT: vzeroupper5618; AVX2-FCP-NEXT: retq5619;5620; AVX512-LABEL: load_i16_stride4_vf64:5621; AVX512: # %bb.0:5622; AVX512-NEXT: vmovdqa64 256(%rdi), %zmm225623; AVX512-NEXT: vmovdqa64 384(%rdi), %zmm235624; AVX512-NEXT: vmovdqa64 (%rdi), %zmm295625; AVX512-NEXT: vmovdqa64 128(%rdi), %zmm305626; AVX512-NEXT: vmovdqa 192(%rdi), %ymm55627; AVX512-NEXT: vpmovqw %ymm5, %xmm05628; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm05629; AVX512-NEXT: vmovdqa64 240(%rdi), %xmm245630; AVX512-NEXT: vpshufd {{.*#+}} xmm10 = xmm24[0,2,2,3]5631; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm10[0,1,0,2,4,5,6,7]5632; AVX512-NEXT: vmovdqa64 224(%rdi), %xmm265633; AVX512-NEXT: vpshufd {{.*#+}} xmm14 = xmm26[0,2,2,3]5634; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm14[0,1,0,2,4,5,6,7]5635; AVX512-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]5636; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm15637; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]5638; AVX512-NEXT: vpmovqw %zmm30, %xmm15639; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]5640; AVX512-NEXT: vmovdqa 64(%rdi), %ymm65641; AVX512-NEXT: vpmovqw %ymm6, %xmm15642; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm15643; AVX512-NEXT: vmovdqa64 112(%rdi), %xmm275644; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm27[0,2,2,3]5645; AVX512-NEXT: vpshuflw {{.*#+}} xmm7 = xmm3[0,1,0,2,4,5,6,7]5646; AVX512-NEXT: vmovdqa64 96(%rdi), %xmm285647; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm28[0,2,2,3]5648; AVX512-NEXT: vpshuflw {{.*#+}} xmm8 = xmm9[0,1,0,2,4,5,6,7]5649; AVX512-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]5650; AVX512-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm75651; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm7[6,7]5652; AVX512-NEXT: vpmovqw %zmm29, %xmm75653; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm7[0,1,2,3],ymm1[4,5,6,7]5654; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm20 = zmm1[0,1,2,3],zmm0[0,1,2,3]5655; AVX512-NEXT: vmovdqa 448(%rdi), %ymm75656; AVX512-NEXT: vpmovqw %ymm7, %xmm05657; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm85658; AVX512-NEXT: vmovdqa64 496(%rdi), %xmm165659; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm16[0,2,2,3]5660; AVX512-NEXT: vpshuflw {{.*#+}} xmm11 = xmm0[0,1,0,2,4,5,6,7]5661; AVX512-NEXT: vmovdqa64 480(%rdi), %xmm175662; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm17[0,2,2,3]5663; AVX512-NEXT: vpshuflw {{.*#+}} xmm12 = xmm1[0,1,0,2,4,5,6,7]5664; AVX512-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]5665; AVX512-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm115666; AVX512-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm11[6,7]5667; AVX512-NEXT: vpmovqw %zmm23, %xmm115668; AVX512-NEXT: vpblendd {{.*#+}} ymm13 = ymm11[0,1,2,3],ymm8[4,5,6,7]5669; AVX512-NEXT: vmovdqa 320(%rdi), %ymm85670; AVX512-NEXT: vpmovqw %ymm8, %xmm115671; AVX512-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm155672; AVX512-NEXT: vmovdqa64 368(%rdi), %xmm185673; AVX512-NEXT: vpshufd {{.*#+}} xmm11 = xmm18[0,2,2,3]5674; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm11[0,1,0,2,4,5,6,7]5675; AVX512-NEXT: vmovdqa64 352(%rdi), %xmm195676; AVX512-NEXT: vpshufd {{.*#+}} xmm12 = xmm19[0,2,2,3]5677; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm12[0,1,0,2,4,5,6,7]5678; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]5679; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm25680; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm15[0,1,2,3,4,5],ymm2[6,7]5681; AVX512-NEXT: vpmovqw %zmm22, %xmm45682; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]5683; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm21 = zmm2[0,1,2,3],zmm13[0,1,2,3]5684; AVX512-NEXT: vpsrlq $16, %ymm5, %ymm25685; AVX512-NEXT: vpmovqw %ymm2, %xmm25686; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm25687; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm10[0,1,1,3,4,5,6,7]5688; AVX512-NEXT: vpshuflw {{.*#+}} xmm10 = xmm14[0,1,1,3,4,5,6,7]5689; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1]5690; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm45691; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]5692; AVX512-NEXT: vpsrlq $16, %zmm30, %zmm45693; AVX512-NEXT: vpmovqw %zmm4, %xmm45694; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]5695; AVX512-NEXT: vpsrlq $16, %ymm6, %ymm45696; AVX512-NEXT: vpmovqw %ymm4, %xmm45697; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm45698; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]5699; AVX512-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]5700; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm9[0],xmm3[0],xmm9[1],xmm3[1]5701; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm35702; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]5703; AVX512-NEXT: vpsrlq $16, %zmm29, %zmm45704; AVX512-NEXT: vpmovqw %zmm4, %xmm45705; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]5706; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm25 = zmm3[0,1,2,3],zmm2[0,1,2,3]5707; AVX512-NEXT: vpsrlq $16, %ymm7, %ymm25708; AVX512-NEXT: vpmovqw %ymm2, %xmm25709; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm25710; AVX512-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]5711; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]5712; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]5713; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm05714; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]5715; AVX512-NEXT: vpsrlq $16, %zmm23, %zmm15716; AVX512-NEXT: vpmovqw %zmm1, %xmm15717; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]5718; AVX512-NEXT: vpsrlq $16, %ymm8, %ymm15719; AVX512-NEXT: vpmovqw %ymm1, %xmm15720; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm15721; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm11[0,1,1,3,4,5,6,7]5722; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm12[0,1,1,3,4,5,6,7]5723; AVX512-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]5724; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm25725; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]5726; AVX512-NEXT: vpsrlq $16, %zmm22, %zmm25727; AVX512-NEXT: vpmovqw %zmm2, %xmm25728; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]5729; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm31 = zmm1[0,1,2,3],zmm0[0,1,2,3]5730; AVX512-NEXT: vpsrlq $32, %ymm5, %ymm05731; AVX512-NEXT: vpmovqw %ymm0, %xmm05732; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm05733; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm24[3,1,2,3]5734; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm3[0,1,2,0,4,5,6,7]5735; AVX512-NEXT: vpshufd {{.*#+}} xmm12 = xmm26[3,1,2,3]5736; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm12[0,1,2,0,4,5,6,7]5737; AVX512-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]5738; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm15739; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]5740; AVX512-NEXT: vpsrlq $32, %zmm30, %zmm15741; AVX512-NEXT: vpmovqw %zmm1, %xmm15742; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5,6,7]5743; AVX512-NEXT: vpsrlq $32, %ymm6, %ymm05744; AVX512-NEXT: vpmovqw %ymm0, %xmm05745; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm25746; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm27[3,1,2,3]5747; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm9[0,1,2,0,4,5,6,7]5748; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm28[3,1,2,3]5749; AVX512-NEXT: vpshuflw {{.*#+}} xmm10 = xmm0[0,1,2,0,4,5,6,7]5750; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1]5751; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm45752; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]5753; AVX512-NEXT: vpsrlq $32, %zmm29, %zmm45754; AVX512-NEXT: vpmovqw %zmm4, %xmm45755; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]5756; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm24 = zmm2[0,1,2,3],zmm1[0,1,2,3]5757; AVX512-NEXT: vpsrlq $32, %ymm7, %ymm15758; AVX512-NEXT: vpmovqw %ymm1, %xmm15759; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm25760; AVX512-NEXT: vpshufd {{.*#+}} xmm13 = xmm16[3,1,2,3]5761; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm13[0,1,2,0,4,5,6,7]5762; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm17[3,1,2,3]5763; AVX512-NEXT: vpshuflw {{.*#+}} xmm10 = xmm1[0,1,2,0,4,5,6,7]5764; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1]5765; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm45766; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]5767; AVX512-NEXT: vpsrlq $32, %zmm23, %zmm45768; AVX512-NEXT: vpmovqw %zmm4, %xmm45769; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]5770; AVX512-NEXT: vpsrlq $32, %ymm8, %ymm45771; AVX512-NEXT: vpmovqw %ymm4, %xmm45772; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm45773; AVX512-NEXT: vpshufd {{.*#+}} xmm15 = xmm18[3,1,2,3]5774; AVX512-NEXT: vpshuflw {{.*#+}} xmm14 = xmm15[0,1,2,0,4,5,6,7]5775; AVX512-NEXT: vpshufd {{.*#+}} xmm10 = xmm19[3,1,2,3]5776; AVX512-NEXT: vpshuflw {{.*#+}} xmm11 = xmm10[0,1,2,0,4,5,6,7]5777; AVX512-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm14[0],xmm11[1],xmm14[1]5778; AVX512-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm115779; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],ymm11[6,7]5780; AVX512-NEXT: vpsrlq $32, %zmm22, %zmm115781; AVX512-NEXT: vpmovqw %zmm11, %xmm115782; AVX512-NEXT: vpblendd {{.*#+}} ymm4 = ymm11[0,1,2,3],ymm4[4,5,6,7]5783; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm4[0,1,2,3],zmm2[0,1,2,3]5784; AVX512-NEXT: vpsrlq $48, %ymm5, %ymm45785; AVX512-NEXT: vpmovqw %ymm4, %xmm45786; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm45787; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]5788; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm12[0,1,3,1,4,5,6,7]5789; AVX512-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]5790; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm35791; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]5792; AVX512-NEXT: vpsrlq $48, %zmm30, %zmm45793; AVX512-NEXT: vpmovqw %zmm4, %xmm45794; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]5795; AVX512-NEXT: vpsrlq $48, %ymm6, %ymm45796; AVX512-NEXT: vpmovqw %ymm4, %xmm45797; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm45798; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm9[0,1,3,1,4,5,6,7]5799; AVX512-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7]5800; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]5801; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm05802; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]5803; AVX512-NEXT: vpsrlq $48, %zmm29, %zmm45804; AVX512-NEXT: vpmovqw %zmm4, %xmm45805; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]5806; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm3[0,1,2,3]5807; AVX512-NEXT: vpsrlq $48, %ymm7, %ymm35808; AVX512-NEXT: vpmovqw %ymm3, %xmm35809; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm35810; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm13[0,1,3,1,4,5,6,7]5811; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7]5812; AVX512-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]5813; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm15814; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]5815; AVX512-NEXT: vpsrlq $48, %zmm23, %zmm35816; AVX512-NEXT: vpmovqw %zmm3, %xmm35817; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]5818; AVX512-NEXT: vpsrlq $48, %ymm8, %ymm35819; AVX512-NEXT: vpmovqw %ymm3, %xmm35820; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm35821; AVX512-NEXT: vpshuflw {{.*#+}} xmm4 = xmm15[0,1,3,1,4,5,6,7]5822; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm10[0,1,3,1,4,5,6,7]5823; AVX512-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]5824; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm45825; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm4[6,7]5826; AVX512-NEXT: vpsrlq $48, %zmm22, %zmm45827; AVX512-NEXT: vpmovqw %zmm4, %xmm45828; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]5829; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm3[0,1,2,3],zmm1[0,1,2,3]5830; AVX512-NEXT: vmovdqa64 %zmm21, 64(%rsi)5831; AVX512-NEXT: vmovdqa64 %zmm20, (%rsi)5832; AVX512-NEXT: vmovdqa64 %zmm31, 64(%rdx)5833; AVX512-NEXT: vmovdqa64 %zmm25, (%rdx)5834; AVX512-NEXT: vmovdqa64 %zmm2, 64(%rcx)5835; AVX512-NEXT: vmovdqa64 %zmm24, (%rcx)5836; AVX512-NEXT: vmovdqa64 %zmm1, 64(%r8)5837; AVX512-NEXT: vmovdqa64 %zmm0, (%r8)5838; AVX512-NEXT: vzeroupper5839; AVX512-NEXT: retq5840;5841; AVX512-FCP-LABEL: load_i16_stride4_vf64:5842; AVX512-FCP: # %bb.0:5843; AVX512-FCP-NEXT: vmovdqa64 256(%rdi), %zmm225844; AVX512-FCP-NEXT: vmovdqa64 384(%rdi), %zmm255845; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %zmm25846; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %zmm45847; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [16,17,20,21,0,0,0,0,24,25,28,29,0,0,0,0,16,17,20,21,0,0,0,0,24,25,28,29,0,0,0,0]5848; AVX512-FCP-NEXT: # ymm5 = mem[0,1,0,1]5849; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm9 = [0,2,2,3,0,2,4,6]5850; AVX512-FCP-NEXT: vmovdqa64 224(%rdi), %ymm235851; AVX512-FCP-NEXT: vpermd %ymm23, %ymm9, %ymm105852; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm10, %ymm05853; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [0,1,4,5,u,u,u,u,8,9,12,13,12,13,14,15,16,17,20,21,u,u,u,u,24,25,28,29,u,u,u,u]5854; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm245855; AVX512-FCP-NEXT: vpermd %ymm24, %ymm9, %ymm35856; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm15857; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm7 = [0,2,2,3,4,6,12,14]5858; AVX512-FCP-NEXT: vpermt2d %ymm0, %ymm7, %ymm15859; AVX512-FCP-NEXT: vpmovqw %zmm4, %xmm05860; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]5861; AVX512-FCP-NEXT: vmovdqa64 96(%rdi), %ymm265862; AVX512-FCP-NEXT: vpermd %ymm26, %ymm9, %ymm85863; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm8, %ymm115864; AVX512-FCP-NEXT: vmovdqa64 64(%rdi), %ymm275865; AVX512-FCP-NEXT: vpermd %ymm27, %ymm9, %ymm05866; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm125867; AVX512-FCP-NEXT: vpermt2d %ymm11, %ymm7, %ymm125868; AVX512-FCP-NEXT: vpmovqw %zmm2, %xmm115869; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm12[4,5,6,7]5870; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm20 = zmm11[0,1,2,3],zmm1[0,1,2,3]5871; AVX512-FCP-NEXT: vmovdqa64 480(%rdi), %ymm285872; AVX512-FCP-NEXT: vpermd %ymm28, %ymm9, %ymm115873; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm11, %ymm15874; AVX512-FCP-NEXT: vmovdqa64 448(%rdi), %ymm175875; AVX512-FCP-NEXT: vpermd %ymm17, %ymm9, %ymm125876; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm12, %ymm135877; AVX512-FCP-NEXT: vpermt2d %ymm1, %ymm7, %ymm135878; AVX512-FCP-NEXT: vpmovqw %zmm25, %xmm15879; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm14 = ymm1[0,1,2,3],ymm13[4,5,6,7]5880; AVX512-FCP-NEXT: vmovdqa64 352(%rdi), %ymm185881; AVX512-FCP-NEXT: vpermd %ymm18, %ymm9, %ymm15882; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm1, %ymm155883; AVX512-FCP-NEXT: vmovdqa64 320(%rdi), %ymm195884; AVX512-FCP-NEXT: vpermd %ymm19, %ymm9, %ymm135885; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm13, %ymm95886; AVX512-FCP-NEXT: vpermt2d %ymm15, %ymm7, %ymm95887; AVX512-FCP-NEXT: vpmovqw %zmm22, %xmm155888; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm15[0,1,2,3],ymm9[4,5,6,7]5889; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm21 = zmm9[0,1,2,3],zmm14[0,1,2,3]5890; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm9 = [2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]5891; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm10, %ymm105892; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm3, %ymm35893; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm10[6,7]5894; AVX512-FCP-NEXT: vpsrlq $16, %zmm4, %zmm105895; AVX512-FCP-NEXT: vpmovqw %zmm10, %xmm105896; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm10[0,1,2,3],ymm3[4,5,6,7]5897; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm8, %ymm85898; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm0, %ymm05899; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm8[6,7]5900; AVX512-FCP-NEXT: vpsrlq $16, %zmm2, %zmm85901; AVX512-FCP-NEXT: vpmovqw %zmm8, %xmm85902; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm0[4,5,6,7]5903; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm29 = zmm0[0,1,2,3],zmm3[0,1,2,3]5904; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm11, %ymm05905; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm12, %ymm35906; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]5907; AVX512-FCP-NEXT: vpsrlq $16, %zmm25, %zmm35908; AVX512-FCP-NEXT: vpmovqw %zmm3, %xmm35909; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]5910; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm1, %ymm15911; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm13, %ymm35912; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]5913; AVX512-FCP-NEXT: vpsrlq $16, %zmm22, %zmm35914; AVX512-FCP-NEXT: vpmovqw %zmm3, %xmm35915; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]5916; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm16 = zmm1[0,1,2,3],zmm0[0,1,2,3]5917; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm14 = [1,3,2,3,1,3,5,7]5918; AVX512-FCP-NEXT: vpermd %ymm23, %ymm14, %ymm35919; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm3, %ymm05920; AVX512-FCP-NEXT: vpermd %ymm24, %ymm14, %ymm85921; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm8, %ymm15922; AVX512-FCP-NEXT: vpermt2d %ymm0, %ymm7, %ymm15923; AVX512-FCP-NEXT: vpsrlq $32, %zmm4, %zmm05924; AVX512-FCP-NEXT: vpmovqw %zmm0, %xmm05925; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]5926; AVX512-FCP-NEXT: vpermd %ymm26, %ymm14, %ymm05927; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm0, %ymm125928; AVX512-FCP-NEXT: vpermd %ymm27, %ymm14, %ymm115929; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm11, %ymm135930; AVX512-FCP-NEXT: vpermt2d %ymm12, %ymm7, %ymm135931; AVX512-FCP-NEXT: vpsrlq $32, %zmm2, %zmm125932; AVX512-FCP-NEXT: vpmovqw %zmm12, %xmm125933; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3],ymm13[4,5,6,7]5934; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm23 = zmm12[0,1,2,3],zmm1[0,1,2,3]5935; AVX512-FCP-NEXT: vpermd %ymm28, %ymm14, %ymm125936; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm12, %ymm15937; AVX512-FCP-NEXT: vpermd %ymm17, %ymm14, %ymm135938; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm13, %ymm105939; AVX512-FCP-NEXT: vpermt2d %ymm1, %ymm7, %ymm105940; AVX512-FCP-NEXT: vpsrlq $32, %zmm25, %zmm15941; AVX512-FCP-NEXT: vpmovqw %zmm1, %xmm15942; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm1[0,1,2,3],ymm10[4,5,6,7]5943; AVX512-FCP-NEXT: vpermd %ymm18, %ymm14, %ymm15944; AVX512-FCP-NEXT: vpshufb %ymm5, %ymm1, %ymm155945; AVX512-FCP-NEXT: vpermd %ymm19, %ymm14, %ymm55946; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm5, %ymm65947; AVX512-FCP-NEXT: vpermt2d %ymm15, %ymm7, %ymm65948; AVX512-FCP-NEXT: vpsrlq $32, %zmm22, %zmm75949; AVX512-FCP-NEXT: vpmovqw %zmm7, %xmm75950; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7]5951; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm6 = zmm6[0,1,2,3],zmm10[0,1,2,3]5952; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm3, %ymm35953; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm8, %ymm75954; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm7[0,1,2,3,4,5],ymm3[6,7]5955; AVX512-FCP-NEXT: vpsrlq $48, %zmm4, %zmm45956; AVX512-FCP-NEXT: vpmovqw %zmm4, %xmm45957; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]5958; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm0, %ymm05959; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm11, %ymm45960; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]5961; AVX512-FCP-NEXT: vpsrlq $48, %zmm2, %zmm25962; AVX512-FCP-NEXT: vpmovqw %zmm2, %xmm25963; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]5964; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm3[0,1,2,3]5965; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm12, %ymm25966; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm13, %ymm35967; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]5968; AVX512-FCP-NEXT: vpsrlq $48, %zmm25, %zmm35969; AVX512-FCP-NEXT: vpmovqw %zmm3, %xmm35970; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]5971; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm1, %ymm15972; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm5, %ymm35973; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]5974; AVX512-FCP-NEXT: vpsrlq $48, %zmm22, %zmm35975; AVX512-FCP-NEXT: vpmovqw %zmm3, %xmm35976; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]5977; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm1[0,1,2,3],zmm2[0,1,2,3]5978; AVX512-FCP-NEXT: vmovdqa64 %zmm21, 64(%rsi)5979; AVX512-FCP-NEXT: vmovdqa64 %zmm20, (%rsi)5980; AVX512-FCP-NEXT: vmovdqa64 %zmm16, 64(%rdx)5981; AVX512-FCP-NEXT: vmovdqa64 %zmm29, (%rdx)5982; AVX512-FCP-NEXT: vmovdqa64 %zmm6, 64(%rcx)5983; AVX512-FCP-NEXT: vmovdqa64 %zmm23, (%rcx)5984; AVX512-FCP-NEXT: vmovdqa64 %zmm1, 64(%r8)5985; AVX512-FCP-NEXT: vmovdqa64 %zmm0, (%r8)5986; AVX512-FCP-NEXT: vzeroupper5987; AVX512-FCP-NEXT: retq5988;5989; AVX512DQ-LABEL: load_i16_stride4_vf64:5990; AVX512DQ: # %bb.0:5991; AVX512DQ-NEXT: vmovdqa64 256(%rdi), %zmm225992; AVX512DQ-NEXT: vmovdqa64 384(%rdi), %zmm235993; AVX512DQ-NEXT: vmovdqa64 (%rdi), %zmm295994; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %zmm305995; AVX512DQ-NEXT: vmovdqa 192(%rdi), %ymm55996; AVX512DQ-NEXT: vpmovqw %ymm5, %xmm05997; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm05998; AVX512DQ-NEXT: vmovdqa64 240(%rdi), %xmm245999; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm10 = xmm24[0,2,2,3]6000; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm10[0,1,0,2,4,5,6,7]6001; AVX512DQ-NEXT: vmovdqa64 224(%rdi), %xmm266002; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm14 = xmm26[0,2,2,3]6003; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm14[0,1,0,2,4,5,6,7]6004; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]6005; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm16006; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]6007; AVX512DQ-NEXT: vpmovqw %zmm30, %xmm16008; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]6009; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm66010; AVX512DQ-NEXT: vpmovqw %ymm6, %xmm16011; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm16012; AVX512DQ-NEXT: vmovdqa64 112(%rdi), %xmm276013; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm3 = xmm27[0,2,2,3]6014; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm7 = xmm3[0,1,0,2,4,5,6,7]6015; AVX512DQ-NEXT: vmovdqa64 96(%rdi), %xmm286016; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm9 = xmm28[0,2,2,3]6017; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm8 = xmm9[0,1,0,2,4,5,6,7]6018; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]6019; AVX512DQ-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm76020; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm7[6,7]6021; AVX512DQ-NEXT: vpmovqw %zmm29, %xmm76022; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm7[0,1,2,3],ymm1[4,5,6,7]6023; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm20 = zmm1[0,1,2,3],zmm0[0,1,2,3]6024; AVX512DQ-NEXT: vmovdqa 448(%rdi), %ymm76025; AVX512DQ-NEXT: vpmovqw %ymm7, %xmm06026; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm86027; AVX512DQ-NEXT: vmovdqa64 496(%rdi), %xmm166028; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm16[0,2,2,3]6029; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm11 = xmm0[0,1,0,2,4,5,6,7]6030; AVX512DQ-NEXT: vmovdqa64 480(%rdi), %xmm176031; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm17[0,2,2,3]6032; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm12 = xmm1[0,1,0,2,4,5,6,7]6033; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]6034; AVX512DQ-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm116035; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm11[6,7]6036; AVX512DQ-NEXT: vpmovqw %zmm23, %xmm116037; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm13 = ymm11[0,1,2,3],ymm8[4,5,6,7]6038; AVX512DQ-NEXT: vmovdqa 320(%rdi), %ymm86039; AVX512DQ-NEXT: vpmovqw %ymm8, %xmm116040; AVX512DQ-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm156041; AVX512DQ-NEXT: vmovdqa64 368(%rdi), %xmm186042; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm11 = xmm18[0,2,2,3]6043; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm11[0,1,0,2,4,5,6,7]6044; AVX512DQ-NEXT: vmovdqa64 352(%rdi), %xmm196045; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm12 = xmm19[0,2,2,3]6046; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm12[0,1,0,2,4,5,6,7]6047; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]6048; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm26049; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm15[0,1,2,3,4,5],ymm2[6,7]6050; AVX512DQ-NEXT: vpmovqw %zmm22, %xmm46051; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]6052; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm21 = zmm2[0,1,2,3],zmm13[0,1,2,3]6053; AVX512DQ-NEXT: vpsrlq $16, %ymm5, %ymm26054; AVX512DQ-NEXT: vpmovqw %ymm2, %xmm26055; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm26056; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm10[0,1,1,3,4,5,6,7]6057; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm10 = xmm14[0,1,1,3,4,5,6,7]6058; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1]6059; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm46060; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]6061; AVX512DQ-NEXT: vpsrlq $16, %zmm30, %zmm46062; AVX512DQ-NEXT: vpmovqw %zmm4, %xmm46063; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]6064; AVX512DQ-NEXT: vpsrlq $16, %ymm6, %ymm46065; AVX512DQ-NEXT: vpmovqw %ymm4, %xmm46066; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm46067; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]6068; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]6069; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm9[0],xmm3[0],xmm9[1],xmm3[1]6070; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm36071; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]6072; AVX512DQ-NEXT: vpsrlq $16, %zmm29, %zmm46073; AVX512DQ-NEXT: vpmovqw %zmm4, %xmm46074; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]6075; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm25 = zmm3[0,1,2,3],zmm2[0,1,2,3]6076; AVX512DQ-NEXT: vpsrlq $16, %ymm7, %ymm26077; AVX512DQ-NEXT: vpmovqw %ymm2, %xmm26078; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm26079; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]6080; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]6081; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]6082; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm06083; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]6084; AVX512DQ-NEXT: vpsrlq $16, %zmm23, %zmm16085; AVX512DQ-NEXT: vpmovqw %zmm1, %xmm16086; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]6087; AVX512DQ-NEXT: vpsrlq $16, %ymm8, %ymm16088; AVX512DQ-NEXT: vpmovqw %ymm1, %xmm16089; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm16090; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm11[0,1,1,3,4,5,6,7]6091; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm12[0,1,1,3,4,5,6,7]6092; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]6093; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm26094; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]6095; AVX512DQ-NEXT: vpsrlq $16, %zmm22, %zmm26096; AVX512DQ-NEXT: vpmovqw %zmm2, %xmm26097; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]6098; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm31 = zmm1[0,1,2,3],zmm0[0,1,2,3]6099; AVX512DQ-NEXT: vpsrlq $32, %ymm5, %ymm06100; AVX512DQ-NEXT: vpmovqw %ymm0, %xmm06101; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm06102; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm3 = xmm24[3,1,2,3]6103; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm3[0,1,2,0,4,5,6,7]6104; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm12 = xmm26[3,1,2,3]6105; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm12[0,1,2,0,4,5,6,7]6106; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]6107; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm16108; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]6109; AVX512DQ-NEXT: vpsrlq $32, %zmm30, %zmm16110; AVX512DQ-NEXT: vpmovqw %zmm1, %xmm16111; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5,6,7]6112; AVX512DQ-NEXT: vpsrlq $32, %ymm6, %ymm06113; AVX512DQ-NEXT: vpmovqw %ymm0, %xmm06114; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm26115; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm9 = xmm27[3,1,2,3]6116; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm9[0,1,2,0,4,5,6,7]6117; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm28[3,1,2,3]6118; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm10 = xmm0[0,1,2,0,4,5,6,7]6119; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1]6120; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm46121; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]6122; AVX512DQ-NEXT: vpsrlq $32, %zmm29, %zmm46123; AVX512DQ-NEXT: vpmovqw %zmm4, %xmm46124; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]6125; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm24 = zmm2[0,1,2,3],zmm1[0,1,2,3]6126; AVX512DQ-NEXT: vpsrlq $32, %ymm7, %ymm16127; AVX512DQ-NEXT: vpmovqw %ymm1, %xmm16128; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm26129; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm13 = xmm16[3,1,2,3]6130; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm13[0,1,2,0,4,5,6,7]6131; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm17[3,1,2,3]6132; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm10 = xmm1[0,1,2,0,4,5,6,7]6133; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1]6134; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm46135; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]6136; AVX512DQ-NEXT: vpsrlq $32, %zmm23, %zmm46137; AVX512DQ-NEXT: vpmovqw %zmm4, %xmm46138; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]6139; AVX512DQ-NEXT: vpsrlq $32, %ymm8, %ymm46140; AVX512DQ-NEXT: vpmovqw %ymm4, %xmm46141; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm46142; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm15 = xmm18[3,1,2,3]6143; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm14 = xmm15[0,1,2,0,4,5,6,7]6144; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm10 = xmm19[3,1,2,3]6145; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm11 = xmm10[0,1,2,0,4,5,6,7]6146; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm14[0],xmm11[1],xmm14[1]6147; AVX512DQ-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm116148; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],ymm11[6,7]6149; AVX512DQ-NEXT: vpsrlq $32, %zmm22, %zmm116150; AVX512DQ-NEXT: vpmovqw %zmm11, %xmm116151; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm4 = ymm11[0,1,2,3],ymm4[4,5,6,7]6152; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm4[0,1,2,3],zmm2[0,1,2,3]6153; AVX512DQ-NEXT: vpsrlq $48, %ymm5, %ymm46154; AVX512DQ-NEXT: vpmovqw %ymm4, %xmm46155; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm46156; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]6157; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm12[0,1,3,1,4,5,6,7]6158; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]6159; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm36160; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]6161; AVX512DQ-NEXT: vpsrlq $48, %zmm30, %zmm46162; AVX512DQ-NEXT: vpmovqw %zmm4, %xmm46163; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]6164; AVX512DQ-NEXT: vpsrlq $48, %ymm6, %ymm46165; AVX512DQ-NEXT: vpmovqw %ymm4, %xmm46166; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm46167; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm9[0,1,3,1,4,5,6,7]6168; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7]6169; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]6170; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm06171; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]6172; AVX512DQ-NEXT: vpsrlq $48, %zmm29, %zmm46173; AVX512DQ-NEXT: vpmovqw %zmm4, %xmm46174; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]6175; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm3[0,1,2,3]6176; AVX512DQ-NEXT: vpsrlq $48, %ymm7, %ymm36177; AVX512DQ-NEXT: vpmovqw %ymm3, %xmm36178; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm36179; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm13[0,1,3,1,4,5,6,7]6180; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7]6181; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]6182; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm16183; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]6184; AVX512DQ-NEXT: vpsrlq $48, %zmm23, %zmm36185; AVX512DQ-NEXT: vpmovqw %zmm3, %xmm36186; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]6187; AVX512DQ-NEXT: vpsrlq $48, %ymm8, %ymm36188; AVX512DQ-NEXT: vpmovqw %ymm3, %xmm36189; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm36190; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm4 = xmm15[0,1,3,1,4,5,6,7]6191; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm5 = xmm10[0,1,3,1,4,5,6,7]6192; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]6193; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm46194; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm4[6,7]6195; AVX512DQ-NEXT: vpsrlq $48, %zmm22, %zmm46196; AVX512DQ-NEXT: vpmovqw %zmm4, %xmm46197; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]6198; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm3[0,1,2,3],zmm1[0,1,2,3]6199; AVX512DQ-NEXT: vmovdqa64 %zmm21, 64(%rsi)6200; AVX512DQ-NEXT: vmovdqa64 %zmm20, (%rsi)6201; AVX512DQ-NEXT: vmovdqa64 %zmm31, 64(%rdx)6202; AVX512DQ-NEXT: vmovdqa64 %zmm25, (%rdx)6203; AVX512DQ-NEXT: vmovdqa64 %zmm2, 64(%rcx)6204; AVX512DQ-NEXT: vmovdqa64 %zmm24, (%rcx)6205; AVX512DQ-NEXT: vmovdqa64 %zmm1, 64(%r8)6206; AVX512DQ-NEXT: vmovdqa64 %zmm0, (%r8)6207; AVX512DQ-NEXT: vzeroupper6208; AVX512DQ-NEXT: retq6209;6210; AVX512DQ-FCP-LABEL: load_i16_stride4_vf64:6211; AVX512DQ-FCP: # %bb.0:6212; AVX512DQ-FCP-NEXT: vmovdqa64 256(%rdi), %zmm226213; AVX512DQ-FCP-NEXT: vmovdqa64 384(%rdi), %zmm256214; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %zmm26215; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %zmm46216; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [16,17,20,21,0,0,0,0,24,25,28,29,0,0,0,0,16,17,20,21,0,0,0,0,24,25,28,29,0,0,0,0]6217; AVX512DQ-FCP-NEXT: # ymm5 = mem[0,1,0,1]6218; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm9 = [0,2,2,3,0,2,4,6]6219; AVX512DQ-FCP-NEXT: vmovdqa64 224(%rdi), %ymm236220; AVX512DQ-FCP-NEXT: vpermd %ymm23, %ymm9, %ymm106221; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm10, %ymm06222; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [0,1,4,5,u,u,u,u,8,9,12,13,12,13,14,15,16,17,20,21,u,u,u,u,24,25,28,29,u,u,u,u]6223; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm246224; AVX512DQ-FCP-NEXT: vpermd %ymm24, %ymm9, %ymm36225; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm16226; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm7 = [0,2,2,3,4,6,12,14]6227; AVX512DQ-FCP-NEXT: vpermt2d %ymm0, %ymm7, %ymm16228; AVX512DQ-FCP-NEXT: vpmovqw %zmm4, %xmm06229; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]6230; AVX512DQ-FCP-NEXT: vmovdqa64 96(%rdi), %ymm266231; AVX512DQ-FCP-NEXT: vpermd %ymm26, %ymm9, %ymm86232; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm8, %ymm116233; AVX512DQ-FCP-NEXT: vmovdqa64 64(%rdi), %ymm276234; AVX512DQ-FCP-NEXT: vpermd %ymm27, %ymm9, %ymm06235; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm126236; AVX512DQ-FCP-NEXT: vpermt2d %ymm11, %ymm7, %ymm126237; AVX512DQ-FCP-NEXT: vpmovqw %zmm2, %xmm116238; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm12[4,5,6,7]6239; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm20 = zmm11[0,1,2,3],zmm1[0,1,2,3]6240; AVX512DQ-FCP-NEXT: vmovdqa64 480(%rdi), %ymm286241; AVX512DQ-FCP-NEXT: vpermd %ymm28, %ymm9, %ymm116242; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm11, %ymm16243; AVX512DQ-FCP-NEXT: vmovdqa64 448(%rdi), %ymm176244; AVX512DQ-FCP-NEXT: vpermd %ymm17, %ymm9, %ymm126245; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm12, %ymm136246; AVX512DQ-FCP-NEXT: vpermt2d %ymm1, %ymm7, %ymm136247; AVX512DQ-FCP-NEXT: vpmovqw %zmm25, %xmm16248; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm14 = ymm1[0,1,2,3],ymm13[4,5,6,7]6249; AVX512DQ-FCP-NEXT: vmovdqa64 352(%rdi), %ymm186250; AVX512DQ-FCP-NEXT: vpermd %ymm18, %ymm9, %ymm16251; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm1, %ymm156252; AVX512DQ-FCP-NEXT: vmovdqa64 320(%rdi), %ymm196253; AVX512DQ-FCP-NEXT: vpermd %ymm19, %ymm9, %ymm136254; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm13, %ymm96255; AVX512DQ-FCP-NEXT: vpermt2d %ymm15, %ymm7, %ymm96256; AVX512DQ-FCP-NEXT: vpmovqw %zmm22, %xmm156257; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm15[0,1,2,3],ymm9[4,5,6,7]6258; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm21 = zmm9[0,1,2,3],zmm14[0,1,2,3]6259; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm9 = [2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]6260; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm10, %ymm106261; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm3, %ymm36262; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm10[6,7]6263; AVX512DQ-FCP-NEXT: vpsrlq $16, %zmm4, %zmm106264; AVX512DQ-FCP-NEXT: vpmovqw %zmm10, %xmm106265; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm10[0,1,2,3],ymm3[4,5,6,7]6266; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm8, %ymm86267; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm0, %ymm06268; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm8[6,7]6269; AVX512DQ-FCP-NEXT: vpsrlq $16, %zmm2, %zmm86270; AVX512DQ-FCP-NEXT: vpmovqw %zmm8, %xmm86271; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm0[4,5,6,7]6272; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm29 = zmm0[0,1,2,3],zmm3[0,1,2,3]6273; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm11, %ymm06274; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm12, %ymm36275; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]6276; AVX512DQ-FCP-NEXT: vpsrlq $16, %zmm25, %zmm36277; AVX512DQ-FCP-NEXT: vpmovqw %zmm3, %xmm36278; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]6279; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm1, %ymm16280; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm13, %ymm36281; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]6282; AVX512DQ-FCP-NEXT: vpsrlq $16, %zmm22, %zmm36283; AVX512DQ-FCP-NEXT: vpmovqw %zmm3, %xmm36284; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]6285; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm16 = zmm1[0,1,2,3],zmm0[0,1,2,3]6286; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm14 = [1,3,2,3,1,3,5,7]6287; AVX512DQ-FCP-NEXT: vpermd %ymm23, %ymm14, %ymm36288; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm3, %ymm06289; AVX512DQ-FCP-NEXT: vpermd %ymm24, %ymm14, %ymm86290; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm8, %ymm16291; AVX512DQ-FCP-NEXT: vpermt2d %ymm0, %ymm7, %ymm16292; AVX512DQ-FCP-NEXT: vpsrlq $32, %zmm4, %zmm06293; AVX512DQ-FCP-NEXT: vpmovqw %zmm0, %xmm06294; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]6295; AVX512DQ-FCP-NEXT: vpermd %ymm26, %ymm14, %ymm06296; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm0, %ymm126297; AVX512DQ-FCP-NEXT: vpermd %ymm27, %ymm14, %ymm116298; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm11, %ymm136299; AVX512DQ-FCP-NEXT: vpermt2d %ymm12, %ymm7, %ymm136300; AVX512DQ-FCP-NEXT: vpsrlq $32, %zmm2, %zmm126301; AVX512DQ-FCP-NEXT: vpmovqw %zmm12, %xmm126302; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3],ymm13[4,5,6,7]6303; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm23 = zmm12[0,1,2,3],zmm1[0,1,2,3]6304; AVX512DQ-FCP-NEXT: vpermd %ymm28, %ymm14, %ymm126305; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm12, %ymm16306; AVX512DQ-FCP-NEXT: vpermd %ymm17, %ymm14, %ymm136307; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm13, %ymm106308; AVX512DQ-FCP-NEXT: vpermt2d %ymm1, %ymm7, %ymm106309; AVX512DQ-FCP-NEXT: vpsrlq $32, %zmm25, %zmm16310; AVX512DQ-FCP-NEXT: vpmovqw %zmm1, %xmm16311; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm1[0,1,2,3],ymm10[4,5,6,7]6312; AVX512DQ-FCP-NEXT: vpermd %ymm18, %ymm14, %ymm16313; AVX512DQ-FCP-NEXT: vpshufb %ymm5, %ymm1, %ymm156314; AVX512DQ-FCP-NEXT: vpermd %ymm19, %ymm14, %ymm56315; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm5, %ymm66316; AVX512DQ-FCP-NEXT: vpermt2d %ymm15, %ymm7, %ymm66317; AVX512DQ-FCP-NEXT: vpsrlq $32, %zmm22, %zmm76318; AVX512DQ-FCP-NEXT: vpmovqw %zmm7, %xmm76319; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7]6320; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm6 = zmm6[0,1,2,3],zmm10[0,1,2,3]6321; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm3, %ymm36322; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm8, %ymm76323; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm7[0,1,2,3,4,5],ymm3[6,7]6324; AVX512DQ-FCP-NEXT: vpsrlq $48, %zmm4, %zmm46325; AVX512DQ-FCP-NEXT: vpmovqw %zmm4, %xmm46326; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]6327; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm0, %ymm06328; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm11, %ymm46329; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]6330; AVX512DQ-FCP-NEXT: vpsrlq $48, %zmm2, %zmm26331; AVX512DQ-FCP-NEXT: vpmovqw %zmm2, %xmm26332; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]6333; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm3[0,1,2,3]6334; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm12, %ymm26335; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm13, %ymm36336; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]6337; AVX512DQ-FCP-NEXT: vpsrlq $48, %zmm25, %zmm36338; AVX512DQ-FCP-NEXT: vpmovqw %zmm3, %xmm36339; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]6340; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm1, %ymm16341; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm5, %ymm36342; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]6343; AVX512DQ-FCP-NEXT: vpsrlq $48, %zmm22, %zmm36344; AVX512DQ-FCP-NEXT: vpmovqw %zmm3, %xmm36345; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]6346; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm1[0,1,2,3],zmm2[0,1,2,3]6347; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm21, 64(%rsi)6348; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm20, (%rsi)6349; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm16, 64(%rdx)6350; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm29, (%rdx)6351; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm6, 64(%rcx)6352; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm23, (%rcx)6353; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, 64(%r8)6354; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, (%r8)6355; AVX512DQ-FCP-NEXT: vzeroupper6356; AVX512DQ-FCP-NEXT: retq6357;6358; AVX512BW-LABEL: load_i16_stride4_vf64:6359; AVX512BW: # %bb.0:6360; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm06361; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm16362; AVX512BW-NEXT: vmovdqa64 128(%rdi), %zmm26363; AVX512BW-NEXT: vmovdqa64 192(%rdi), %zmm36364; AVX512BW-NEXT: vmovdqa64 320(%rdi), %zmm46365; AVX512BW-NEXT: vmovdqa64 256(%rdi), %zmm56366; AVX512BW-NEXT: vmovdqa64 448(%rdi), %zmm66367; AVX512BW-NEXT: vmovdqa64 384(%rdi), %zmm76368; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm8 = [0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60]6369; AVX512BW-NEXT: # zmm8 = mem[0,1,2,3,0,1,2,3]6370; AVX512BW-NEXT: vmovdqa64 %zmm7, %zmm96371; AVX512BW-NEXT: vpermt2w %zmm6, %zmm8, %zmm96372; AVX512BW-NEXT: vmovdqa64 %zmm5, %zmm106373; AVX512BW-NEXT: vpermt2w %zmm4, %zmm8, %zmm106374; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm9 = zmm10[0,1,2,3],zmm9[4,5,6,7]6375; AVX512BW-NEXT: vmovdqa64 %zmm2, %zmm106376; AVX512BW-NEXT: vpermt2w %zmm3, %zmm8, %zmm106377; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm86378; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm8 = zmm8[0,1,2,3],zmm10[4,5,6,7]6379; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm10 = [1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61,1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61]6380; AVX512BW-NEXT: # zmm10 = mem[0,1,2,3,0,1,2,3]6381; AVX512BW-NEXT: vmovdqa64 %zmm7, %zmm116382; AVX512BW-NEXT: vpermt2w %zmm6, %zmm10, %zmm116383; AVX512BW-NEXT: vmovdqa64 %zmm5, %zmm126384; AVX512BW-NEXT: vpermt2w %zmm4, %zmm10, %zmm126385; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm11 = zmm12[0,1,2,3],zmm11[4,5,6,7]6386; AVX512BW-NEXT: vmovdqa64 %zmm2, %zmm126387; AVX512BW-NEXT: vpermt2w %zmm3, %zmm10, %zmm126388; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm106389; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm10 = zmm10[0,1,2,3],zmm12[4,5,6,7]6390; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm12 = [2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62,2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62]6391; AVX512BW-NEXT: # zmm12 = mem[0,1,2,3,0,1,2,3]6392; AVX512BW-NEXT: vmovdqa64 %zmm7, %zmm136393; AVX512BW-NEXT: vpermt2w %zmm6, %zmm12, %zmm136394; AVX512BW-NEXT: vmovdqa64 %zmm5, %zmm146395; AVX512BW-NEXT: vpermt2w %zmm4, %zmm12, %zmm146396; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm13 = zmm14[0,1,2,3],zmm13[4,5,6,7]6397; AVX512BW-NEXT: vmovdqa64 %zmm2, %zmm146398; AVX512BW-NEXT: vpermt2w %zmm3, %zmm12, %zmm146399; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm126400; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm12 = zmm12[0,1,2,3],zmm14[4,5,6,7]6401; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm14 = [3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63,3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63]6402; AVX512BW-NEXT: # zmm14 = mem[0,1,2,3,0,1,2,3]6403; AVX512BW-NEXT: vpermt2w %zmm6, %zmm14, %zmm76404; AVX512BW-NEXT: vpermt2w %zmm4, %zmm14, %zmm56405; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm4 = zmm5[0,1,2,3],zmm7[4,5,6,7]6406; AVX512BW-NEXT: vpermt2w %zmm3, %zmm14, %zmm26407; AVX512BW-NEXT: vpermt2w %zmm1, %zmm14, %zmm06408; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm2[4,5,6,7]6409; AVX512BW-NEXT: vmovdqa64 %zmm9, 64(%rsi)6410; AVX512BW-NEXT: vmovdqa64 %zmm8, (%rsi)6411; AVX512BW-NEXT: vmovdqa64 %zmm11, 64(%rdx)6412; AVX512BW-NEXT: vmovdqa64 %zmm10, (%rdx)6413; AVX512BW-NEXT: vmovdqa64 %zmm13, 64(%rcx)6414; AVX512BW-NEXT: vmovdqa64 %zmm12, (%rcx)6415; AVX512BW-NEXT: vmovdqa64 %zmm4, 64(%r8)6416; AVX512BW-NEXT: vmovdqa64 %zmm0, (%r8)6417; AVX512BW-NEXT: vzeroupper6418; AVX512BW-NEXT: retq6419;6420; AVX512BW-FCP-LABEL: load_i16_stride4_vf64:6421; AVX512BW-FCP: # %bb.0:6422; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm06423; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm16424; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %zmm26425; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %zmm36426; AVX512BW-FCP-NEXT: vmovdqa64 320(%rdi), %zmm46427; AVX512BW-FCP-NEXT: vmovdqa64 256(%rdi), %zmm56428; AVX512BW-FCP-NEXT: vmovdqa64 448(%rdi), %zmm66429; AVX512BW-FCP-NEXT: vmovdqa64 384(%rdi), %zmm76430; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm8 = [0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60]6431; AVX512BW-FCP-NEXT: # zmm8 = mem[0,1,2,3,0,1,2,3]6432; AVX512BW-FCP-NEXT: vmovdqa64 %zmm7, %zmm96433; AVX512BW-FCP-NEXT: vpermt2w %zmm6, %zmm8, %zmm96434; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm106435; AVX512BW-FCP-NEXT: vpermt2w %zmm4, %zmm8, %zmm106436; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm9 = zmm10[0,1,2,3],zmm9[4,5,6,7]6437; AVX512BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm106438; AVX512BW-FCP-NEXT: vpermt2w %zmm3, %zmm8, %zmm106439; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm86440; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm8 = zmm8[0,1,2,3],zmm10[4,5,6,7]6441; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm10 = [1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61,1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61]6442; AVX512BW-FCP-NEXT: # zmm10 = mem[0,1,2,3,0,1,2,3]6443; AVX512BW-FCP-NEXT: vmovdqa64 %zmm7, %zmm116444; AVX512BW-FCP-NEXT: vpermt2w %zmm6, %zmm10, %zmm116445; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm126446; AVX512BW-FCP-NEXT: vpermt2w %zmm4, %zmm10, %zmm126447; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm11 = zmm12[0,1,2,3],zmm11[4,5,6,7]6448; AVX512BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm126449; AVX512BW-FCP-NEXT: vpermt2w %zmm3, %zmm10, %zmm126450; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm106451; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm10 = zmm10[0,1,2,3],zmm12[4,5,6,7]6452; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm12 = [2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62,2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62]6453; AVX512BW-FCP-NEXT: # zmm12 = mem[0,1,2,3,0,1,2,3]6454; AVX512BW-FCP-NEXT: vmovdqa64 %zmm7, %zmm136455; AVX512BW-FCP-NEXT: vpermt2w %zmm6, %zmm12, %zmm136456; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm146457; AVX512BW-FCP-NEXT: vpermt2w %zmm4, %zmm12, %zmm146458; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm13 = zmm14[0,1,2,3],zmm13[4,5,6,7]6459; AVX512BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm146460; AVX512BW-FCP-NEXT: vpermt2w %zmm3, %zmm12, %zmm146461; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm126462; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm12 = zmm12[0,1,2,3],zmm14[4,5,6,7]6463; AVX512BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm14 = [3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63,3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63]6464; AVX512BW-FCP-NEXT: # zmm14 = mem[0,1,2,3,0,1,2,3]6465; AVX512BW-FCP-NEXT: vpermt2w %zmm6, %zmm14, %zmm76466; AVX512BW-FCP-NEXT: vpermt2w %zmm4, %zmm14, %zmm56467; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm4 = zmm5[0,1,2,3],zmm7[4,5,6,7]6468; AVX512BW-FCP-NEXT: vpermt2w %zmm3, %zmm14, %zmm26469; AVX512BW-FCP-NEXT: vpermt2w %zmm1, %zmm14, %zmm06470; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm2[4,5,6,7]6471; AVX512BW-FCP-NEXT: vmovdqa64 %zmm9, 64(%rsi)6472; AVX512BW-FCP-NEXT: vmovdqa64 %zmm8, (%rsi)6473; AVX512BW-FCP-NEXT: vmovdqa64 %zmm11, 64(%rdx)6474; AVX512BW-FCP-NEXT: vmovdqa64 %zmm10, (%rdx)6475; AVX512BW-FCP-NEXT: vmovdqa64 %zmm13, 64(%rcx)6476; AVX512BW-FCP-NEXT: vmovdqa64 %zmm12, (%rcx)6477; AVX512BW-FCP-NEXT: vmovdqa64 %zmm4, 64(%r8)6478; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, (%r8)6479; AVX512BW-FCP-NEXT: vzeroupper6480; AVX512BW-FCP-NEXT: retq6481;6482; AVX512DQ-BW-LABEL: load_i16_stride4_vf64:6483; AVX512DQ-BW: # %bb.0:6484; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm06485; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %zmm16486; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %zmm26487; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %zmm36488; AVX512DQ-BW-NEXT: vmovdqa64 320(%rdi), %zmm46489; AVX512DQ-BW-NEXT: vmovdqa64 256(%rdi), %zmm56490; AVX512DQ-BW-NEXT: vmovdqa64 448(%rdi), %zmm66491; AVX512DQ-BW-NEXT: vmovdqa64 384(%rdi), %zmm76492; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm8 = [0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60]6493; AVX512DQ-BW-NEXT: # zmm8 = mem[0,1,2,3,0,1,2,3]6494; AVX512DQ-BW-NEXT: vmovdqa64 %zmm7, %zmm96495; AVX512DQ-BW-NEXT: vpermt2w %zmm6, %zmm8, %zmm96496; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, %zmm106497; AVX512DQ-BW-NEXT: vpermt2w %zmm4, %zmm8, %zmm106498; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm9 = zmm10[0,1,2,3],zmm9[4,5,6,7]6499; AVX512DQ-BW-NEXT: vmovdqa64 %zmm2, %zmm106500; AVX512DQ-BW-NEXT: vpermt2w %zmm3, %zmm8, %zmm106501; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm86502; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm8 = zmm8[0,1,2,3],zmm10[4,5,6,7]6503; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm10 = [1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61,1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61]6504; AVX512DQ-BW-NEXT: # zmm10 = mem[0,1,2,3,0,1,2,3]6505; AVX512DQ-BW-NEXT: vmovdqa64 %zmm7, %zmm116506; AVX512DQ-BW-NEXT: vpermt2w %zmm6, %zmm10, %zmm116507; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, %zmm126508; AVX512DQ-BW-NEXT: vpermt2w %zmm4, %zmm10, %zmm126509; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm11 = zmm12[0,1,2,3],zmm11[4,5,6,7]6510; AVX512DQ-BW-NEXT: vmovdqa64 %zmm2, %zmm126511; AVX512DQ-BW-NEXT: vpermt2w %zmm3, %zmm10, %zmm126512; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm106513; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm10 = zmm10[0,1,2,3],zmm12[4,5,6,7]6514; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm12 = [2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62,2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62]6515; AVX512DQ-BW-NEXT: # zmm12 = mem[0,1,2,3,0,1,2,3]6516; AVX512DQ-BW-NEXT: vmovdqa64 %zmm7, %zmm136517; AVX512DQ-BW-NEXT: vpermt2w %zmm6, %zmm12, %zmm136518; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, %zmm146519; AVX512DQ-BW-NEXT: vpermt2w %zmm4, %zmm12, %zmm146520; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm13 = zmm14[0,1,2,3],zmm13[4,5,6,7]6521; AVX512DQ-BW-NEXT: vmovdqa64 %zmm2, %zmm146522; AVX512DQ-BW-NEXT: vpermt2w %zmm3, %zmm12, %zmm146523; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm126524; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm12 = zmm12[0,1,2,3],zmm14[4,5,6,7]6525; AVX512DQ-BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm14 = [3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63,3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63]6526; AVX512DQ-BW-NEXT: # zmm14 = mem[0,1,2,3,0,1,2,3]6527; AVX512DQ-BW-NEXT: vpermt2w %zmm6, %zmm14, %zmm76528; AVX512DQ-BW-NEXT: vpermt2w %zmm4, %zmm14, %zmm56529; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm4 = zmm5[0,1,2,3],zmm7[4,5,6,7]6530; AVX512DQ-BW-NEXT: vpermt2w %zmm3, %zmm14, %zmm26531; AVX512DQ-BW-NEXT: vpermt2w %zmm1, %zmm14, %zmm06532; AVX512DQ-BW-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm2[4,5,6,7]6533; AVX512DQ-BW-NEXT: vmovdqa64 %zmm9, 64(%rsi)6534; AVX512DQ-BW-NEXT: vmovdqa64 %zmm8, (%rsi)6535; AVX512DQ-BW-NEXT: vmovdqa64 %zmm11, 64(%rdx)6536; AVX512DQ-BW-NEXT: vmovdqa64 %zmm10, (%rdx)6537; AVX512DQ-BW-NEXT: vmovdqa64 %zmm13, 64(%rcx)6538; AVX512DQ-BW-NEXT: vmovdqa64 %zmm12, (%rcx)6539; AVX512DQ-BW-NEXT: vmovdqa64 %zmm4, 64(%r8)6540; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, (%r8)6541; AVX512DQ-BW-NEXT: vzeroupper6542; AVX512DQ-BW-NEXT: retq6543;6544; AVX512DQ-BW-FCP-LABEL: load_i16_stride4_vf64:6545; AVX512DQ-BW-FCP: # %bb.0:6546; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm06547; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm16548; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %zmm26549; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %zmm36550; AVX512DQ-BW-FCP-NEXT: vmovdqa64 320(%rdi), %zmm46551; AVX512DQ-BW-FCP-NEXT: vmovdqa64 256(%rdi), %zmm56552; AVX512DQ-BW-FCP-NEXT: vmovdqa64 448(%rdi), %zmm66553; AVX512DQ-BW-FCP-NEXT: vmovdqa64 384(%rdi), %zmm76554; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm8 = [0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60]6555; AVX512DQ-BW-FCP-NEXT: # zmm8 = mem[0,1,2,3,0,1,2,3]6556; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm7, %zmm96557; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm6, %zmm8, %zmm96558; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm106559; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm4, %zmm8, %zmm106560; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm9 = zmm10[0,1,2,3],zmm9[4,5,6,7]6561; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm106562; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm3, %zmm8, %zmm106563; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm86564; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm8 = zmm8[0,1,2,3],zmm10[4,5,6,7]6565; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm10 = [1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61,1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61]6566; AVX512DQ-BW-FCP-NEXT: # zmm10 = mem[0,1,2,3,0,1,2,3]6567; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm7, %zmm116568; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm6, %zmm10, %zmm116569; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm126570; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm4, %zmm10, %zmm126571; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm11 = zmm12[0,1,2,3],zmm11[4,5,6,7]6572; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm126573; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm3, %zmm10, %zmm126574; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm106575; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm10 = zmm10[0,1,2,3],zmm12[4,5,6,7]6576; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm12 = [2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62,2,6,10,14,18,22,26,30,34,38,42,46,50,54,58,62]6577; AVX512DQ-BW-FCP-NEXT: # zmm12 = mem[0,1,2,3,0,1,2,3]6578; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm7, %zmm136579; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm6, %zmm12, %zmm136580; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm146581; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm4, %zmm12, %zmm146582; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm13 = zmm14[0,1,2,3],zmm13[4,5,6,7]6583; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm2, %zmm146584; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm3, %zmm12, %zmm146585; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm126586; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm12 = zmm12[0,1,2,3],zmm14[4,5,6,7]6587; AVX512DQ-BW-FCP-NEXT: vbroadcasti64x4 {{.*#+}} zmm14 = [3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63,3,7,11,15,19,23,27,31,35,39,43,47,51,55,59,63]6588; AVX512DQ-BW-FCP-NEXT: # zmm14 = mem[0,1,2,3,0,1,2,3]6589; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm6, %zmm14, %zmm76590; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm4, %zmm14, %zmm56591; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm4 = zmm5[0,1,2,3],zmm7[4,5,6,7]6592; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm3, %zmm14, %zmm26593; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm1, %zmm14, %zmm06594; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm2[4,5,6,7]6595; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm9, 64(%rsi)6596; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm8, (%rsi)6597; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm11, 64(%rdx)6598; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm10, (%rdx)6599; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm13, 64(%rcx)6600; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm12, (%rcx)6601; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm4, 64(%r8)6602; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, (%r8)6603; AVX512DQ-BW-FCP-NEXT: vzeroupper6604; AVX512DQ-BW-FCP-NEXT: retq6605 %wide.vec = load <256 x i16>, ptr %in.vec, align 646606 %strided.vec0 = shufflevector <256 x i16> %wide.vec, <256 x i16> poison, <64 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60, i32 64, i32 68, i32 72, i32 76, i32 80, i32 84, i32 88, i32 92, i32 96, i32 100, i32 104, i32 108, i32 112, i32 116, i32 120, i32 124, i32 128, i32 132, i32 136, i32 140, i32 144, i32 148, i32 152, i32 156, i32 160, i32 164, i32 168, i32 172, i32 176, i32 180, i32 184, i32 188, i32 192, i32 196, i32 200, i32 204, i32 208, i32 212, i32 216, i32 220, i32 224, i32 228, i32 232, i32 236, i32 240, i32 244, i32 248, i32 252>6607 %strided.vec1 = shufflevector <256 x i16> %wide.vec, <256 x i16> poison, <64 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61, i32 65, i32 69, i32 73, i32 77, i32 81, i32 85, i32 89, i32 93, i32 97, i32 101, i32 105, i32 109, i32 113, i32 117, i32 121, i32 125, i32 129, i32 133, i32 137, i32 141, i32 145, i32 149, i32 153, i32 157, i32 161, i32 165, i32 169, i32 173, i32 177, i32 181, i32 185, i32 189, i32 193, i32 197, i32 201, i32 205, i32 209, i32 213, i32 217, i32 221, i32 225, i32 229, i32 233, i32 237, i32 241, i32 245, i32 249, i32 253>6608 %strided.vec2 = shufflevector <256 x i16> %wide.vec, <256 x i16> poison, <64 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62, i32 66, i32 70, i32 74, i32 78, i32 82, i32 86, i32 90, i32 94, i32 98, i32 102, i32 106, i32 110, i32 114, i32 118, i32 122, i32 126, i32 130, i32 134, i32 138, i32 142, i32 146, i32 150, i32 154, i32 158, i32 162, i32 166, i32 170, i32 174, i32 178, i32 182, i32 186, i32 190, i32 194, i32 198, i32 202, i32 206, i32 210, i32 214, i32 218, i32 222, i32 226, i32 230, i32 234, i32 238, i32 242, i32 246, i32 250, i32 254>6609 %strided.vec3 = shufflevector <256 x i16> %wide.vec, <256 x i16> poison, <64 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63, i32 67, i32 71, i32 75, i32 79, i32 83, i32 87, i32 91, i32 95, i32 99, i32 103, i32 107, i32 111, i32 115, i32 119, i32 123, i32 127, i32 131, i32 135, i32 139, i32 143, i32 147, i32 151, i32 155, i32 159, i32 163, i32 167, i32 171, i32 175, i32 179, i32 183, i32 187, i32 191, i32 195, i32 199, i32 203, i32 207, i32 211, i32 215, i32 219, i32 223, i32 227, i32 231, i32 235, i32 239, i32 243, i32 247, i32 251, i32 255>6610 store <64 x i16> %strided.vec0, ptr %out.vec0, align 646611 store <64 x i16> %strided.vec1, ptr %out.vec1, align 646612 store <64 x i16> %strided.vec2, ptr %out.vec2, align 646613 store <64 x i16> %strided.vec3, ptr %out.vec3, align 646614 ret void6615}6616