4240 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE3; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX4; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX25; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FP6; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FCP7; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX5128; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512-FCP9; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ10; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ-FCP11; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW12; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW-FCP13; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512DQ-BW14; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ-BW-FCP15 16; These patterns are produced by LoopVectorizer for interleaved loads.17 18define void @load_i16_stride3_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2) nounwind {19; SSE-LABEL: load_i16_stride3_vf2:20; SSE: # %bb.0:21; SSE-NEXT: movdqa (%rdi), %xmm022; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm0[0,3,2,3,4,5,6,7]23; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]24; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,2,3,4,5,6,7]25; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]26; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]27; SSE-NEXT: movd %xmm1, (%rsi)28; SSE-NEXT: movd %xmm2, (%rdx)29; SSE-NEXT: movd %xmm0, (%rcx)30; SSE-NEXT: retq31;32; AVX-LABEL: load_i16_stride3_vf2:33; AVX: # %bb.0:34; AVX-NEXT: vmovdqa (%rdi), %xmm035; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,3,2,3,4,5,6,7]36; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]37; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,2,2,3,4,5,6,7]38; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]39; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]40; AVX-NEXT: vmovd %xmm1, (%rsi)41; AVX-NEXT: vmovd %xmm2, (%rdx)42; AVX-NEXT: vmovd %xmm0, (%rcx)43; AVX-NEXT: retq44;45; AVX2-LABEL: load_i16_stride3_vf2:46; AVX2: # %bb.0:47; AVX2-NEXT: vmovdqa (%rdi), %xmm048; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,3,2,3,4,5,6,7]49; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]50; AVX2-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,2,2,3,4,5,6,7]51; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]52; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]53; AVX2-NEXT: vmovd %xmm1, (%rsi)54; AVX2-NEXT: vmovd %xmm2, (%rdx)55; AVX2-NEXT: vmovd %xmm0, (%rcx)56; AVX2-NEXT: retq57;58; AVX2-FP-LABEL: load_i16_stride3_vf2:59; AVX2-FP: # %bb.0:60; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm061; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,3,2,3,4,5,6,7]62; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[2,3,8,9,u,u,u,u,u,u,u,u,u,u,u,u]63; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,u,u,u,u,u,u,u,u,u,u,u,u]64; AVX2-FP-NEXT: vmovd %xmm1, (%rsi)65; AVX2-FP-NEXT: vmovd %xmm2, (%rdx)66; AVX2-FP-NEXT: vmovd %xmm0, (%rcx)67; AVX2-FP-NEXT: retq68;69; AVX2-FCP-LABEL: load_i16_stride3_vf2:70; AVX2-FCP: # %bb.0:71; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm072; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,3,2,3,4,5,6,7]73; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[2,3,8,9,u,u,u,u,u,u,u,u,u,u,u,u]74; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,u,u,u,u,u,u,u,u,u,u,u,u]75; AVX2-FCP-NEXT: vmovd %xmm1, (%rsi)76; AVX2-FCP-NEXT: vmovd %xmm2, (%rdx)77; AVX2-FCP-NEXT: vmovd %xmm0, (%rcx)78; AVX2-FCP-NEXT: retq79;80; AVX512-LABEL: load_i16_stride3_vf2:81; AVX512: # %bb.0:82; AVX512-NEXT: vmovdqa (%rdi), %xmm083; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,3,2,3,4,5,6,7]84; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]85; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,2,2,3,4,5,6,7]86; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]87; AVX512-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]88; AVX512-NEXT: vmovd %xmm1, (%rsi)89; AVX512-NEXT: vmovd %xmm2, (%rdx)90; AVX512-NEXT: vmovd %xmm0, (%rcx)91; AVX512-NEXT: retq92;93; AVX512-FCP-LABEL: load_i16_stride3_vf2:94; AVX512-FCP: # %bb.0:95; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm096; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,3,2,3,4,5,6,7]97; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[2,3,8,9,u,u,u,u,u,u,u,u,u,u,u,u]98; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,u,u,u,u,u,u,u,u,u,u,u,u]99; AVX512-FCP-NEXT: vmovd %xmm1, (%rsi)100; AVX512-FCP-NEXT: vmovd %xmm2, (%rdx)101; AVX512-FCP-NEXT: vmovd %xmm0, (%rcx)102; AVX512-FCP-NEXT: retq103;104; AVX512DQ-LABEL: load_i16_stride3_vf2:105; AVX512DQ: # %bb.0:106; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0107; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,3,2,3,4,5,6,7]108; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]109; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,2,2,3,4,5,6,7]110; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]111; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]112; AVX512DQ-NEXT: vmovd %xmm1, (%rsi)113; AVX512DQ-NEXT: vmovd %xmm2, (%rdx)114; AVX512DQ-NEXT: vmovd %xmm0, (%rcx)115; AVX512DQ-NEXT: retq116;117; AVX512DQ-FCP-LABEL: load_i16_stride3_vf2:118; AVX512DQ-FCP: # %bb.0:119; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm0120; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,3,2,3,4,5,6,7]121; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[2,3,8,9,u,u,u,u,u,u,u,u,u,u,u,u]122; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,u,u,u,u,u,u,u,u,u,u,u,u]123; AVX512DQ-FCP-NEXT: vmovd %xmm1, (%rsi)124; AVX512DQ-FCP-NEXT: vmovd %xmm2, (%rdx)125; AVX512DQ-FCP-NEXT: vmovd %xmm0, (%rcx)126; AVX512DQ-FCP-NEXT: retq127;128; AVX512BW-LABEL: load_i16_stride3_vf2:129; AVX512BW: # %bb.0:130; AVX512BW-NEXT: vmovdqa (%rdi), %xmm0131; AVX512BW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,3,2,3,4,5,6,7]132; AVX512BW-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]133; AVX512BW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,2,2,3,4,5,6,7]134; AVX512BW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]135; AVX512BW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]136; AVX512BW-NEXT: vmovd %xmm1, (%rsi)137; AVX512BW-NEXT: vmovd %xmm2, (%rdx)138; AVX512BW-NEXT: vmovd %xmm0, (%rcx)139; AVX512BW-NEXT: retq140;141; AVX512BW-FCP-LABEL: load_i16_stride3_vf2:142; AVX512BW-FCP: # %bb.0:143; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %xmm0144; AVX512BW-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,3,2,3,4,5,6,7]145; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[2,3,8,9,u,u,u,u,u,u,u,u,u,u,u,u]146; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,u,u,u,u,u,u,u,u,u,u,u,u]147; AVX512BW-FCP-NEXT: vmovd %xmm1, (%rsi)148; AVX512BW-FCP-NEXT: vmovd %xmm2, (%rdx)149; AVX512BW-FCP-NEXT: vmovd %xmm0, (%rcx)150; AVX512BW-FCP-NEXT: retq151;152; AVX512DQ-BW-LABEL: load_i16_stride3_vf2:153; AVX512DQ-BW: # %bb.0:154; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %xmm0155; AVX512DQ-BW-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,3,2,3,4,5,6,7]156; AVX512DQ-BW-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]157; AVX512DQ-BW-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[1,2,2,3,4,5,6,7]158; AVX512DQ-BW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]159; AVX512DQ-BW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]160; AVX512DQ-BW-NEXT: vmovd %xmm1, (%rsi)161; AVX512DQ-BW-NEXT: vmovd %xmm2, (%rdx)162; AVX512DQ-BW-NEXT: vmovd %xmm0, (%rcx)163; AVX512DQ-BW-NEXT: retq164;165; AVX512DQ-BW-FCP-LABEL: load_i16_stride3_vf2:166; AVX512DQ-BW-FCP: # %bb.0:167; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %xmm0168; AVX512DQ-BW-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[0,3,2,3,4,5,6,7]169; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[2,3,8,9,u,u,u,u,u,u,u,u,u,u,u,u]170; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,u,u,u,u,u,u,u,u,u,u,u,u]171; AVX512DQ-BW-FCP-NEXT: vmovd %xmm1, (%rsi)172; AVX512DQ-BW-FCP-NEXT: vmovd %xmm2, (%rdx)173; AVX512DQ-BW-FCP-NEXT: vmovd %xmm0, (%rcx)174; AVX512DQ-BW-FCP-NEXT: retq175 %wide.vec = load <6 x i16>, ptr %in.vec, align 64176 %strided.vec0 = shufflevector <6 x i16> %wide.vec, <6 x i16> poison, <2 x i32> <i32 0, i32 3>177 %strided.vec1 = shufflevector <6 x i16> %wide.vec, <6 x i16> poison, <2 x i32> <i32 1, i32 4>178 %strided.vec2 = shufflevector <6 x i16> %wide.vec, <6 x i16> poison, <2 x i32> <i32 2, i32 5>179 store <2 x i16> %strided.vec0, ptr %out.vec0, align 64180 store <2 x i16> %strided.vec1, ptr %out.vec1, align 64181 store <2 x i16> %strided.vec2, ptr %out.vec2, align 64182 ret void183}184 185define void @load_i16_stride3_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2) nounwind {186; SSE-LABEL: load_i16_stride3_vf4:187; SSE: # %bb.0:188; SSE-NEXT: movdqa (%rdi), %xmm0189; SSE-NEXT: movdqa 16(%rdi), %xmm1190; SSE-NEXT: movdqa {{.*#+}} xmm2 = [65535,0,65535,65535,65535,65535,65535,65535]191; SSE-NEXT: movdqa %xmm0, %xmm3192; SSE-NEXT: pand %xmm2, %xmm3193; SSE-NEXT: pandn %xmm1, %xmm2194; SSE-NEXT: por %xmm3, %xmm2195; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3]196; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,6,7]197; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]198; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,3,2,1,4,5,6,7]199; SSE-NEXT: movq %xmm2, (%rsi)200; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm1[0,3,2,3,4,5,6,7]201; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,0],xmm0[0,0]202; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[2,3]203; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]204; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,7]205; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]206; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,0,4,5,6,7]207; SSE-NEXT: movq %xmm1, (%rdx)208; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]209; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]210; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]211; SSE-NEXT: movq %xmm0, (%rcx)212; SSE-NEXT: retq213;214; AVX-LABEL: load_i16_stride3_vf4:215; AVX: # %bb.0:216; AVX-NEXT: vmovdqa (%rdi), %xmm0217; AVX-NEXT: vmovdqa 16(%rdi), %xmm1218; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]219; AVX-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,1,6,7,12,13,2,3,u,u,u,u,u,u,u,u]220; AVX-NEXT: vmovq %xmm2, (%rsi)221; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]222; AVX-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[2,3,8,9,14,15,4,5,u,u,u,u,u,u,u,u]223; AVX-NEXT: vmovq %xmm2, (%rdx)224; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7]225; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]226; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]227; AVX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]228; AVX-NEXT: vmovq %xmm0, (%rcx)229; AVX-NEXT: retq230;231; AVX2-LABEL: load_i16_stride3_vf4:232; AVX2: # %bb.0:233; AVX2-NEXT: vmovdqa (%rdi), %xmm0234; AVX2-NEXT: vmovdqa 16(%rdi), %xmm1235; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]236; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,1,6,7,12,13,2,3,u,u,u,u,u,u,u,u]237; AVX2-NEXT: vmovq %xmm2, (%rsi)238; AVX2-NEXT: vpblendd {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2,3]239; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[2,3,8,9,14,15,4,5,u,u,u,u,u,u,u,u]240; AVX2-NEXT: vmovq %xmm2, (%rdx)241; AVX2-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7]242; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]243; AVX2-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]244; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]245; AVX2-NEXT: vmovq %xmm0, (%rcx)246; AVX2-NEXT: retq247;248; AVX2-FP-LABEL: load_i16_stride3_vf4:249; AVX2-FP: # %bb.0:250; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm0251; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm1252; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]253; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,1,6,7,12,13,2,3,u,u,u,u,u,u,u,u]254; AVX2-FP-NEXT: vmovq %xmm2, (%rsi)255; AVX2-FP-NEXT: vpblendd {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2,3]256; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[2,3,8,9,14,15,4,5,u,u,u,u,u,u,u,u]257; AVX2-FP-NEXT: vmovq %xmm2, (%rdx)258; AVX2-FP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7]259; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,u,u,u,u,u,u,u,u,u,u,u,u]260; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]261; AVX2-FP-NEXT: vmovq %xmm0, (%rcx)262; AVX2-FP-NEXT: retq263;264; AVX2-FCP-LABEL: load_i16_stride3_vf4:265; AVX2-FCP: # %bb.0:266; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm0267; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm1268; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]269; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,1,6,7,12,13,2,3,u,u,u,u,u,u,u,u]270; AVX2-FCP-NEXT: vmovq %xmm2, (%rsi)271; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2,3]272; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[2,3,8,9,14,15,4,5,u,u,u,u,u,u,u,u]273; AVX2-FCP-NEXT: vmovq %xmm2, (%rdx)274; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7]275; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,u,u,u,u,u,u,u,u,u,u,u,u]276; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]277; AVX2-FCP-NEXT: vmovq %xmm0, (%rcx)278; AVX2-FCP-NEXT: retq279;280; AVX512-LABEL: load_i16_stride3_vf4:281; AVX512: # %bb.0:282; AVX512-NEXT: vmovdqa (%rdi), %xmm0283; AVX512-NEXT: vmovdqa 16(%rdi), %xmm1284; AVX512-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]285; AVX512-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,1,6,7,12,13,2,3,u,u,u,u,u,u,u,u]286; AVX512-NEXT: vmovq %xmm2, (%rsi)287; AVX512-NEXT: vpblendd {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2,3]288; AVX512-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[2,3,8,9,14,15,4,5,u,u,u,u,u,u,u,u]289; AVX512-NEXT: vmovq %xmm2, (%rdx)290; AVX512-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7]291; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]292; AVX512-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]293; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]294; AVX512-NEXT: vmovq %xmm0, (%rcx)295; AVX512-NEXT: retq296;297; AVX512-FCP-LABEL: load_i16_stride3_vf4:298; AVX512-FCP: # %bb.0:299; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm0300; AVX512-FCP-NEXT: vmovdqa 16(%rdi), %xmm1301; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]302; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,1,6,7,12,13,2,3,u,u,u,u,u,u,u,u]303; AVX512-FCP-NEXT: vmovq %xmm2, (%rsi)304; AVX512-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2,3]305; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[2,3,8,9,14,15,4,5,u,u,u,u,u,u,u,u]306; AVX512-FCP-NEXT: vmovq %xmm2, (%rdx)307; AVX512-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7]308; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,u,u,u,u,u,u,u,u,u,u,u,u]309; AVX512-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]310; AVX512-FCP-NEXT: vmovq %xmm0, (%rcx)311; AVX512-FCP-NEXT: retq312;313; AVX512DQ-LABEL: load_i16_stride3_vf4:314; AVX512DQ: # %bb.0:315; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0316; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm1317; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]318; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,1,6,7,12,13,2,3,u,u,u,u,u,u,u,u]319; AVX512DQ-NEXT: vmovq %xmm2, (%rsi)320; AVX512DQ-NEXT: vpblendd {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2,3]321; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[2,3,8,9,14,15,4,5,u,u,u,u,u,u,u,u]322; AVX512DQ-NEXT: vmovq %xmm2, (%rdx)323; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7]324; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]325; AVX512DQ-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]326; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]327; AVX512DQ-NEXT: vmovq %xmm0, (%rcx)328; AVX512DQ-NEXT: retq329;330; AVX512DQ-FCP-LABEL: load_i16_stride3_vf4:331; AVX512DQ-FCP: # %bb.0:332; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm0333; AVX512DQ-FCP-NEXT: vmovdqa 16(%rdi), %xmm1334; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]335; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,1,6,7,12,13,2,3,u,u,u,u,u,u,u,u]336; AVX512DQ-FCP-NEXT: vmovq %xmm2, (%rsi)337; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm0[0],xmm1[1],xmm0[2,3]338; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[2,3,8,9,14,15,4,5,u,u,u,u,u,u,u,u]339; AVX512DQ-FCP-NEXT: vmovq %xmm2, (%rdx)340; AVX512DQ-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7]341; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,u,u,u,u,u,u,u,u,u,u,u,u]342; AVX512DQ-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]343; AVX512DQ-FCP-NEXT: vmovq %xmm0, (%rcx)344; AVX512DQ-FCP-NEXT: retq345;346; AVX512BW-LABEL: load_i16_stride3_vf4:347; AVX512BW: # %bb.0:348; AVX512BW-NEXT: vpmovsxbw {{.*#+}} xmm0 = [0,3,6,9,6,3,6,7]349; AVX512BW-NEXT: vmovdqa (%rdi), %ymm1350; AVX512BW-NEXT: vpermw %ymm1, %ymm0, %ymm0351; AVX512BW-NEXT: vpshuflw {{.*#+}} xmm2 = mem[0,3,2,3,4,5,6,7]352; AVX512BW-NEXT: vmovdqa (%rdi), %xmm3353; AVX512BW-NEXT: vmovq %xmm0, (%rsi)354; AVX512BW-NEXT: vpmovsxbw {{.*#+}} xmm0 = [1,4,7,10,4,7,6,7]355; AVX512BW-NEXT: vpermw %ymm1, %ymm0, %ymm0356; AVX512BW-NEXT: vmovq %xmm0, (%rdx)357; AVX512BW-NEXT: vpshufd {{.*#+}} xmm0 = xmm3[2,1,2,3]358; AVX512BW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]359; AVX512BW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]360; AVX512BW-NEXT: vmovq %xmm0, (%rcx)361; AVX512BW-NEXT: vzeroupper362; AVX512BW-NEXT: retq363;364; AVX512BW-FCP-LABEL: load_i16_stride3_vf4:365; AVX512BW-FCP: # %bb.0:366; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm0 = [0,3,6,9,6,3,6,7]367; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %ymm1368; AVX512BW-FCP-NEXT: vpermw %ymm1, %ymm0, %ymm0369; AVX512BW-FCP-NEXT: vmovq %xmm0, (%rsi)370; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm0 = [1,4,7,10,4,7,6,7]371; AVX512BW-FCP-NEXT: vpermw %ymm1, %ymm0, %ymm0372; AVX512BW-FCP-NEXT: vmovq %xmm0, (%rdx)373; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm0 = [2,5,8,11,2,3,10,11]374; AVX512BW-FCP-NEXT: vpermw %ymm1, %ymm0, %ymm0375; AVX512BW-FCP-NEXT: vmovq %xmm0, (%rcx)376; AVX512BW-FCP-NEXT: vzeroupper377; AVX512BW-FCP-NEXT: retq378;379; AVX512DQ-BW-LABEL: load_i16_stride3_vf4:380; AVX512DQ-BW: # %bb.0:381; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} xmm0 = [0,3,6,9,6,3,6,7]382; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %ymm1383; AVX512DQ-BW-NEXT: vpermw %ymm1, %ymm0, %ymm0384; AVX512DQ-BW-NEXT: vpshuflw {{.*#+}} xmm2 = mem[0,3,2,3,4,5,6,7]385; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %xmm3386; AVX512DQ-BW-NEXT: vmovq %xmm0, (%rsi)387; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} xmm0 = [1,4,7,10,4,7,6,7]388; AVX512DQ-BW-NEXT: vpermw %ymm1, %ymm0, %ymm0389; AVX512DQ-BW-NEXT: vmovq %xmm0, (%rdx)390; AVX512DQ-BW-NEXT: vpshufd {{.*#+}} xmm0 = xmm3[2,1,2,3]391; AVX512DQ-BW-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]392; AVX512DQ-BW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]393; AVX512DQ-BW-NEXT: vmovq %xmm0, (%rcx)394; AVX512DQ-BW-NEXT: vzeroupper395; AVX512DQ-BW-NEXT: retq396;397; AVX512DQ-BW-FCP-LABEL: load_i16_stride3_vf4:398; AVX512DQ-BW-FCP: # %bb.0:399; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm0 = [0,3,6,9,6,3,6,7]400; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %ymm1401; AVX512DQ-BW-FCP-NEXT: vpermw %ymm1, %ymm0, %ymm0402; AVX512DQ-BW-FCP-NEXT: vmovq %xmm0, (%rsi)403; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm0 = [1,4,7,10,4,7,6,7]404; AVX512DQ-BW-FCP-NEXT: vpermw %ymm1, %ymm0, %ymm0405; AVX512DQ-BW-FCP-NEXT: vmovq %xmm0, (%rdx)406; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm0 = [2,5,8,11,2,3,10,11]407; AVX512DQ-BW-FCP-NEXT: vpermw %ymm1, %ymm0, %ymm0408; AVX512DQ-BW-FCP-NEXT: vmovq %xmm0, (%rcx)409; AVX512DQ-BW-FCP-NEXT: vzeroupper410; AVX512DQ-BW-FCP-NEXT: retq411 %wide.vec = load <12 x i16>, ptr %in.vec, align 64412 %strided.vec0 = shufflevector <12 x i16> %wide.vec, <12 x i16> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>413 %strided.vec1 = shufflevector <12 x i16> %wide.vec, <12 x i16> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>414 %strided.vec2 = shufflevector <12 x i16> %wide.vec, <12 x i16> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>415 store <4 x i16> %strided.vec0, ptr %out.vec0, align 64416 store <4 x i16> %strided.vec1, ptr %out.vec1, align 64417 store <4 x i16> %strided.vec2, ptr %out.vec2, align 64418 ret void419}420 421define void @load_i16_stride3_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2) nounwind {422; SSE-LABEL: load_i16_stride3_vf8:423; SSE: # %bb.0:424; SSE-NEXT: movdqa (%rdi), %xmm3425; SSE-NEXT: movdqa 16(%rdi), %xmm2426; SSE-NEXT: movdqa 32(%rdi), %xmm0427; SSE-NEXT: movdqa {{.*#+}} xmm1 = [65535,0,65535,65535,0,65535,65535,0]428; SSE-NEXT: movdqa %xmm3, %xmm4429; SSE-NEXT: pand %xmm1, %xmm4430; SSE-NEXT: pandn %xmm2, %xmm1431; SSE-NEXT: por %xmm4, %xmm1432; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]433; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7]434; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]435; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,1,4,5,6,7]436; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm2[0,1,2,3,4,7,6,7]437; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,1,2,1]438; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,6,5]439; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[3,0],xmm4[2,0]440; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm5[2,0]441; SSE-NEXT: movdqa {{.*#+}} xmm4 = [65535,65535,0,65535,65535,0,65535,65535]442; SSE-NEXT: movdqa %xmm4, %xmm5443; SSE-NEXT: pandn %xmm2, %xmm5444; SSE-NEXT: movdqa %xmm3, %xmm6445; SSE-NEXT: pand %xmm4, %xmm6446; SSE-NEXT: por %xmm5, %xmm6447; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm6[2,1,2,3,4,5,6,7]448; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,4,7]449; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,3,2,3]450; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,2,3,0,4,5,6,7]451; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,5,5]452; SSE-NEXT: movdqa {{.*#+}} xmm6 = [65535,65535,65535,65535,65535,0,0,0]453; SSE-NEXT: pand %xmm6, %xmm5454; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm0[0,3,2,3,4,5,6,7]455; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,1,0,3]456; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,4,5,6]457; SSE-NEXT: movdqa %xmm6, %xmm8458; SSE-NEXT: pandn %xmm7, %xmm8459; SSE-NEXT: por %xmm5, %xmm8460; SSE-NEXT: pand %xmm4, %xmm2461; SSE-NEXT: pandn %xmm3, %xmm4462; SSE-NEXT: por %xmm2, %xmm4463; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm4[3,1,2,0]464; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,6,7]465; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,1,0,3]466; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,0,3,4,5,6,7]467; SSE-NEXT: pand %xmm6, %xmm2468; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7]469; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,2]470; SSE-NEXT: pandn %xmm0, %xmm6471; SSE-NEXT: por %xmm2, %xmm6472; SSE-NEXT: movaps %xmm1, (%rsi)473; SSE-NEXT: movdqa %xmm8, (%rdx)474; SSE-NEXT: movdqa %xmm6, (%rcx)475; SSE-NEXT: retq476;477; AVX-LABEL: load_i16_stride3_vf8:478; AVX: # %bb.0:479; AVX-NEXT: vmovdqa (%rdi), %xmm0480; AVX-NEXT: vmovdqa 16(%rdi), %xmm1481; AVX-NEXT: vmovdqa 32(%rdi), %xmm2482; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,1,2,1]483; AVX-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,6,5]484; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]485; AVX-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]486; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3,4,5],xmm3[6,7]487; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]488; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm4[1,2],xmm2[3],xmm4[4,5],xmm2[6],xmm4[7]489; AVX-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13]490; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]491; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3],xmm2[4],xmm0[5,6],xmm2[7]492; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15]493; AVX-NEXT: vmovdqa %xmm3, (%rsi)494; AVX-NEXT: vmovdqa %xmm4, (%rdx)495; AVX-NEXT: vmovdqa %xmm0, (%rcx)496; AVX-NEXT: retq497;498; AVX2-LABEL: load_i16_stride3_vf8:499; AVX2: # %bb.0:500; AVX2-NEXT: vmovdqa (%rdi), %xmm0501; AVX2-NEXT: vmovdqa 16(%rdi), %xmm1502; AVX2-NEXT: vmovdqa 32(%rdi), %xmm2503; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]504; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm1[1],xmm3[2,3],xmm1[4],xmm3[5,6],xmm1[7]505; AVX2-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11]506; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1,2],xmm2[3],xmm0[4,5],xmm2[6],xmm0[7]507; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm1[2],xmm4[3,4],xmm1[5],xmm4[6,7]508; AVX2-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13]509; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3],xmm2[4],xmm0[5,6],xmm2[7]510; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]511; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15]512; AVX2-NEXT: vmovdqa %xmm3, (%rsi)513; AVX2-NEXT: vmovdqa %xmm4, (%rdx)514; AVX2-NEXT: vmovdqa %xmm0, (%rcx)515; AVX2-NEXT: retq516;517; AVX2-FP-LABEL: load_i16_stride3_vf8:518; AVX2-FP: # %bb.0:519; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm0520; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm1521; AVX2-FP-NEXT: vmovdqa 32(%rdi), %xmm2522; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]523; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm1[1],xmm3[2,3],xmm1[4],xmm3[5,6],xmm1[7]524; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11]525; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1,2],xmm2[3],xmm0[4,5],xmm2[6],xmm0[7]526; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm1[2],xmm4[3,4],xmm1[5],xmm4[6,7]527; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13]528; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3],xmm2[4],xmm0[5,6],xmm2[7]529; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]530; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15]531; AVX2-FP-NEXT: vmovdqa %xmm3, (%rsi)532; AVX2-FP-NEXT: vmovdqa %xmm4, (%rdx)533; AVX2-FP-NEXT: vmovdqa %xmm0, (%rcx)534; AVX2-FP-NEXT: retq535;536; AVX2-FCP-LABEL: load_i16_stride3_vf8:537; AVX2-FCP: # %bb.0:538; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm0539; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm1540; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %xmm2541; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]542; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm1[1],xmm3[2,3],xmm1[4],xmm3[5,6],xmm1[7]543; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11]544; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1,2],xmm2[3],xmm0[4,5],xmm2[6],xmm0[7]545; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm1[2],xmm4[3,4],xmm1[5],xmm4[6,7]546; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13]547; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3],xmm2[4],xmm0[5,6],xmm2[7]548; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]549; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15]550; AVX2-FCP-NEXT: vmovdqa %xmm3, (%rsi)551; AVX2-FCP-NEXT: vmovdqa %xmm4, (%rdx)552; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rcx)553; AVX2-FCP-NEXT: retq554;555; AVX512-LABEL: load_i16_stride3_vf8:556; AVX512: # %bb.0:557; AVX512-NEXT: vmovdqa (%rdi), %xmm0558; AVX512-NEXT: vmovdqa 16(%rdi), %xmm1559; AVX512-NEXT: vmovdqa 32(%rdi), %xmm2560; AVX512-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]561; AVX512-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm1[1],xmm3[2,3],xmm1[4],xmm3[5,6],xmm1[7]562; AVX512-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11]563; AVX512-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1,2],xmm2[3],xmm0[4,5],xmm2[6],xmm0[7]564; AVX512-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm1[2],xmm4[3,4],xmm1[5],xmm4[6,7]565; AVX512-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13]566; AVX512-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3],xmm2[4],xmm0[5,6],xmm2[7]567; AVX512-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]568; AVX512-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15]569; AVX512-NEXT: vmovdqa %xmm3, (%rsi)570; AVX512-NEXT: vmovdqa %xmm4, (%rdx)571; AVX512-NEXT: vmovdqa %xmm0, (%rcx)572; AVX512-NEXT: retq573;574; AVX512-FCP-LABEL: load_i16_stride3_vf8:575; AVX512-FCP: # %bb.0:576; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm0577; AVX512-FCP-NEXT: vmovdqa 16(%rdi), %xmm1578; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %xmm2579; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]580; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm1[1],xmm3[2,3],xmm1[4],xmm3[5,6],xmm1[7]581; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11]582; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1,2],xmm2[3],xmm0[4,5],xmm2[6],xmm0[7]583; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm1[2],xmm4[3,4],xmm1[5],xmm4[6,7]584; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13]585; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3],xmm2[4],xmm0[5,6],xmm2[7]586; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]587; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15]588; AVX512-FCP-NEXT: vmovdqa %xmm3, (%rsi)589; AVX512-FCP-NEXT: vmovdqa %xmm4, (%rdx)590; AVX512-FCP-NEXT: vmovdqa %xmm0, (%rcx)591; AVX512-FCP-NEXT: retq592;593; AVX512DQ-LABEL: load_i16_stride3_vf8:594; AVX512DQ: # %bb.0:595; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0596; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm1597; AVX512DQ-NEXT: vmovdqa 32(%rdi), %xmm2598; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]599; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm1[1],xmm3[2,3],xmm1[4],xmm3[5,6],xmm1[7]600; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11]601; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1,2],xmm2[3],xmm0[4,5],xmm2[6],xmm0[7]602; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm1[2],xmm4[3,4],xmm1[5],xmm4[6,7]603; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13]604; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3],xmm2[4],xmm0[5,6],xmm2[7]605; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]606; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15]607; AVX512DQ-NEXT: vmovdqa %xmm3, (%rsi)608; AVX512DQ-NEXT: vmovdqa %xmm4, (%rdx)609; AVX512DQ-NEXT: vmovdqa %xmm0, (%rcx)610; AVX512DQ-NEXT: retq611;612; AVX512DQ-FCP-LABEL: load_i16_stride3_vf8:613; AVX512DQ-FCP: # %bb.0:614; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm0615; AVX512DQ-FCP-NEXT: vmovdqa 16(%rdi), %xmm1616; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %xmm2617; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]618; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm1[1],xmm3[2,3],xmm1[4],xmm3[5,6],xmm1[7]619; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11]620; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1,2],xmm2[3],xmm0[4,5],xmm2[6],xmm0[7]621; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm1[2],xmm4[3,4],xmm1[5],xmm4[6,7]622; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13]623; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3],xmm2[4],xmm0[5,6],xmm2[7]624; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]625; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15]626; AVX512DQ-FCP-NEXT: vmovdqa %xmm3, (%rsi)627; AVX512DQ-FCP-NEXT: vmovdqa %xmm4, (%rdx)628; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, (%rcx)629; AVX512DQ-FCP-NEXT: retq630;631; AVX512BW-LABEL: load_i16_stride3_vf8:632; AVX512BW: # %bb.0:633; AVX512BW-NEXT: vpmovsxbw {{.*#+}} xmm0 = [0,3,6,9,12,15,18,21]634; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm1635; AVX512BW-NEXT: vpermw %zmm1, %zmm0, %zmm0636; AVX512BW-NEXT: vpmovsxbw {{.*#+}} xmm2 = [1,4,7,10,13,16,19,22]637; AVX512BW-NEXT: vpermw %zmm1, %zmm2, %zmm2638; AVX512BW-NEXT: vpmovsxbw {{.*#+}} xmm3 = [2,5,8,11,14,17,20,23]639; AVX512BW-NEXT: vpermw %zmm1, %zmm3, %zmm1640; AVX512BW-NEXT: vmovdqa %xmm0, (%rsi)641; AVX512BW-NEXT: vmovdqa %xmm2, (%rdx)642; AVX512BW-NEXT: vmovdqa %xmm1, (%rcx)643; AVX512BW-NEXT: vzeroupper644; AVX512BW-NEXT: retq645;646; AVX512BW-FCP-LABEL: load_i16_stride3_vf8:647; AVX512BW-FCP: # %bb.0:648; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm0 = [0,3,6,9,12,15,18,21]649; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm1650; AVX512BW-FCP-NEXT: vpermw %zmm1, %zmm0, %zmm0651; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm2 = [1,4,7,10,13,16,19,22]652; AVX512BW-FCP-NEXT: vpermw %zmm1, %zmm2, %zmm2653; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm3 = [2,5,8,11,14,17,20,23]654; AVX512BW-FCP-NEXT: vpermw %zmm1, %zmm3, %zmm1655; AVX512BW-FCP-NEXT: vmovdqa %xmm0, (%rsi)656; AVX512BW-FCP-NEXT: vmovdqa %xmm2, (%rdx)657; AVX512BW-FCP-NEXT: vmovdqa %xmm1, (%rcx)658; AVX512BW-FCP-NEXT: vzeroupper659; AVX512BW-FCP-NEXT: retq660;661; AVX512DQ-BW-LABEL: load_i16_stride3_vf8:662; AVX512DQ-BW: # %bb.0:663; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} xmm0 = [0,3,6,9,12,15,18,21]664; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm1665; AVX512DQ-BW-NEXT: vpermw %zmm1, %zmm0, %zmm0666; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} xmm2 = [1,4,7,10,13,16,19,22]667; AVX512DQ-BW-NEXT: vpermw %zmm1, %zmm2, %zmm2668; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} xmm3 = [2,5,8,11,14,17,20,23]669; AVX512DQ-BW-NEXT: vpermw %zmm1, %zmm3, %zmm1670; AVX512DQ-BW-NEXT: vmovdqa %xmm0, (%rsi)671; AVX512DQ-BW-NEXT: vmovdqa %xmm2, (%rdx)672; AVX512DQ-BW-NEXT: vmovdqa %xmm1, (%rcx)673; AVX512DQ-BW-NEXT: vzeroupper674; AVX512DQ-BW-NEXT: retq675;676; AVX512DQ-BW-FCP-LABEL: load_i16_stride3_vf8:677; AVX512DQ-BW-FCP: # %bb.0:678; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm0 = [0,3,6,9,12,15,18,21]679; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm1680; AVX512DQ-BW-FCP-NEXT: vpermw %zmm1, %zmm0, %zmm0681; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm2 = [1,4,7,10,13,16,19,22]682; AVX512DQ-BW-FCP-NEXT: vpermw %zmm1, %zmm2, %zmm2683; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm3 = [2,5,8,11,14,17,20,23]684; AVX512DQ-BW-FCP-NEXT: vpermw %zmm1, %zmm3, %zmm1685; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm0, (%rsi)686; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm2, (%rdx)687; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm1, (%rcx)688; AVX512DQ-BW-FCP-NEXT: vzeroupper689; AVX512DQ-BW-FCP-NEXT: retq690 %wide.vec = load <24 x i16>, ptr %in.vec, align 64691 %strided.vec0 = shufflevector <24 x i16> %wide.vec, <24 x i16> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>692 %strided.vec1 = shufflevector <24 x i16> %wide.vec, <24 x i16> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>693 %strided.vec2 = shufflevector <24 x i16> %wide.vec, <24 x i16> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>694 store <8 x i16> %strided.vec0, ptr %out.vec0, align 64695 store <8 x i16> %strided.vec1, ptr %out.vec1, align 64696 store <8 x i16> %strided.vec2, ptr %out.vec2, align 64697 ret void698}699 700define void @load_i16_stride3_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2) nounwind {701; SSE-LABEL: load_i16_stride3_vf16:702; SSE: # %bb.0:703; SSE-NEXT: movdqa 80(%rdi), %xmm0704; SSE-NEXT: movdqa 64(%rdi), %xmm1705; SSE-NEXT: movdqa (%rdi), %xmm7706; SSE-NEXT: movdqa 16(%rdi), %xmm4707; SSE-NEXT: movdqa 32(%rdi), %xmm3708; SSE-NEXT: movdqa 48(%rdi), %xmm2709; SSE-NEXT: movdqa {{.*#+}} xmm6 = [65535,0,65535,65535,0,65535,65535,0]710; SSE-NEXT: movdqa %xmm6, %xmm8711; SSE-NEXT: pandn %xmm4, %xmm8712; SSE-NEXT: movdqa {{.*#+}} xmm5 = [65535,65535,0,65535,65535,0,65535,65535]713; SSE-NEXT: movdqa %xmm7, %xmm9714; SSE-NEXT: movdqa %xmm5, %xmm10715; SSE-NEXT: pandn %xmm7, %xmm10716; SSE-NEXT: pand %xmm6, %xmm7717; SSE-NEXT: por %xmm8, %xmm7718; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,2,1,3]719; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,6,5,6,7]720; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,2,1,3]721; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,3,2,1,4,5,6,7]722; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm4[0,1,2,3,4,7,6,7]723; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm3[0,1,2,1]724; SSE-NEXT: pshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,5,6,5]725; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[3,0],xmm8[2,0]726; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[0,1],xmm11[2,0]727; SSE-NEXT: movdqa %xmm2, %xmm8728; SSE-NEXT: pand %xmm6, %xmm8729; SSE-NEXT: pandn %xmm1, %xmm6730; SSE-NEXT: por %xmm8, %xmm6731; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,2,1,3]732; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,6,5,6,7]733; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,2,1,3]734; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,3,2,1,4,5,6,7]735; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm1[0,1,2,3,4,7,6,7]736; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm0[0,1,2,1]737; SSE-NEXT: pshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,5,6,5]738; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[3,0],xmm8[2,0]739; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,1],xmm11[2,0]740; SSE-NEXT: movdqa %xmm5, %xmm8741; SSE-NEXT: pandn %xmm4, %xmm8742; SSE-NEXT: pand %xmm5, %xmm9743; SSE-NEXT: por %xmm8, %xmm9744; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm9[2,1,2,3,4,5,6,7]745; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,4,7]746; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,3,2,3]747; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[1,2,3,0,4,5,6,7]748; SSE-NEXT: pshufhw {{.*#+}} xmm11 = xmm8[0,1,2,3,5,5,5,5]749; SSE-NEXT: movdqa {{.*#+}} xmm8 = [65535,65535,65535,65535,65535,0,0,0]750; SSE-NEXT: pand %xmm8, %xmm11751; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm3[0,3,2,3,4,5,6,7]752; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,1,0,3]753; SSE-NEXT: pshufhw {{.*#+}} xmm12 = xmm9[0,1,2,3,4,4,5,6]754; SSE-NEXT: movdqa %xmm8, %xmm9755; SSE-NEXT: pandn %xmm12, %xmm9756; SSE-NEXT: por %xmm11, %xmm9757; SSE-NEXT: movdqa %xmm5, %xmm11758; SSE-NEXT: pandn %xmm1, %xmm11759; SSE-NEXT: movdqa %xmm2, %xmm12760; SSE-NEXT: pand %xmm5, %xmm12761; SSE-NEXT: por %xmm11, %xmm12762; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm12[2,1,2,3,4,5,6,7]763; SSE-NEXT: pshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,5,4,7]764; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm11[0,3,2,3]765; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm11[1,2,3,0,4,5,6,7]766; SSE-NEXT: pshufhw {{.*#+}} xmm12 = xmm11[0,1,2,3,5,5,5,5]767; SSE-NEXT: pand %xmm8, %xmm12768; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm0[0,3,2,3,4,5,6,7]769; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm11[0,1,0,3]770; SSE-NEXT: pshufhw {{.*#+}} xmm13 = xmm11[0,1,2,3,4,4,5,6]771; SSE-NEXT: movdqa %xmm8, %xmm11772; SSE-NEXT: pandn %xmm13, %xmm11773; SSE-NEXT: por %xmm12, %xmm11774; SSE-NEXT: pand %xmm5, %xmm4775; SSE-NEXT: por %xmm10, %xmm4776; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[3,1,2,0]777; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,6,5,6,7]778; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[2,1,0,3]779; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[2,1,0,3,4,5,6,7]780; SSE-NEXT: pand %xmm8, %xmm4781; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,7]782; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,2]783; SSE-NEXT: movdqa %xmm8, %xmm10784; SSE-NEXT: pandn %xmm3, %xmm10785; SSE-NEXT: por %xmm4, %xmm10786; SSE-NEXT: pand %xmm5, %xmm1787; SSE-NEXT: pandn %xmm2, %xmm5788; SSE-NEXT: por %xmm1, %xmm5789; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm5[3,1,2,0]790; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7]791; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,0,3]792; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,0,3,4,5,6,7]793; SSE-NEXT: pand %xmm8, %xmm1794; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7]795; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,2]796; SSE-NEXT: pandn %xmm0, %xmm8797; SSE-NEXT: por %xmm1, %xmm8798; SSE-NEXT: movaps %xmm6, 16(%rsi)799; SSE-NEXT: movaps %xmm7, (%rsi)800; SSE-NEXT: movdqa %xmm11, 16(%rdx)801; SSE-NEXT: movdqa %xmm9, (%rdx)802; SSE-NEXT: movdqa %xmm8, 16(%rcx)803; SSE-NEXT: movdqa %xmm10, (%rcx)804; SSE-NEXT: retq805;806; AVX-LABEL: load_i16_stride3_vf16:807; AVX: # %bb.0:808; AVX-NEXT: vmovdqa 80(%rdi), %xmm0809; AVX-NEXT: vmovdqa 64(%rdi), %xmm1810; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]811; AVX-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,2,3,8,9,14,15,4,5,10,11]812; AVX-NEXT: vmovdqa (%rdi), %xmm3813; AVX-NEXT: vmovdqa 16(%rdi), %xmm4814; AVX-NEXT: vmovdqa 32(%rdi), %xmm5815; AVX-NEXT: vmovdqa 48(%rdi), %xmm6816; AVX-NEXT: vpshuflw {{.*#+}} xmm7 = xmm6[0,3,2,3,4,5,6,7]817; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[0,3,2,3]818; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm7[0,1,2],xmm2[3,4,5,6,7]819; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[0,1,2,1]820; AVX-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5,6,5]821; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm3[0],xmm4[1],xmm3[2,3],xmm4[4],xmm3[5,6],xmm4[7]822; AVX-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]823; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm8[0,1,2,3,4,5],xmm7[6,7]824; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm7, %ymm2825; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]826; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0],xmm6[1],xmm7[2,3],xmm6[4],xmm7[5,6],xmm6[7]827; AVX-NEXT: vmovdqa {{.*#+}} xmm8 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13]828; AVX-NEXT: vpshufb %xmm8, %xmm7, %xmm7829; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm3[0,1],xmm4[2],xmm3[3,4],xmm4[5],xmm3[6,7]830; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm5[0],xmm9[1,2],xmm5[3],xmm9[4,5],xmm5[6],xmm9[7]831; AVX-NEXT: vpshufb %xmm8, %xmm9, %xmm8832; AVX-NEXT: vinsertf128 $1, %xmm7, %ymm8, %ymm7833; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3],xmm0[4],xmm1[5,6],xmm0[7]834; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,0,1,6,7,12,13,2,3,8,9,14,15]835; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm6[2,1,2,3]836; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]837; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3,4,5,6,7]838; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm4[0,1],xmm3[2],xmm4[3,4],xmm3[5],xmm4[6,7]839; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm5[1],xmm1[2,3],xmm5[4],xmm1[5,6],xmm5[7]840; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15]841; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0842; AVX-NEXT: vmovaps %ymm2, (%rsi)843; AVX-NEXT: vmovaps %ymm7, (%rdx)844; AVX-NEXT: vmovaps %ymm0, (%rcx)845; AVX-NEXT: vzeroupper846; AVX-NEXT: retq847;848; AVX2-LABEL: load_i16_stride3_vf16:849; AVX2: # %bb.0:850; AVX2-NEXT: vmovdqa (%rdi), %ymm0851; AVX2-NEXT: vmovdqa 32(%rdi), %ymm1852; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm2 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]853; AVX2-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm2854; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]855; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2,3],ymm3[4],ymm2[5,6],ymm3[7],ymm2[8],ymm3[9],ymm2[10,11],ymm3[12],ymm2[13,14],ymm3[15]856; AVX2-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]857; AVX2-NEXT: vmovdqa 80(%rdi), %xmm3858; AVX2-NEXT: vmovdqa 64(%rdi), %xmm4859; AVX2-NEXT: vpblendw {{.*#+}} xmm5 = xmm4[0,1],xmm3[2],xmm4[3,4],xmm3[5],xmm4[6,7]860; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u,u,u,2,3,8,9,14,15,4,5,10,11]861; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5862; AVX2-NEXT: vpblendw {{.*#+}} ymm5 = ymm2[0,1,2],ymm5[3,4,5,6,7],ymm2[8,9,10],ymm5[11,12,13,14,15]863; AVX2-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7]864; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm5[4,5,6,7]865; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm5 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]866; AVX2-NEXT: vpblendvb %ymm5, %ymm1, %ymm0, %ymm5867; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm5[2,3,0,1]868; AVX2-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0,1],ymm6[2],ymm5[3,4],ymm6[5],ymm5[6,7,8,9],ymm6[10],ymm5[11,12],ymm6[13],ymm5[14,15]869; AVX2-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]870; AVX2-NEXT: vpblendw {{.*#+}} xmm6 = xmm3[0,1],xmm4[2],xmm3[3,4],xmm4[5],xmm3[6,7]871; AVX2-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,u,u,4,5,10,11,0,1,6,7,12,13]872; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6873; AVX2-NEXT: vpblendw {{.*#+}} ymm6 = ymm5[0,1,2],ymm6[3,4,5,6,7],ymm5[8,9,10],ymm6[11,12,13,14,15]874; AVX2-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,6,7,4]875; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]876; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm6 = [0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0]877; AVX2-NEXT: vpblendvb %ymm6, %ymm1, %ymm0, %ymm0878; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1879; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7],ymm1[8],ymm0[9,10],ymm1[11],ymm0[12,13],ymm1[14],ymm0[15]880; AVX2-NEXT: vmovdqa {{.*#+}} ymm1 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,u,u,u,u,u,u,u,u,u,u,u,u]881; AVX2-NEXT: vpshufb %ymm1, %ymm0, %ymm0882; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]883; AVX2-NEXT: vpshufb %xmm1, %xmm3, %xmm1884; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1885; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]886; AVX2-NEXT: vmovdqa %ymm2, (%rsi)887; AVX2-NEXT: vmovdqa %ymm5, (%rdx)888; AVX2-NEXT: vmovdqa %ymm0, (%rcx)889; AVX2-NEXT: vzeroupper890; AVX2-NEXT: retq891;892; AVX2-FP-LABEL: load_i16_stride3_vf16:893; AVX2-FP: # %bb.0:894; AVX2-FP-NEXT: vmovdqa (%rdi), %ymm0895; AVX2-FP-NEXT: vmovdqa 32(%rdi), %ymm1896; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]897; AVX2-FP-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm2898; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]899; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2,3],ymm3[4],ymm2[5,6],ymm3[7],ymm2[8],ymm3[9],ymm2[10,11],ymm3[12],ymm2[13,14],ymm3[15]900; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]901; AVX2-FP-NEXT: vmovdqa 80(%rdi), %xmm3902; AVX2-FP-NEXT: vmovdqa 64(%rdi), %xmm4903; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm5 = xmm4[0,1],xmm3[2],xmm4[3,4],xmm3[5],xmm4[6,7]904; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u,u,u,2,3,8,9,14,15,4,5,10,11]905; AVX2-FP-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5906; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm5 = ymm2[0,1,2],ymm5[3,4,5,6,7],ymm2[8,9,10],ymm5[11,12,13,14,15]907; AVX2-FP-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7]908; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm5[4,5,6,7]909; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm5 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]910; AVX2-FP-NEXT: vpblendvb %ymm5, %ymm1, %ymm0, %ymm5911; AVX2-FP-NEXT: vpermq {{.*#+}} ymm6 = ymm5[2,3,0,1]912; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0,1],ymm6[2],ymm5[3,4],ymm6[5],ymm5[6,7,8,9],ymm6[10],ymm5[11,12],ymm6[13],ymm5[14,15]913; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]914; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm6 = xmm3[0,1],xmm4[2],xmm3[3,4],xmm4[5],xmm3[6,7]915; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,u,u,4,5,10,11,0,1,6,7,12,13]916; AVX2-FP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6917; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm6 = ymm5[0,1,2],ymm6[3,4,5,6,7],ymm5[8,9,10],ymm6[11,12,13,14,15]918; AVX2-FP-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,6,7,4]919; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]920; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm6 = [0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0]921; AVX2-FP-NEXT: vpblendvb %ymm6, %ymm1, %ymm0, %ymm0922; AVX2-FP-NEXT: vextracti128 $1, %ymm0, %xmm1923; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7],ymm1[8],ymm0[9,10],ymm1[11],ymm0[12,13],ymm1[14],ymm0[15]924; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm1 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,u,u,u,u,u,u,u,u,u,u,u,u]925; AVX2-FP-NEXT: vpshufb %ymm1, %ymm0, %ymm0926; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]927; AVX2-FP-NEXT: vpshufb %xmm1, %xmm3, %xmm1928; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1929; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]930; AVX2-FP-NEXT: vmovdqa %ymm2, (%rsi)931; AVX2-FP-NEXT: vmovdqa %ymm5, (%rdx)932; AVX2-FP-NEXT: vmovdqa %ymm0, (%rcx)933; AVX2-FP-NEXT: vzeroupper934; AVX2-FP-NEXT: retq935;936; AVX2-FCP-LABEL: load_i16_stride3_vf16:937; AVX2-FCP: # %bb.0:938; AVX2-FCP-NEXT: vmovdqa (%rdi), %ymm0939; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %ymm1940; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]941; AVX2-FCP-NEXT: vpblendvb %ymm2, %ymm0, %ymm1, %ymm2942; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]943; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2,3],ymm3[4],ymm2[5,6],ymm3[7],ymm2[8],ymm3[9],ymm2[10,11],ymm3[12],ymm2[13,14],ymm3[15]944; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]945; AVX2-FCP-NEXT: vmovdqa 80(%rdi), %xmm3946; AVX2-FCP-NEXT: vmovdqa 64(%rdi), %xmm4947; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm5 = xmm4[0,1],xmm3[2],xmm4[3,4],xmm3[5],xmm4[6,7]948; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u,u,u,2,3,8,9,14,15,4,5,10,11]949; AVX2-FCP-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm5950; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm5 = ymm2[0,1,2],ymm5[3,4,5,6,7],ymm2[8,9,10],ymm5[11,12,13,14,15]951; AVX2-FCP-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7]952; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm5[4,5,6,7]953; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm5 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]954; AVX2-FCP-NEXT: vpblendvb %ymm5, %ymm1, %ymm0, %ymm5955; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm6 = ymm5[2,3,0,1]956; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0,1],ymm6[2],ymm5[3,4],ymm6[5],ymm5[6,7,8,9],ymm6[10],ymm5[11,12],ymm6[13],ymm5[14,15]957; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]958; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm3[0,1],xmm4[2],xmm3[3,4],xmm4[5],xmm3[6,7]959; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,u,u,4,5,10,11,0,1,6,7,12,13]960; AVX2-FCP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6961; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm5[0,1,2],ymm6[3,4,5,6,7],ymm5[8,9,10],ymm6[11,12,13,14,15]962; AVX2-FCP-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,6,7,4]963; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]964; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm6 = [0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0]965; AVX2-FCP-NEXT: vpblendvb %ymm6, %ymm1, %ymm0, %ymm0966; AVX2-FCP-NEXT: vextracti128 $1, %ymm0, %xmm1967; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7],ymm1[8],ymm0[9,10],ymm1[11],ymm0[12,13],ymm1[14],ymm0[15]968; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,u,u,u,u,u,u,u,u,u,u,u,u]969; AVX2-FCP-NEXT: vpshufb %ymm1, %ymm0, %ymm0970; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]971; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm3, %xmm1972; AVX2-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1973; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]974; AVX2-FCP-NEXT: vmovdqa %ymm2, (%rsi)975; AVX2-FCP-NEXT: vmovdqa %ymm5, (%rdx)976; AVX2-FCP-NEXT: vmovdqa %ymm0, (%rcx)977; AVX2-FCP-NEXT: vzeroupper978; AVX2-FCP-NEXT: retq979;980; AVX512-LABEL: load_i16_stride3_vf16:981; AVX512: # %bb.0:982; AVX512-NEXT: vmovdqa 32(%rdi), %ymm1983; AVX512-NEXT: vmovdqa (%rdi), %ymm2984; AVX512-NEXT: vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]985; AVX512-NEXT: vmovdqa %ymm0, %ymm3986; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm1 ^ (ymm3 & (ymm2 ^ ymm1))987; AVX512-NEXT: vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]988; AVX512-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7],ymm3[8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13,14],ymm4[15]989; AVX512-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]990; AVX512-NEXT: vmovdqa 80(%rdi), %xmm4991; AVX512-NEXT: vmovdqa 64(%rdi), %xmm5992; AVX512-NEXT: vpblendw {{.*#+}} xmm6 = xmm5[0,1],xmm4[2],xmm5[3,4],xmm4[5],xmm5[6,7]993; AVX512-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,u,u,2,3,8,9,14,15,4,5,10,11]994; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm6995; AVX512-NEXT: vpblendw {{.*#+}} ymm6 = ymm3[0,1,2],ymm6[3,4,5,6,7],ymm3[8,9,10],ymm6[11,12,13,14,15]996; AVX512-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,4,7]997; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm6[4,5,6,7]998; AVX512-NEXT: vmovdqa {{.*#+}} ymm6 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]999; AVX512-NEXT: vpternlogq {{.*#+}} ymm6 = ymm2 ^ (ymm6 & (ymm1 ^ ymm2))1000; AVX512-NEXT: vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]1001; AVX512-NEXT: vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7,8,9],ymm7[10],ymm6[11,12],ymm7[13],ymm6[14,15]1002; AVX512-NEXT: vpshufb {{.*#+}} ymm6 = ymm6[2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]1003; AVX512-NEXT: vpblendw {{.*#+}} xmm7 = xmm4[0,1],xmm5[2],xmm4[3,4],xmm5[5],xmm4[6,7]1004; AVX512-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u,u,4,5,10,11,0,1,6,7,12,13]1005; AVX512-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm71006; AVX512-NEXT: vpblendw {{.*#+}} ymm7 = ymm6[0,1,2],ymm7[3,4,5,6,7],ymm6[8,9,10],ymm7[11,12,13,14,15]1007; AVX512-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,5,6,7,4]1008; AVX512-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm7[4,5,6,7]1009; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm2 ^ (ymm0 & (ymm1 ^ ymm2))1010; AVX512-NEXT: vmovdqa 16(%rdi), %xmm11011; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7],ymm1[8],ymm0[9,10],ymm1[11],ymm0[12,13],ymm1[14],ymm0[15]1012; AVX512-NEXT: vmovdqa {{.*#+}} ymm1 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,u,u,u,u,u,u,u,u,u,u,u,u]1013; AVX512-NEXT: vpshufb %ymm1, %ymm0, %ymm01014; AVX512-NEXT: vpblendw {{.*#+}} xmm2 = xmm5[0],xmm4[1],xmm5[2,3],xmm4[4],xmm5[5,6],xmm4[7]1015; AVX512-NEXT: vpshufb %xmm1, %xmm2, %xmm11016; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm11017; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]1018; AVX512-NEXT: vmovdqa %ymm3, (%rsi)1019; AVX512-NEXT: vmovdqa %ymm6, (%rdx)1020; AVX512-NEXT: vmovdqa %ymm0, (%rcx)1021; AVX512-NEXT: vzeroupper1022; AVX512-NEXT: retq1023;1024; AVX512-FCP-LABEL: load_i16_stride3_vf16:1025; AVX512-FCP: # %bb.0:1026; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm11027; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm21028; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]1029; AVX512-FCP-NEXT: vmovdqa %ymm0, %ymm31030; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm3 = ymm1 ^ (ymm3 & (ymm2 ^ ymm1))1031; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]1032; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7],ymm3[8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13,14],ymm4[15]1033; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]1034; AVX512-FCP-NEXT: vmovdqa 80(%rdi), %xmm41035; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %xmm51036; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm5[0,1],xmm4[2],xmm5[3,4],xmm4[5],xmm5[6,7]1037; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,u,u,2,3,8,9,14,15,4,5,10,11]1038; AVX512-FCP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm61039; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm3[0,1,2],ymm6[3,4,5,6,7],ymm3[8,9,10],ymm6[11,12,13,14,15]1040; AVX512-FCP-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,4,7]1041; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm6[4,5,6,7]1042; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]1043; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm6 = ymm2 ^ (ymm6 & (ymm1 ^ ymm2))1044; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]1045; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7,8,9],ymm7[10],ymm6[11,12],ymm7[13],ymm6[14,15]1046; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm6 = ymm6[2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]1047; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm7 = xmm4[0,1],xmm5[2],xmm4[3,4],xmm5[5],xmm4[6,7]1048; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u,u,4,5,10,11,0,1,6,7,12,13]1049; AVX512-FCP-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm71050; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm6[0,1,2],ymm7[3,4,5,6,7],ymm6[8,9,10],ymm7[11,12,13,14,15]1051; AVX512-FCP-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,5,6,7,4]1052; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm7[4,5,6,7]1053; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm2 ^ (ymm0 & (ymm1 ^ ymm2))1054; AVX512-FCP-NEXT: vmovdqa 16(%rdi), %xmm11055; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7],ymm1[8],ymm0[9,10],ymm1[11],ymm0[12,13],ymm1[14],ymm0[15]1056; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,u,u,u,u,u,u,u,u,u,u,u,u]1057; AVX512-FCP-NEXT: vpshufb %ymm1, %ymm0, %ymm01058; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm5[0],xmm4[1],xmm5[2,3],xmm4[4],xmm5[5,6],xmm4[7]1059; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm2, %xmm11060; AVX512-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm11061; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]1062; AVX512-FCP-NEXT: vmovdqa %ymm3, (%rsi)1063; AVX512-FCP-NEXT: vmovdqa %ymm6, (%rdx)1064; AVX512-FCP-NEXT: vmovdqa %ymm0, (%rcx)1065; AVX512-FCP-NEXT: vzeroupper1066; AVX512-FCP-NEXT: retq1067;1068; AVX512DQ-LABEL: load_i16_stride3_vf16:1069; AVX512DQ: # %bb.0:1070; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm11071; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm21072; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]1073; AVX512DQ-NEXT: vmovdqa %ymm0, %ymm31074; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm3 = ymm1 ^ (ymm3 & (ymm2 ^ ymm1))1075; AVX512DQ-NEXT: vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]1076; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7],ymm3[8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13,14],ymm4[15]1077; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]1078; AVX512DQ-NEXT: vmovdqa 80(%rdi), %xmm41079; AVX512DQ-NEXT: vmovdqa 64(%rdi), %xmm51080; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm6 = xmm5[0,1],xmm4[2],xmm5[3,4],xmm4[5],xmm5[6,7]1081; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,u,u,2,3,8,9,14,15,4,5,10,11]1082; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm61083; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm6 = ymm3[0,1,2],ymm6[3,4,5,6,7],ymm3[8,9,10],ymm6[11,12,13,14,15]1084; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,4,7]1085; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm6[4,5,6,7]1086; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm6 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]1087; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm6 = ymm2 ^ (ymm6 & (ymm1 ^ ymm2))1088; AVX512DQ-NEXT: vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]1089; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7,8,9],ymm7[10],ymm6[11,12],ymm7[13],ymm6[14,15]1090; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm6 = ymm6[2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]1091; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm7 = xmm4[0,1],xmm5[2],xmm4[3,4],xmm5[5],xmm4[6,7]1092; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u,u,4,5,10,11,0,1,6,7,12,13]1093; AVX512DQ-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm71094; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm7 = ymm6[0,1,2],ymm7[3,4,5,6,7],ymm6[8,9,10],ymm7[11,12,13,14,15]1095; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,5,6,7,4]1096; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm7[4,5,6,7]1097; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm0 = ymm2 ^ (ymm0 & (ymm1 ^ ymm2))1098; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm11099; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7],ymm1[8],ymm0[9,10],ymm1[11],ymm0[12,13],ymm1[14],ymm0[15]1100; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm1 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,u,u,u,u,u,u,u,u,u,u,u,u]1101; AVX512DQ-NEXT: vpshufb %ymm1, %ymm0, %ymm01102; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm5[0],xmm4[1],xmm5[2,3],xmm4[4],xmm5[5,6],xmm4[7]1103; AVX512DQ-NEXT: vpshufb %xmm1, %xmm2, %xmm11104; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm11105; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]1106; AVX512DQ-NEXT: vmovdqa %ymm3, (%rsi)1107; AVX512DQ-NEXT: vmovdqa %ymm6, (%rdx)1108; AVX512DQ-NEXT: vmovdqa %ymm0, (%rcx)1109; AVX512DQ-NEXT: vzeroupper1110; AVX512DQ-NEXT: retq1111;1112; AVX512DQ-FCP-LABEL: load_i16_stride3_vf16:1113; AVX512DQ-FCP: # %bb.0:1114; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm11115; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm21116; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]1117; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, %ymm31118; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm3 = ymm1 ^ (ymm3 & (ymm2 ^ ymm1))1119; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]1120; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7],ymm3[8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13,14],ymm4[15]1121; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]1122; AVX512DQ-FCP-NEXT: vmovdqa 80(%rdi), %xmm41123; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %xmm51124; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm5[0,1],xmm4[2],xmm5[3,4],xmm4[5],xmm5[6,7]1125; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,u,u,2,3,8,9,14,15,4,5,10,11]1126; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm61127; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm3[0,1,2],ymm6[3,4,5,6,7],ymm3[8,9,10],ymm6[11,12,13,14,15]1128; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,4,7]1129; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm6[4,5,6,7]1130; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]1131; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm6 = ymm2 ^ (ymm6 & (ymm1 ^ ymm2))1132; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]1133; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7,8,9],ymm7[10],ymm6[11,12],ymm7[13],ymm6[14,15]1134; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm6 = ymm6[2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]1135; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm7 = xmm4[0,1],xmm5[2],xmm4[3,4],xmm5[5],xmm4[6,7]1136; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u,u,4,5,10,11,0,1,6,7,12,13]1137; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm71138; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm6[0,1,2],ymm7[3,4,5,6,7],ymm6[8,9,10],ymm7[11,12,13,14,15]1139; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,5,6,7,4]1140; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm7[4,5,6,7]1141; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm2 ^ (ymm0 & (ymm1 ^ ymm2))1142; AVX512DQ-FCP-NEXT: vmovdqa 16(%rdi), %xmm11143; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7],ymm1[8],ymm0[9,10],ymm1[11],ymm0[12,13],ymm1[14],ymm0[15]1144; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm1 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,u,u,u,u,u,u,u,u,u,u,u,u]1145; AVX512DQ-FCP-NEXT: vpshufb %ymm1, %ymm0, %ymm01146; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm5[0],xmm4[1],xmm5[2,3],xmm4[4],xmm5[5,6],xmm4[7]1147; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm2, %xmm11148; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm11149; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]1150; AVX512DQ-FCP-NEXT: vmovdqa %ymm3, (%rsi)1151; AVX512DQ-FCP-NEXT: vmovdqa %ymm6, (%rdx)1152; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, (%rcx)1153; AVX512DQ-FCP-NEXT: vzeroupper1154; AVX512DQ-FCP-NEXT: retq1155;1156; AVX512BW-LABEL: load_i16_stride3_vf16:1157; AVX512BW: # %bb.0:1158; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm01159; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm11160; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [0,3,6,9,12,15,18,21,24,27,30,33,36,39,42,45]1161; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm21162; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm3 = [1,4,7,10,13,16,19,22,25,28,31,34,37,40,43,46]1163; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm31164; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm4 = [2,5,8,11,14,17,20,23,26,29,32,35,38,41,44,47]1165; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm41166; AVX512BW-NEXT: vmovdqa %ymm2, (%rsi)1167; AVX512BW-NEXT: vmovdqa %ymm3, (%rdx)1168; AVX512BW-NEXT: vmovdqa %ymm4, (%rcx)1169; AVX512BW-NEXT: vzeroupper1170; AVX512BW-NEXT: retq1171;1172; AVX512BW-FCP-LABEL: load_i16_stride3_vf16:1173; AVX512BW-FCP: # %bb.0:1174; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm01175; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm11176; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [0,3,6,9,12,15,18,21,24,27,30,33,36,39,42,45]1177; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm21178; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm3 = [1,4,7,10,13,16,19,22,25,28,31,34,37,40,43,46]1179; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm31180; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm4 = [2,5,8,11,14,17,20,23,26,29,32,35,38,41,44,47]1181; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm41182; AVX512BW-FCP-NEXT: vmovdqa %ymm2, (%rsi)1183; AVX512BW-FCP-NEXT: vmovdqa %ymm3, (%rdx)1184; AVX512BW-FCP-NEXT: vmovdqa %ymm4, (%rcx)1185; AVX512BW-FCP-NEXT: vzeroupper1186; AVX512BW-FCP-NEXT: retq1187;1188; AVX512DQ-BW-LABEL: load_i16_stride3_vf16:1189; AVX512DQ-BW: # %bb.0:1190; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm01191; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %zmm11192; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [0,3,6,9,12,15,18,21,24,27,30,33,36,39,42,45]1193; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm21194; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm3 = [1,4,7,10,13,16,19,22,25,28,31,34,37,40,43,46]1195; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm31196; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm4 = [2,5,8,11,14,17,20,23,26,29,32,35,38,41,44,47]1197; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm41198; AVX512DQ-BW-NEXT: vmovdqa %ymm2, (%rsi)1199; AVX512DQ-BW-NEXT: vmovdqa %ymm3, (%rdx)1200; AVX512DQ-BW-NEXT: vmovdqa %ymm4, (%rcx)1201; AVX512DQ-BW-NEXT: vzeroupper1202; AVX512DQ-BW-NEXT: retq1203;1204; AVX512DQ-BW-FCP-LABEL: load_i16_stride3_vf16:1205; AVX512DQ-BW-FCP: # %bb.0:1206; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm01207; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm11208; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [0,3,6,9,12,15,18,21,24,27,30,33,36,39,42,45]1209; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm21210; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm3 = [1,4,7,10,13,16,19,22,25,28,31,34,37,40,43,46]1211; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm31212; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm4 = [2,5,8,11,14,17,20,23,26,29,32,35,38,41,44,47]1213; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm41214; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm2, (%rsi)1215; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm3, (%rdx)1216; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm4, (%rcx)1217; AVX512DQ-BW-FCP-NEXT: vzeroupper1218; AVX512DQ-BW-FCP-NEXT: retq1219 %wide.vec = load <48 x i16>, ptr %in.vec, align 641220 %strided.vec0 = shufflevector <48 x i16> %wide.vec, <48 x i16> poison, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>1221 %strided.vec1 = shufflevector <48 x i16> %wide.vec, <48 x i16> poison, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>1222 %strided.vec2 = shufflevector <48 x i16> %wide.vec, <48 x i16> poison, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>1223 store <16 x i16> %strided.vec0, ptr %out.vec0, align 641224 store <16 x i16> %strided.vec1, ptr %out.vec1, align 641225 store <16 x i16> %strided.vec2, ptr %out.vec2, align 641226 ret void1227}1228 1229define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2) nounwind {1230; SSE-LABEL: load_i16_stride3_vf32:1231; SSE: # %bb.0:1232; SSE-NEXT: subq $40, %rsp1233; SSE-NEXT: movdqa 96(%rdi), %xmm51234; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1235; SSE-NEXT: movdqa 176(%rdi), %xmm61236; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1237; SSE-NEXT: movdqa 144(%rdi), %xmm131238; SSE-NEXT: movdqa 160(%rdi), %xmm91239; SSE-NEXT: movdqa 80(%rdi), %xmm81240; SSE-NEXT: movdqa (%rdi), %xmm151241; SSE-NEXT: movdqa 16(%rdi), %xmm101242; SSE-NEXT: movdqa 32(%rdi), %xmm71243; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1244; SSE-NEXT: movdqa 48(%rdi), %xmm01245; SSE-NEXT: movdqa 64(%rdi), %xmm121246; SSE-NEXT: movdqa {{.*#+}} xmm1 = [65535,0,65535,65535,0,65535,65535,0]1247; SSE-NEXT: movdqa %xmm1, %xmm21248; SSE-NEXT: pandn %xmm12, %xmm21249; SSE-NEXT: movdqa %xmm0, %xmm31250; SSE-NEXT: pand %xmm1, %xmm31251; SSE-NEXT: por %xmm2, %xmm31252; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,2,1,3]1253; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,6,7]1254; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3]1255; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm2[0,3,2,1,4,5,6,7]1256; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm12[0,1,2,3,4,7,6,7]1257; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm8[0,1,2,1]1258; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1259; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,6,5]1260; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,0],xmm2[2,0]1261; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,0]1262; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1263; SSE-NEXT: movdqa %xmm1, %xmm21264; SSE-NEXT: pandn %xmm10, %xmm21265; SSE-NEXT: movdqa %xmm15, %xmm31266; SSE-NEXT: pand %xmm1, %xmm31267; SSE-NEXT: por %xmm2, %xmm31268; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,2,1,3]1269; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,6,7]1270; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3]1271; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm2[0,3,2,1,4,5,6,7]1272; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm10[0,1,2,3,4,7,6,7]1273; SSE-NEXT: movdqa %xmm10, %xmm111274; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1275; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm7[0,1,2,1]1276; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,6,5]1277; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,0],xmm2[2,0]1278; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,0]1279; SSE-NEXT: movaps %xmm4, (%rsp) # 16-byte Spill1280; SSE-NEXT: movdqa %xmm1, %xmm21281; SSE-NEXT: movdqa %xmm9, %xmm71282; SSE-NEXT: pandn %xmm9, %xmm21283; SSE-NEXT: movdqa %xmm13, %xmm31284; SSE-NEXT: pand %xmm1, %xmm31285; SSE-NEXT: por %xmm2, %xmm31286; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm9[0,1,2,3,4,7,6,7]1287; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm6[0,1,2,1]1288; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6,5]1289; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[3,0],xmm2[2,0]1290; SSE-NEXT: movdqa 112(%rdi), %xmm91291; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,2,1,3]1292; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,6,7]1293; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3]1294; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,3,2,1,4,5,6,7]1295; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,0]1296; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1297; SSE-NEXT: movdqa %xmm5, %xmm21298; SSE-NEXT: pand %xmm1, %xmm21299; SSE-NEXT: pandn %xmm9, %xmm11300; SSE-NEXT: por %xmm2, %xmm11301; SSE-NEXT: movdqa 128(%rdi), %xmm51302; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm5[0,1,2,1]1303; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,6,5]1304; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm9[0,1,2,3,4,7,6,7]1305; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1306; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,0],xmm3[2,0]1307; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]1308; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7]1309; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]1310; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,1,4,5,6,7]1311; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,0]1312; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1313; SSE-NEXT: movdqa {{.*#+}} xmm14 = [65535,65535,0,65535,65535,0,65535,65535]1314; SSE-NEXT: movdqa %xmm14, %xmm41315; SSE-NEXT: pandn %xmm0, %xmm41316; SSE-NEXT: movdqa %xmm0, %xmm11317; SSE-NEXT: movdqa %xmm14, %xmm01318; SSE-NEXT: pandn %xmm12, %xmm01319; SSE-NEXT: pand %xmm14, %xmm11320; SSE-NEXT: por %xmm0, %xmm11321; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm8[0,3,2,3,4,5,6,7]1322; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]1323; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,4,5,6]1324; SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,65535,65535,65535,65535,0,0,0]1325; SSE-NEXT: movdqa %xmm0, %xmm101326; SSE-NEXT: pandn %xmm2, %xmm101327; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]1328; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,7]1329; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3]1330; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,0,4,5,6,7]1331; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5]1332; SSE-NEXT: pand %xmm0, %xmm11333; SSE-NEXT: por %xmm1, %xmm101334; SSE-NEXT: movdqa %xmm14, %xmm31335; SSE-NEXT: pandn %xmm15, %xmm31336; SSE-NEXT: movdqa %xmm15, %xmm11337; SSE-NEXT: movdqa %xmm14, %xmm21338; SSE-NEXT: pandn %xmm11, %xmm21339; SSE-NEXT: pand %xmm14, %xmm11340; SSE-NEXT: por %xmm2, %xmm11341; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload1342; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm11[0,3,2,3,4,5,6,7]1343; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,3]1344; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,5,6]1345; SSE-NEXT: movdqa %xmm0, %xmm151346; SSE-NEXT: pandn %xmm2, %xmm151347; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]1348; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,7]1349; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3]1350; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,0,4,5,6,7]1351; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5]1352; SSE-NEXT: pand %xmm0, %xmm11353; SSE-NEXT: por %xmm1, %xmm151354; SSE-NEXT: movdqa %xmm14, %xmm61355; SSE-NEXT: pandn %xmm13, %xmm61356; SSE-NEXT: movdqa %xmm13, %xmm11357; SSE-NEXT: movdqa %xmm14, %xmm131358; SSE-NEXT: pandn %xmm7, %xmm131359; SSE-NEXT: pand %xmm14, %xmm11360; SSE-NEXT: por %xmm13, %xmm11361; SSE-NEXT: pshuflw $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload1362; SSE-NEXT: # xmm13 = mem[0,3,2,3,4,5,6,7]1363; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm13[0,1,0,3]1364; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm13[0,1,2,3,4,4,5,6]1365; SSE-NEXT: movdqa %xmm0, %xmm131366; SSE-NEXT: pandn %xmm2, %xmm131367; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]1368; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,7]1369; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3]1370; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,0,4,5,6,7]1371; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5]1372; SSE-NEXT: pand %xmm0, %xmm11373; SSE-NEXT: por %xmm1, %xmm131374; SSE-NEXT: movdqa %xmm14, %xmm11375; SSE-NEXT: pandn %xmm9, %xmm11376; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload1377; SSE-NEXT: movdqa %xmm8, %xmm21378; SSE-NEXT: pand %xmm14, %xmm21379; SSE-NEXT: por %xmm1, %xmm21380; SSE-NEXT: movdqa %xmm5, %xmm91381; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm5[0,3,2,3,4,5,6,7]1382; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3]1383; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,5,6]1384; SSE-NEXT: movdqa %xmm0, %xmm51385; SSE-NEXT: pandn %xmm1, %xmm51386; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm2[2,1,2,3,4,5,6,7]1387; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,7]1388; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3]1389; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,0,4,5,6,7]1390; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5]1391; SSE-NEXT: pand %xmm0, %xmm11392; SSE-NEXT: por %xmm1, %xmm51393; SSE-NEXT: pand %xmm14, %xmm121394; SSE-NEXT: por %xmm4, %xmm121395; SSE-NEXT: pshufhw $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload1396; SSE-NEXT: # xmm1 = mem[0,1,2,3,4,7,6,7]1397; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,2]1398; SSE-NEXT: movdqa %xmm0, %xmm41399; SSE-NEXT: pandn %xmm1, %xmm41400; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm12[3,1,2,0]1401; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7]1402; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,0,3]1403; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,0,3,4,5,6,7]1404; SSE-NEXT: pand %xmm0, %xmm11405; SSE-NEXT: por %xmm1, %xmm41406; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload1407; SSE-NEXT: pand %xmm14, %xmm11408; SSE-NEXT: por %xmm3, %xmm11409; SSE-NEXT: movdqa %xmm1, %xmm21410; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm11[0,1,2,3,4,7,6,7]1411; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,2]1412; SSE-NEXT: movdqa %xmm0, %xmm31413; SSE-NEXT: pandn %xmm1, %xmm31414; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,1,2,0]1415; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7]1416; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,0,3]1417; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,0,3,4,5,6,7]1418; SSE-NEXT: pand %xmm0, %xmm11419; SSE-NEXT: por %xmm1, %xmm31420; SSE-NEXT: pand %xmm14, %xmm71421; SSE-NEXT: por %xmm6, %xmm71422; SSE-NEXT: pshufhw $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload1423; SSE-NEXT: # xmm1 = mem[0,1,2,3,4,7,6,7]1424; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,2]1425; SSE-NEXT: movdqa %xmm0, %xmm21426; SSE-NEXT: pandn %xmm1, %xmm21427; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[3,1,2,0]1428; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7]1429; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,0,3]1430; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,0,3,4,5,6,7]1431; SSE-NEXT: pand %xmm0, %xmm11432; SSE-NEXT: por %xmm1, %xmm21433; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload1434; SSE-NEXT: pand %xmm14, %xmm11435; SSE-NEXT: pandn %xmm8, %xmm141436; SSE-NEXT: por %xmm1, %xmm141437; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm14[3,1,2,0]1438; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7]1439; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,0,3]1440; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,0,3,4,5,6,7]1441; SSE-NEXT: pand %xmm0, %xmm11442; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm9[0,1,2,3,4,7,6,7]1443; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,1,0,2]1444; SSE-NEXT: pandn %xmm6, %xmm01445; SSE-NEXT: por %xmm1, %xmm01446; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload1447; SSE-NEXT: movaps %xmm1, 32(%rsi)1448; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload1449; SSE-NEXT: movaps %xmm1, 48(%rsi)1450; SSE-NEXT: movaps (%rsp), %xmm1 # 16-byte Reload1451; SSE-NEXT: movaps %xmm1, (%rsi)1452; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload1453; SSE-NEXT: movaps %xmm1, 16(%rsi)1454; SSE-NEXT: movdqa %xmm5, 32(%rdx)1455; SSE-NEXT: movdqa %xmm13, 48(%rdx)1456; SSE-NEXT: movdqa %xmm15, (%rdx)1457; SSE-NEXT: movdqa %xmm10, 16(%rdx)1458; SSE-NEXT: movdqa %xmm0, 32(%rcx)1459; SSE-NEXT: movdqa %xmm2, 48(%rcx)1460; SSE-NEXT: movdqa %xmm3, (%rcx)1461; SSE-NEXT: movdqa %xmm4, 16(%rcx)1462; SSE-NEXT: addq $40, %rsp1463; SSE-NEXT: retq1464;1465; AVX-LABEL: load_i16_stride3_vf32:1466; AVX: # %bb.0:1467; AVX-NEXT: vmovdqa 80(%rdi), %xmm21468; AVX-NEXT: vmovdqa 64(%rdi), %xmm51469; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm5[0,1],xmm2[2],xmm5[3,4],xmm2[5],xmm5[6,7]1470; AVX-NEXT: vmovdqa {{.*#+}} xmm9 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]1471; AVX-NEXT: vpshufb %xmm9, %xmm0, %xmm01472; AVX-NEXT: vmovdqa (%rdi), %xmm11473; AVX-NEXT: vmovdqa 16(%rdi), %xmm31474; AVX-NEXT: vmovdqa 32(%rdi), %xmm41475; AVX-NEXT: vmovdqa 48(%rdi), %xmm61476; AVX-NEXT: vpshuflw {{.*#+}} xmm7 = xmm6[0,3,2,3,4,5,6,7]1477; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[0,3,2,3]1478; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm7[0,1,2],xmm0[3,4,5,6,7]1479; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm1[0],xmm3[1],xmm1[2,3],xmm3[4],xmm1[5,6],xmm3[7]1480; AVX-NEXT: vmovdqa {{.*#+}} xmm12 = [0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]1481; AVX-NEXT: vpshufb %xmm12, %xmm7, %xmm71482; AVX-NEXT: vpshufd {{.*#+}} xmm8 = xmm4[0,1,2,1]1483; AVX-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,6,5]1484; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5],xmm8[6,7]1485; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm7, %ymm01486; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill1487; AVX-NEXT: vmovdqa 176(%rdi), %xmm71488; AVX-NEXT: vmovdqa 160(%rdi), %xmm81489; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm8[0,1],xmm7[2],xmm8[3,4],xmm7[5],xmm8[6,7]1490; AVX-NEXT: vpshufb %xmm9, %xmm10, %xmm101491; AVX-NEXT: vmovdqa 144(%rdi), %xmm91492; AVX-NEXT: vpshuflw {{.*#+}} xmm11 = xmm9[0,3,2,3,4,5,6,7]1493; AVX-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[0,3,2,3]1494; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm11[0,1,2],xmm10[3,4,5,6,7]1495; AVX-NEXT: vmovdqa 112(%rdi), %xmm101496; AVX-NEXT: vmovdqa 96(%rdi), %xmm111497; AVX-NEXT: vpblendw {{.*#+}} xmm14 = xmm11[0],xmm10[1],xmm11[2,3],xmm10[4],xmm11[5,6],xmm10[7]1498; AVX-NEXT: vpshufb %xmm12, %xmm14, %xmm141499; AVX-NEXT: vmovdqa 128(%rdi), %xmm121500; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm12[0,1,2,1]1501; AVX-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,5,6,5]1502; AVX-NEXT: vpblendw {{.*#+}} xmm14 = xmm14[0,1,2,3,4,5],xmm15[6,7]1503; AVX-NEXT: vinsertf128 $1, %xmm13, %ymm14, %ymm01504; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill1505; AVX-NEXT: vpblendw {{.*#+}} xmm14 = xmm2[0,1],xmm5[2],xmm2[3,4],xmm5[5],xmm2[6,7]1506; AVX-NEXT: vpblendw {{.*#+}} xmm14 = xmm14[0],xmm6[1],xmm14[2,3],xmm6[4],xmm14[5,6],xmm6[7]1507; AVX-NEXT: vmovdqa {{.*#+}} xmm15 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13]1508; AVX-NEXT: vpshufb %xmm15, %xmm14, %xmm141509; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm3[2],xmm1[3,4],xmm3[5],xmm1[6,7]1510; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm4[0],xmm0[1,2],xmm4[3],xmm0[4,5],xmm4[6],xmm0[7]1511; AVX-NEXT: vpshufb %xmm15, %xmm0, %xmm01512; AVX-NEXT: vinsertf128 $1, %xmm14, %ymm0, %ymm141513; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm7[0,1],xmm8[2],xmm7[3,4],xmm8[5],xmm7[6,7]1514; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm9[1],xmm0[2,3],xmm9[4],xmm0[5,6],xmm9[7]1515; AVX-NEXT: vpshufb %xmm15, %xmm0, %xmm01516; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm11[0,1],xmm10[2],xmm11[3,4],xmm10[5],xmm11[6,7]1517; AVX-NEXT: vpblendw {{.*#+}} xmm13 = xmm12[0],xmm13[1,2],xmm12[3],xmm13[4,5],xmm12[6],xmm13[7]1518; AVX-NEXT: vpshufb %xmm15, %xmm13, %xmm131519; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm13, %ymm01520; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm5[0],xmm2[1],xmm5[2,3],xmm2[4],xmm5[5,6],xmm2[7]1521; AVX-NEXT: vmovdqa {{.*#+}} xmm5 = [u,u,u,u,0,1,6,7,12,13,2,3,8,9,14,15]1522; AVX-NEXT: vpshufb %xmm5, %xmm2, %xmm21523; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[2,1,2,3]1524; AVX-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[2,1,2,3,4,5,6,7]1525; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm6[0,1],xmm2[2,3,4,5,6,7]1526; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1],xmm1[2],xmm3[3,4],xmm1[5],xmm3[6,7]1527; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm4[1],xmm1[2,3],xmm4[4],xmm1[5,6],xmm4[7]1528; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15]1529; AVX-NEXT: vpshufb %xmm3, %xmm1, %xmm11530; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm11531; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm8[0],xmm7[1],xmm8[2,3],xmm7[4],xmm8[5,6],xmm7[7]1532; AVX-NEXT: vpshufb %xmm5, %xmm2, %xmm21533; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm9[2,1,2,3]1534; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[2,1,2,3,4,5,6,7]1535; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0,1],xmm2[2,3,4,5,6,7]1536; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm10[0,1],xmm11[2],xmm10[3,4],xmm11[5],xmm10[6,7]1537; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm12[1],xmm4[2,3],xmm12[4],xmm4[5,6],xmm12[7]1538; AVX-NEXT: vpshufb %xmm3, %xmm4, %xmm31539; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm21540; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload1541; AVX-NEXT: vmovaps %ymm3, 32(%rsi)1542; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload1543; AVX-NEXT: vmovaps %ymm3, (%rsi)1544; AVX-NEXT: vmovaps %ymm0, 32(%rdx)1545; AVX-NEXT: vmovaps %ymm14, (%rdx)1546; AVX-NEXT: vmovaps %ymm2, 32(%rcx)1547; AVX-NEXT: vmovaps %ymm1, (%rcx)1548; AVX-NEXT: vzeroupper1549; AVX-NEXT: retq1550;1551; AVX2-LABEL: load_i16_stride3_vf32:1552; AVX2: # %bb.0:1553; AVX2-NEXT: vmovdqa (%rdi), %ymm01554; AVX2-NEXT: vmovdqa 32(%rdi), %ymm11555; AVX2-NEXT: vmovdqa 96(%rdi), %ymm21556; AVX2-NEXT: vmovdqa 128(%rdi), %ymm41557; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm7 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]1558; AVX2-NEXT: vpblendvb %ymm7, %ymm2, %ymm4, %ymm31559; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm3[2,3,0,1]1560; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm5[1],ymm3[2,3],ymm5[4],ymm3[5,6],ymm5[7],ymm3[8],ymm5[9],ymm3[10,11],ymm5[12],ymm3[13,14],ymm5[15]1561; AVX2-NEXT: vmovdqa {{.*#+}} ymm8 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]1562; AVX2-NEXT: vpshufb %ymm8, %ymm3, %ymm31563; AVX2-NEXT: vmovdqa 176(%rdi), %xmm51564; AVX2-NEXT: vmovdqa 160(%rdi), %xmm61565; AVX2-NEXT: vpblendw {{.*#+}} xmm9 = xmm6[0,1],xmm5[2],xmm6[3,4],xmm5[5],xmm6[6,7]1566; AVX2-NEXT: vmovdqa {{.*#+}} xmm10 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]1567; AVX2-NEXT: vpshufb %xmm10, %xmm9, %xmm91568; AVX2-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm91569; AVX2-NEXT: vpblendw {{.*#+}} ymm9 = ymm3[0,1,2],ymm9[3,4,5,6,7],ymm3[8,9,10],ymm9[11,12,13,14,15]1570; AVX2-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,4,7]1571; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm9[4,5,6,7]1572; AVX2-NEXT: vpblendvb %ymm7, %ymm0, %ymm1, %ymm71573; AVX2-NEXT: vpermq {{.*#+}} ymm9 = ymm7[2,3,0,1]1574; AVX2-NEXT: vpblendw {{.*#+}} ymm7 = ymm7[0],ymm9[1],ymm7[2,3],ymm9[4],ymm7[5,6],ymm9[7],ymm7[8],ymm9[9],ymm7[10,11],ymm9[12],ymm7[13,14],ymm9[15]1575; AVX2-NEXT: vpshufb %ymm8, %ymm7, %ymm91576; AVX2-NEXT: vmovdqa 80(%rdi), %xmm71577; AVX2-NEXT: vmovdqa 64(%rdi), %xmm81578; AVX2-NEXT: vpblendw {{.*#+}} xmm11 = xmm8[0,1],xmm7[2],xmm8[3,4],xmm7[5],xmm8[6,7]1579; AVX2-NEXT: vpshufb %xmm10, %xmm11, %xmm101580; AVX2-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm101581; AVX2-NEXT: vpblendw {{.*#+}} ymm10 = ymm9[0,1,2],ymm10[3,4,5,6,7],ymm9[8,9,10],ymm10[11,12,13,14,15]1582; AVX2-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,6,5,4,7]1583; AVX2-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm10[4,5,6,7]1584; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm11 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]1585; AVX2-NEXT: vpblendvb %ymm11, %ymm4, %ymm2, %ymm101586; AVX2-NEXT: vpermq {{.*#+}} ymm12 = ymm10[2,3,0,1]1587; AVX2-NEXT: vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm12[2],ymm10[3,4],ymm12[5],ymm10[6,7,8,9],ymm12[10],ymm10[11,12],ymm12[13],ymm10[14,15]1588; AVX2-NEXT: vmovdqa {{.*#+}} ymm12 = [2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]1589; AVX2-NEXT: vpshufb %ymm12, %ymm10, %ymm101590; AVX2-NEXT: vpblendw {{.*#+}} xmm13 = xmm5[0,1],xmm6[2],xmm5[3,4],xmm6[5],xmm5[6,7]1591; AVX2-NEXT: vmovdqa {{.*#+}} xmm14 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]1592; AVX2-NEXT: vpshufb %xmm14, %xmm13, %xmm131593; AVX2-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm131594; AVX2-NEXT: vpblendw {{.*#+}} ymm13 = ymm10[0,1,2],ymm13[3,4,5,6,7],ymm10[8,9,10],ymm13[11,12,13,14,15]1595; AVX2-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,6,7,4]1596; AVX2-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm13[4,5,6,7]1597; AVX2-NEXT: vpblendvb %ymm11, %ymm1, %ymm0, %ymm111598; AVX2-NEXT: vpermq {{.*#+}} ymm13 = ymm11[2,3,0,1]1599; AVX2-NEXT: vpblendw {{.*#+}} ymm11 = ymm11[0,1],ymm13[2],ymm11[3,4],ymm13[5],ymm11[6,7,8,9],ymm13[10],ymm11[11,12],ymm13[13],ymm11[14,15]1600; AVX2-NEXT: vpshufb %ymm12, %ymm11, %ymm111601; AVX2-NEXT: vpblendw {{.*#+}} xmm12 = xmm7[0,1],xmm8[2],xmm7[3,4],xmm8[5],xmm7[6,7]1602; AVX2-NEXT: vpshufb %xmm14, %xmm12, %xmm121603; AVX2-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm121604; AVX2-NEXT: vpblendw {{.*#+}} ymm12 = ymm11[0,1,2],ymm12[3,4,5,6,7],ymm11[8,9,10],ymm12[11,12,13,14,15]1605; AVX2-NEXT: vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,5,6,7,4]1606; AVX2-NEXT: vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm12[4,5,6,7]1607; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm12 = [0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0]1608; AVX2-NEXT: vpblendvb %ymm12, %ymm4, %ymm2, %ymm21609; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm2[2,3,0,1]1610; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm4[0],ymm2[1,2],ymm4[3],ymm2[4,5],ymm4[6],ymm2[7],ymm4[8],ymm2[9,10],ymm4[11],ymm2[12,13],ymm4[14],ymm2[15]1611; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]1612; AVX2-NEXT: vpshufb %ymm4, %ymm2, %ymm21613; AVX2-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3],xmm5[4],xmm6[5,6],xmm5[7]1614; AVX2-NEXT: vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]1615; AVX2-NEXT: vpshufb %xmm6, %xmm5, %xmm51616; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm51617; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm5[5,6,7]1618; AVX2-NEXT: vpblendvb %ymm12, %ymm1, %ymm0, %ymm01619; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]1620; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7],ymm1[8],ymm0[9,10],ymm1[11],ymm0[12,13],ymm1[14],ymm0[15]1621; AVX2-NEXT: vpshufb %ymm4, %ymm0, %ymm01622; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm8[0],xmm7[1],xmm8[2,3],xmm7[4],xmm8[5,6],xmm7[7]1623; AVX2-NEXT: vpshufb %xmm6, %xmm1, %xmm11624; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm11625; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]1626; AVX2-NEXT: vmovdqa %ymm3, 32(%rsi)1627; AVX2-NEXT: vmovdqa %ymm9, (%rsi)1628; AVX2-NEXT: vmovdqa %ymm10, 32(%rdx)1629; AVX2-NEXT: vmovdqa %ymm11, (%rdx)1630; AVX2-NEXT: vmovdqa %ymm2, 32(%rcx)1631; AVX2-NEXT: vmovdqa %ymm0, (%rcx)1632; AVX2-NEXT: vzeroupper1633; AVX2-NEXT: retq1634;1635; AVX2-FP-LABEL: load_i16_stride3_vf32:1636; AVX2-FP: # %bb.0:1637; AVX2-FP-NEXT: vmovdqa (%rdi), %ymm01638; AVX2-FP-NEXT: vmovdqa 32(%rdi), %ymm11639; AVX2-FP-NEXT: vmovdqa 96(%rdi), %ymm21640; AVX2-FP-NEXT: vmovdqa 128(%rdi), %ymm41641; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm7 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]1642; AVX2-FP-NEXT: vpblendvb %ymm7, %ymm2, %ymm4, %ymm31643; AVX2-FP-NEXT: vpermq {{.*#+}} ymm5 = ymm3[2,3,0,1]1644; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm5[1],ymm3[2,3],ymm5[4],ymm3[5,6],ymm5[7],ymm3[8],ymm5[9],ymm3[10,11],ymm5[12],ymm3[13,14],ymm5[15]1645; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm8 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]1646; AVX2-FP-NEXT: vpshufb %ymm8, %ymm3, %ymm31647; AVX2-FP-NEXT: vmovdqa 176(%rdi), %xmm51648; AVX2-FP-NEXT: vmovdqa 160(%rdi), %xmm61649; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm9 = xmm6[0,1],xmm5[2],xmm6[3,4],xmm5[5],xmm6[6,7]1650; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm10 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]1651; AVX2-FP-NEXT: vpshufb %xmm10, %xmm9, %xmm91652; AVX2-FP-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm91653; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm9 = ymm3[0,1,2],ymm9[3,4,5,6,7],ymm3[8,9,10],ymm9[11,12,13,14,15]1654; AVX2-FP-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,4,7]1655; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm9[4,5,6,7]1656; AVX2-FP-NEXT: vpblendvb %ymm7, %ymm0, %ymm1, %ymm71657; AVX2-FP-NEXT: vpermq {{.*#+}} ymm9 = ymm7[2,3,0,1]1658; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm7 = ymm7[0],ymm9[1],ymm7[2,3],ymm9[4],ymm7[5,6],ymm9[7],ymm7[8],ymm9[9],ymm7[10,11],ymm9[12],ymm7[13,14],ymm9[15]1659; AVX2-FP-NEXT: vpshufb %ymm8, %ymm7, %ymm91660; AVX2-FP-NEXT: vmovdqa 80(%rdi), %xmm71661; AVX2-FP-NEXT: vmovdqa 64(%rdi), %xmm81662; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm11 = xmm8[0,1],xmm7[2],xmm8[3,4],xmm7[5],xmm8[6,7]1663; AVX2-FP-NEXT: vpshufb %xmm10, %xmm11, %xmm101664; AVX2-FP-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm101665; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm10 = ymm9[0,1,2],ymm10[3,4,5,6,7],ymm9[8,9,10],ymm10[11,12,13,14,15]1666; AVX2-FP-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,6,5,4,7]1667; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm10[4,5,6,7]1668; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm11 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]1669; AVX2-FP-NEXT: vpblendvb %ymm11, %ymm4, %ymm2, %ymm101670; AVX2-FP-NEXT: vpermq {{.*#+}} ymm12 = ymm10[2,3,0,1]1671; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm12[2],ymm10[3,4],ymm12[5],ymm10[6,7,8,9],ymm12[10],ymm10[11,12],ymm12[13],ymm10[14,15]1672; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm12 = [2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]1673; AVX2-FP-NEXT: vpshufb %ymm12, %ymm10, %ymm101674; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm13 = xmm5[0,1],xmm6[2],xmm5[3,4],xmm6[5],xmm5[6,7]1675; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm14 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]1676; AVX2-FP-NEXT: vpshufb %xmm14, %xmm13, %xmm131677; AVX2-FP-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm131678; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm13 = ymm10[0,1,2],ymm13[3,4,5,6,7],ymm10[8,9,10],ymm13[11,12,13,14,15]1679; AVX2-FP-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,6,7,4]1680; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm13[4,5,6,7]1681; AVX2-FP-NEXT: vpblendvb %ymm11, %ymm1, %ymm0, %ymm111682; AVX2-FP-NEXT: vpermq {{.*#+}} ymm13 = ymm11[2,3,0,1]1683; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm11 = ymm11[0,1],ymm13[2],ymm11[3,4],ymm13[5],ymm11[6,7,8,9],ymm13[10],ymm11[11,12],ymm13[13],ymm11[14,15]1684; AVX2-FP-NEXT: vpshufb %ymm12, %ymm11, %ymm111685; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm12 = xmm7[0,1],xmm8[2],xmm7[3,4],xmm8[5],xmm7[6,7]1686; AVX2-FP-NEXT: vpshufb %xmm14, %xmm12, %xmm121687; AVX2-FP-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm121688; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm12 = ymm11[0,1,2],ymm12[3,4,5,6,7],ymm11[8,9,10],ymm12[11,12,13,14,15]1689; AVX2-FP-NEXT: vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,5,6,7,4]1690; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm12[4,5,6,7]1691; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm12 = [0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0]1692; AVX2-FP-NEXT: vpblendvb %ymm12, %ymm4, %ymm2, %ymm21693; AVX2-FP-NEXT: vpermq {{.*#+}} ymm4 = ymm2[2,3,0,1]1694; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = ymm4[0],ymm2[1,2],ymm4[3],ymm2[4,5],ymm4[6],ymm2[7],ymm4[8],ymm2[9,10],ymm4[11],ymm2[12,13],ymm4[14],ymm2[15]1695; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm4 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]1696; AVX2-FP-NEXT: vpshufb %ymm4, %ymm2, %ymm21697; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3],xmm5[4],xmm6[5,6],xmm5[7]1698; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]1699; AVX2-FP-NEXT: vpshufb %xmm6, %xmm5, %xmm51700; AVX2-FP-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm51701; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm5[5,6,7]1702; AVX2-FP-NEXT: vpblendvb %ymm12, %ymm1, %ymm0, %ymm01703; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]1704; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7],ymm1[8],ymm0[9,10],ymm1[11],ymm0[12,13],ymm1[14],ymm0[15]1705; AVX2-FP-NEXT: vpshufb %ymm4, %ymm0, %ymm01706; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm1 = xmm8[0],xmm7[1],xmm8[2,3],xmm7[4],xmm8[5,6],xmm7[7]1707; AVX2-FP-NEXT: vpshufb %xmm6, %xmm1, %xmm11708; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm11709; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]1710; AVX2-FP-NEXT: vmovdqa %ymm3, 32(%rsi)1711; AVX2-FP-NEXT: vmovdqa %ymm9, (%rsi)1712; AVX2-FP-NEXT: vmovdqa %ymm10, 32(%rdx)1713; AVX2-FP-NEXT: vmovdqa %ymm11, (%rdx)1714; AVX2-FP-NEXT: vmovdqa %ymm2, 32(%rcx)1715; AVX2-FP-NEXT: vmovdqa %ymm0, (%rcx)1716; AVX2-FP-NEXT: vzeroupper1717; AVX2-FP-NEXT: retq1718;1719; AVX2-FCP-LABEL: load_i16_stride3_vf32:1720; AVX2-FCP: # %bb.0:1721; AVX2-FCP-NEXT: vmovdqa (%rdi), %ymm01722; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %ymm11723; AVX2-FCP-NEXT: vmovdqa 96(%rdi), %ymm21724; AVX2-FCP-NEXT: vmovdqa 128(%rdi), %ymm41725; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm7 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]1726; AVX2-FCP-NEXT: vpblendvb %ymm7, %ymm2, %ymm4, %ymm31727; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm5 = ymm3[2,3,0,1]1728; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm5[1],ymm3[2,3],ymm5[4],ymm3[5,6],ymm5[7],ymm3[8],ymm5[9],ymm3[10,11],ymm5[12],ymm3[13,14],ymm5[15]1729; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm8 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]1730; AVX2-FCP-NEXT: vpshufb %ymm8, %ymm3, %ymm31731; AVX2-FCP-NEXT: vmovdqa 176(%rdi), %xmm51732; AVX2-FCP-NEXT: vmovdqa 160(%rdi), %xmm61733; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm9 = xmm6[0,1],xmm5[2],xmm6[3,4],xmm5[5],xmm6[6,7]1734; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm10 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]1735; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm9, %xmm91736; AVX2-FCP-NEXT: vinserti128 $1, %xmm9, %ymm0, %ymm91737; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm9 = ymm3[0,1,2],ymm9[3,4,5,6,7],ymm3[8,9,10],ymm9[11,12,13,14,15]1738; AVX2-FCP-NEXT: vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,4,7]1739; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm9[4,5,6,7]1740; AVX2-FCP-NEXT: vpblendvb %ymm7, %ymm0, %ymm1, %ymm71741; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm9 = ymm7[2,3,0,1]1742; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm7 = ymm7[0],ymm9[1],ymm7[2,3],ymm9[4],ymm7[5,6],ymm9[7],ymm7[8],ymm9[9],ymm7[10,11],ymm9[12],ymm7[13,14],ymm9[15]1743; AVX2-FCP-NEXT: vpshufb %ymm8, %ymm7, %ymm91744; AVX2-FCP-NEXT: vmovdqa 80(%rdi), %xmm71745; AVX2-FCP-NEXT: vmovdqa 64(%rdi), %xmm81746; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm11 = xmm8[0,1],xmm7[2],xmm8[3,4],xmm7[5],xmm8[6,7]1747; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm11, %xmm101748; AVX2-FCP-NEXT: vinserti128 $1, %xmm10, %ymm0, %ymm101749; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm10 = ymm9[0,1,2],ymm10[3,4,5,6,7],ymm9[8,9,10],ymm10[11,12,13,14,15]1750; AVX2-FCP-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,6,5,4,7]1751; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm10[4,5,6,7]1752; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm11 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]1753; AVX2-FCP-NEXT: vpblendvb %ymm11, %ymm4, %ymm2, %ymm101754; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm12 = ymm10[2,3,0,1]1755; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm12[2],ymm10[3,4],ymm12[5],ymm10[6,7,8,9],ymm12[10],ymm10[11,12],ymm12[13],ymm10[14,15]1756; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm12 = [2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]1757; AVX2-FCP-NEXT: vpshufb %ymm12, %ymm10, %ymm101758; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm13 = xmm5[0,1],xmm6[2],xmm5[3,4],xmm6[5],xmm5[6,7]1759; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm14 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]1760; AVX2-FCP-NEXT: vpshufb %xmm14, %xmm13, %xmm131761; AVX2-FCP-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm131762; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm13 = ymm10[0,1,2],ymm13[3,4,5,6,7],ymm10[8,9,10],ymm13[11,12,13,14,15]1763; AVX2-FCP-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,6,7,4]1764; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm13[4,5,6,7]1765; AVX2-FCP-NEXT: vpblendvb %ymm11, %ymm1, %ymm0, %ymm111766; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm13 = ymm11[2,3,0,1]1767; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm11 = ymm11[0,1],ymm13[2],ymm11[3,4],ymm13[5],ymm11[6,7,8,9],ymm13[10],ymm11[11,12],ymm13[13],ymm11[14,15]1768; AVX2-FCP-NEXT: vpshufb %ymm12, %ymm11, %ymm111769; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm7[0,1],xmm8[2],xmm7[3,4],xmm8[5],xmm7[6,7]1770; AVX2-FCP-NEXT: vpshufb %xmm14, %xmm12, %xmm121771; AVX2-FCP-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm121772; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm11[0,1,2],ymm12[3,4,5,6,7],ymm11[8,9,10],ymm12[11,12,13,14,15]1773; AVX2-FCP-NEXT: vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,5,6,7,4]1774; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm12[4,5,6,7]1775; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm12 = [0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0]1776; AVX2-FCP-NEXT: vpblendvb %ymm12, %ymm4, %ymm2, %ymm21777; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm2[2,3,0,1]1778; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm4[0],ymm2[1,2],ymm4[3],ymm2[4,5],ymm4[6],ymm2[7],ymm4[8],ymm2[9,10],ymm4[11],ymm2[12,13],ymm4[14],ymm2[15]1779; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]1780; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm2, %ymm21781; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3],xmm5[4],xmm6[5,6],xmm5[7]1782; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]1783; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm5, %xmm51784; AVX2-FCP-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm51785; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm5[5,6,7]1786; AVX2-FCP-NEXT: vpblendvb %ymm12, %ymm1, %ymm0, %ymm01787; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]1788; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7],ymm1[8],ymm0[9,10],ymm1[11],ymm0[12,13],ymm1[14],ymm0[15]1789; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm0, %ymm01790; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm8[0],xmm7[1],xmm8[2,3],xmm7[4],xmm8[5,6],xmm7[7]1791; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm1, %xmm11792; AVX2-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm11793; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]1794; AVX2-FCP-NEXT: vmovdqa %ymm3, 32(%rsi)1795; AVX2-FCP-NEXT: vmovdqa %ymm9, (%rsi)1796; AVX2-FCP-NEXT: vmovdqa %ymm10, 32(%rdx)1797; AVX2-FCP-NEXT: vmovdqa %ymm11, (%rdx)1798; AVX2-FCP-NEXT: vmovdqa %ymm2, 32(%rcx)1799; AVX2-FCP-NEXT: vmovdqa %ymm0, (%rcx)1800; AVX2-FCP-NEXT: vzeroupper1801; AVX2-FCP-NEXT: retq1802;1803; AVX512-LABEL: load_i16_stride3_vf32:1804; AVX512: # %bb.0:1805; AVX512-NEXT: vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]1806; AVX512-NEXT: vmovdqa 128(%rdi), %ymm51807; AVX512-NEXT: vmovdqa 160(%rdi), %ymm61808; AVX512-NEXT: vmovdqa %ymm0, %ymm11809; AVX512-NEXT: vpternlogq {{.*#+}} ymm1 = ymm5 ^ (ymm1 & (ymm6 ^ ymm5))1810; AVX512-NEXT: vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]1811; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]1812; AVX512-NEXT: vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]1813; AVX512-NEXT: vmovdqa 112(%rdi), %xmm11814; AVX512-NEXT: vmovdqa 96(%rdi), %xmm21815; AVX512-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]1816; AVX512-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]1817; AVX512-NEXT: vpblendd {{.*#+}} ymm7 = ymm4[0,1,2],ymm3[3,4,5,6,7]1818; AVX512-NEXT: vmovdqa (%rdi), %ymm81819; AVX512-NEXT: vmovdqa 32(%rdi), %ymm91820; AVX512-NEXT: vmovdqa %ymm0, %ymm31821; AVX512-NEXT: vpternlogq {{.*#+}} ymm3 = ymm9 ^ (ymm3 & (ymm8 ^ ymm9))1822; AVX512-NEXT: vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]1823; AVX512-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7],ymm3[8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13,14],ymm4[15]1824; AVX512-NEXT: vpshufb {{.*#+}} ymm10 = ymm3[0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]1825; AVX512-NEXT: vmovdqa 80(%rdi), %xmm31826; AVX512-NEXT: vmovdqa 64(%rdi), %xmm41827; AVX512-NEXT: vpblendw {{.*#+}} xmm11 = xmm4[0,1],xmm3[2],xmm4[3,4],xmm3[5],xmm4[6,7]1828; AVX512-NEXT: vpshufb {{.*#+}} xmm11 = xmm11[u,u,u,u,u,u,2,3,8,9,14,15,4,5,10,11]1829; AVX512-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm111830; AVX512-NEXT: vpblendw {{.*#+}} ymm11 = ymm10[0,1,2],ymm11[3,4,5,6,7],ymm10[8,9,10],ymm11[11,12,13,14,15]1831; AVX512-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,5,4,7]1832; AVX512-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]1833; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm10, %zmm71834; AVX512-NEXT: vmovdqa %ymm0, %ymm101835; AVX512-NEXT: vpternlogq {{.*#+}} ymm10 = ymm6 ^ (ymm10 & (ymm5 ^ ymm6))1836; AVX512-NEXT: vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]1837; AVX512-NEXT: vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm11[2],ymm10[3,4],ymm11[5],ymm10[6,7,8,9],ymm11[10],ymm10[11,12],ymm11[13],ymm10[14,15]1838; AVX512-NEXT: vmovdqa {{.*#+}} ymm11 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]1839; AVX512-NEXT: vpshufb %ymm11, %ymm10, %ymm101840; AVX512-NEXT: vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]1841; AVX512-NEXT: vpshufb %xmm11, %xmm12, %xmm121842; AVX512-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1,2,3,4],xmm10[5,6,7]1843; AVX512-NEXT: vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]1844; AVX512-NEXT: vmovdqa {{.*#+}} ymm12 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]1845; AVX512-NEXT: vmovdqa %ymm12, %ymm131846; AVX512-NEXT: vpternlogq {{.*#+}} ymm13 = ymm8 ^ (ymm13 & (ymm9 ^ ymm8))1847; AVX512-NEXT: vpermq {{.*#+}} ymm14 = ymm13[2,3,0,1]1848; AVX512-NEXT: vpblendw {{.*#+}} ymm13 = ymm13[0,1],ymm14[2],ymm13[3,4],ymm14[5],ymm13[6,7,8,9],ymm14[10],ymm13[11,12],ymm14[13],ymm13[14,15]1849; AVX512-NEXT: vpshufb {{.*#+}} ymm13 = ymm13[2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]1850; AVX512-NEXT: vpblendw {{.*#+}} xmm14 = xmm3[0,1],xmm4[2],xmm3[3,4],xmm4[5],xmm3[6,7]1851; AVX512-NEXT: vpshufb %xmm11, %xmm14, %xmm111852; AVX512-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm111853; AVX512-NEXT: vpblendw {{.*#+}} ymm11 = ymm13[0,1,2],ymm11[3,4,5,6,7],ymm13[8,9,10],ymm11[11,12,13,14,15]1854; AVX512-NEXT: vpshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,5,6,7,4]1855; AVX512-NEXT: vpblendd {{.*#+}} ymm11 = ymm13[0,1,2,3],ymm11[4,5,6,7]1856; AVX512-NEXT: vinserti64x4 $1, %ymm10, %zmm11, %zmm101857; AVX512-NEXT: vpternlogq {{.*#+}} ymm12 = ymm5 ^ (ymm12 & (ymm6 ^ ymm5))1858; AVX512-NEXT: vpermq {{.*#+}} ymm5 = ymm12[2,3,0,1]1859; AVX512-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1,2],ymm5[3],ymm12[4,5],ymm5[6],ymm12[7],ymm5[8],ymm12[9,10],ymm5[11],ymm12[12,13],ymm5[14],ymm12[15]1860; AVX512-NEXT: vmovdqa {{.*#+}} ymm6 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]1861; AVX512-NEXT: vpshufb %ymm6, %ymm5, %ymm51862; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]1863; AVX512-NEXT: vpshufb %xmm6, %xmm1, %xmm11864; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm5[5,6,7]1865; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm5[4,5,6,7]1866; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm8 ^ (ymm0 & (ymm9 ^ ymm8))1867; AVX512-NEXT: vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]1868; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7],ymm2[8],ymm0[9,10],ymm2[11],ymm0[12,13],ymm2[14],ymm0[15]1869; AVX512-NEXT: vpshufb %ymm6, %ymm0, %ymm01870; AVX512-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]1871; AVX512-NEXT: vpshufb %xmm6, %xmm2, %xmm21872; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm21873; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]1874; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm01875; AVX512-NEXT: vmovdqa64 %zmm7, (%rsi)1876; AVX512-NEXT: vmovdqa64 %zmm10, (%rdx)1877; AVX512-NEXT: vmovdqa64 %zmm0, (%rcx)1878; AVX512-NEXT: vzeroupper1879; AVX512-NEXT: retq1880;1881; AVX512-FCP-LABEL: load_i16_stride3_vf32:1882; AVX512-FCP: # %bb.0:1883; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]1884; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %ymm51885; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %ymm61886; AVX512-FCP-NEXT: vmovdqa %ymm0, %ymm11887; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm5 ^ (ymm1 & (ymm6 ^ ymm5))1888; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]1889; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]1890; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]1891; AVX512-FCP-NEXT: vmovdqa 112(%rdi), %xmm11892; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %xmm21893; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]1894; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]1895; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm4[0,1,2],ymm3[3,4,5,6,7]1896; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm81897; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm91898; AVX512-FCP-NEXT: vmovdqa %ymm0, %ymm31899; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm3 = ymm9 ^ (ymm3 & (ymm8 ^ ymm9))1900; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]1901; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7],ymm3[8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13,14],ymm4[15]1902; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm10 = ymm3[0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]1903; AVX512-FCP-NEXT: vmovdqa 80(%rdi), %xmm31904; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %xmm41905; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm11 = xmm4[0,1],xmm3[2],xmm4[3,4],xmm3[5],xmm4[6,7]1906; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm11 = xmm11[u,u,u,u,u,u,2,3,8,9,14,15,4,5,10,11]1907; AVX512-FCP-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm111908; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm11 = ymm10[0,1,2],ymm11[3,4,5,6,7],ymm10[8,9,10],ymm11[11,12,13,14,15]1909; AVX512-FCP-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,5,4,7]1910; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]1911; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm10, %zmm71912; AVX512-FCP-NEXT: vmovdqa %ymm0, %ymm101913; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm10 = ymm6 ^ (ymm10 & (ymm5 ^ ymm6))1914; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]1915; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm11[2],ymm10[3,4],ymm11[5],ymm10[6,7,8,9],ymm11[10],ymm10[11,12],ymm11[13],ymm10[14,15]1916; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm11 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]1917; AVX512-FCP-NEXT: vpshufb %ymm11, %ymm10, %ymm101918; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]1919; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm12, %xmm121920; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1,2,3,4],xmm10[5,6,7]1921; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]1922; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm12 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]1923; AVX512-FCP-NEXT: vmovdqa %ymm12, %ymm131924; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm13 = ymm8 ^ (ymm13 & (ymm9 ^ ymm8))1925; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm14 = ymm13[2,3,0,1]1926; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm13 = ymm13[0,1],ymm14[2],ymm13[3,4],ymm14[5],ymm13[6,7,8,9],ymm14[10],ymm13[11,12],ymm14[13],ymm13[14,15]1927; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm13 = ymm13[2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]1928; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm14 = xmm3[0,1],xmm4[2],xmm3[3,4],xmm4[5],xmm3[6,7]1929; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm14, %xmm111930; AVX512-FCP-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm111931; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm11 = ymm13[0,1,2],ymm11[3,4,5,6,7],ymm13[8,9,10],ymm11[11,12,13,14,15]1932; AVX512-FCP-NEXT: vpshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,5,6,7,4]1933; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm13[0,1,2,3],ymm11[4,5,6,7]1934; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm10, %zmm11, %zmm101935; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm12 = ymm5 ^ (ymm12 & (ymm6 ^ ymm5))1936; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm5 = ymm12[2,3,0,1]1937; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1,2],ymm5[3],ymm12[4,5],ymm5[6],ymm12[7],ymm5[8],ymm12[9,10],ymm5[11],ymm12[12,13],ymm5[14],ymm12[15]1938; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]1939; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm5, %ymm51940; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]1941; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm1, %xmm11942; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm5[5,6,7]1943; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm5[4,5,6,7]1944; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm8 ^ (ymm0 & (ymm9 ^ ymm8))1945; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]1946; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7],ymm2[8],ymm0[9,10],ymm2[11],ymm0[12,13],ymm2[14],ymm0[15]1947; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm01948; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]1949; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm2, %xmm21950; AVX512-FCP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm21951; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]1952; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm01953; AVX512-FCP-NEXT: vmovdqa64 %zmm7, (%rsi)1954; AVX512-FCP-NEXT: vmovdqa64 %zmm10, (%rdx)1955; AVX512-FCP-NEXT: vmovdqa64 %zmm0, (%rcx)1956; AVX512-FCP-NEXT: vzeroupper1957; AVX512-FCP-NEXT: retq1958;1959; AVX512DQ-LABEL: load_i16_stride3_vf32:1960; AVX512DQ: # %bb.0:1961; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]1962; AVX512DQ-NEXT: vmovdqa 128(%rdi), %ymm51963; AVX512DQ-NEXT: vmovdqa 160(%rdi), %ymm61964; AVX512DQ-NEXT: vmovdqa %ymm0, %ymm11965; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm1 = ymm5 ^ (ymm1 & (ymm6 ^ ymm5))1966; AVX512DQ-NEXT: vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]1967; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]1968; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]1969; AVX512DQ-NEXT: vmovdqa 112(%rdi), %xmm11970; AVX512DQ-NEXT: vmovdqa 96(%rdi), %xmm21971; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]1972; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]1973; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm7 = ymm4[0,1,2],ymm3[3,4,5,6,7]1974; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm81975; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm91976; AVX512DQ-NEXT: vmovdqa %ymm0, %ymm31977; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm3 = ymm9 ^ (ymm3 & (ymm8 ^ ymm9))1978; AVX512DQ-NEXT: vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]1979; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7],ymm3[8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13,14],ymm4[15]1980; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm10 = ymm3[0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]1981; AVX512DQ-NEXT: vmovdqa 80(%rdi), %xmm31982; AVX512DQ-NEXT: vmovdqa 64(%rdi), %xmm41983; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm11 = xmm4[0,1],xmm3[2],xmm4[3,4],xmm3[5],xmm4[6,7]1984; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm11 = xmm11[u,u,u,u,u,u,2,3,8,9,14,15,4,5,10,11]1985; AVX512DQ-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm111986; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm11 = ymm10[0,1,2],ymm11[3,4,5,6,7],ymm10[8,9,10],ymm11[11,12,13,14,15]1987; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,5,4,7]1988; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]1989; AVX512DQ-NEXT: vinserti64x4 $1, %ymm7, %zmm10, %zmm71990; AVX512DQ-NEXT: vmovdqa %ymm0, %ymm101991; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm10 = ymm6 ^ (ymm10 & (ymm5 ^ ymm6))1992; AVX512DQ-NEXT: vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]1993; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm11[2],ymm10[3,4],ymm11[5],ymm10[6,7,8,9],ymm11[10],ymm10[11,12],ymm11[13],ymm10[14,15]1994; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm11 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]1995; AVX512DQ-NEXT: vpshufb %ymm11, %ymm10, %ymm101996; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]1997; AVX512DQ-NEXT: vpshufb %xmm11, %xmm12, %xmm121998; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1,2,3,4],xmm10[5,6,7]1999; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]2000; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm12 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]2001; AVX512DQ-NEXT: vmovdqa %ymm12, %ymm132002; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm13 = ymm8 ^ (ymm13 & (ymm9 ^ ymm8))2003; AVX512DQ-NEXT: vpermq {{.*#+}} ymm14 = ymm13[2,3,0,1]2004; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm13 = ymm13[0,1],ymm14[2],ymm13[3,4],ymm14[5],ymm13[6,7,8,9],ymm14[10],ymm13[11,12],ymm14[13],ymm13[14,15]2005; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm13 = ymm13[2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]2006; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm14 = xmm3[0,1],xmm4[2],xmm3[3,4],xmm4[5],xmm3[6,7]2007; AVX512DQ-NEXT: vpshufb %xmm11, %xmm14, %xmm112008; AVX512DQ-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm112009; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm11 = ymm13[0,1,2],ymm11[3,4,5,6,7],ymm13[8,9,10],ymm11[11,12,13,14,15]2010; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,5,6,7,4]2011; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm11 = ymm13[0,1,2,3],ymm11[4,5,6,7]2012; AVX512DQ-NEXT: vinserti64x4 $1, %ymm10, %zmm11, %zmm102013; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm12 = ymm5 ^ (ymm12 & (ymm6 ^ ymm5))2014; AVX512DQ-NEXT: vpermq {{.*#+}} ymm5 = ymm12[2,3,0,1]2015; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1,2],ymm5[3],ymm12[4,5],ymm5[6],ymm12[7],ymm5[8],ymm12[9,10],ymm5[11],ymm12[12,13],ymm5[14],ymm12[15]2016; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm6 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]2017; AVX512DQ-NEXT: vpshufb %ymm6, %ymm5, %ymm52018; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]2019; AVX512DQ-NEXT: vpshufb %xmm6, %xmm1, %xmm12020; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm5[5,6,7]2021; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm5[4,5,6,7]2022; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm0 = ymm8 ^ (ymm0 & (ymm9 ^ ymm8))2023; AVX512DQ-NEXT: vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]2024; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7],ymm2[8],ymm0[9,10],ymm2[11],ymm0[12,13],ymm2[14],ymm0[15]2025; AVX512DQ-NEXT: vpshufb %ymm6, %ymm0, %ymm02026; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]2027; AVX512DQ-NEXT: vpshufb %xmm6, %xmm2, %xmm22028; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm22029; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]2030; AVX512DQ-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm02031; AVX512DQ-NEXT: vmovdqa64 %zmm7, (%rsi)2032; AVX512DQ-NEXT: vmovdqa64 %zmm10, (%rdx)2033; AVX512DQ-NEXT: vmovdqa64 %zmm0, (%rcx)2034; AVX512DQ-NEXT: vzeroupper2035; AVX512DQ-NEXT: retq2036;2037; AVX512DQ-FCP-LABEL: load_i16_stride3_vf32:2038; AVX512DQ-FCP: # %bb.0:2039; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]2040; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %ymm52041; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %ymm62042; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, %ymm12043; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm5 ^ (ymm1 & (ymm6 ^ ymm5))2044; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]2045; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]2046; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]2047; AVX512DQ-FCP-NEXT: vmovdqa 112(%rdi), %xmm12048; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %xmm22049; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]2050; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]2051; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm4[0,1,2],ymm3[3,4,5,6,7]2052; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm82053; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm92054; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, %ymm32055; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm3 = ymm9 ^ (ymm3 & (ymm8 ^ ymm9))2056; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]2057; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7],ymm3[8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13,14],ymm4[15]2058; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm10 = ymm3[0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]2059; AVX512DQ-FCP-NEXT: vmovdqa 80(%rdi), %xmm32060; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %xmm42061; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm11 = xmm4[0,1],xmm3[2],xmm4[3,4],xmm3[5],xmm4[6,7]2062; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm11 = xmm11[u,u,u,u,u,u,2,3,8,9,14,15,4,5,10,11]2063; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm112064; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm11 = ymm10[0,1,2],ymm11[3,4,5,6,7],ymm10[8,9,10],ymm11[11,12,13,14,15]2065; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,5,4,7]2066; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]2067; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm10, %zmm72068; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, %ymm102069; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm10 = ymm6 ^ (ymm10 & (ymm5 ^ ymm6))2070; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]2071; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm11[2],ymm10[3,4],ymm11[5],ymm10[6,7,8,9],ymm11[10],ymm10[11,12],ymm11[13],ymm10[14,15]2072; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm11 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]2073; AVX512DQ-FCP-NEXT: vpshufb %ymm11, %ymm10, %ymm102074; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]2075; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm12, %xmm122076; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1,2,3,4],xmm10[5,6,7]2077; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]2078; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm12 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]2079; AVX512DQ-FCP-NEXT: vmovdqa %ymm12, %ymm132080; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm13 = ymm8 ^ (ymm13 & (ymm9 ^ ymm8))2081; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm14 = ymm13[2,3,0,1]2082; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm13 = ymm13[0,1],ymm14[2],ymm13[3,4],ymm14[5],ymm13[6,7,8,9],ymm14[10],ymm13[11,12],ymm14[13],ymm13[14,15]2083; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm13 = ymm13[2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]2084; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm14 = xmm3[0,1],xmm4[2],xmm3[3,4],xmm4[5],xmm3[6,7]2085; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm14, %xmm112086; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm11, %ymm0, %ymm112087; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm11 = ymm13[0,1,2],ymm11[3,4,5,6,7],ymm13[8,9,10],ymm11[11,12,13,14,15]2088; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,5,6,7,4]2089; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm13[0,1,2,3],ymm11[4,5,6,7]2090; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm10, %zmm11, %zmm102091; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm12 = ymm5 ^ (ymm12 & (ymm6 ^ ymm5))2092; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm5 = ymm12[2,3,0,1]2093; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1,2],ymm5[3],ymm12[4,5],ymm5[6],ymm12[7],ymm5[8],ymm12[9,10],ymm5[11],ymm12[12,13],ymm5[14],ymm12[15]2094; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]2095; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm5, %ymm52096; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]2097; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm1, %xmm12098; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm5[5,6,7]2099; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm5[4,5,6,7]2100; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm8 ^ (ymm0 & (ymm9 ^ ymm8))2101; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]2102; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7],ymm2[8],ymm0[9,10],ymm2[11],ymm0[12,13],ymm2[14],ymm0[15]2103; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm02104; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]2105; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm2, %xmm22106; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm22107; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]2108; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm02109; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm7, (%rsi)2110; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm10, (%rdx)2111; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, (%rcx)2112; AVX512DQ-FCP-NEXT: vzeroupper2113; AVX512DQ-FCP-NEXT: retq2114;2115; AVX512BW-LABEL: load_i16_stride3_vf32:2116; AVX512BW: # %bb.0:2117; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm02118; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm12119; AVX512BW-NEXT: vmovdqa64 128(%rdi), %zmm22120; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,0,0,0,0,0,0,0,0,0,0]2121; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm32122; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm4 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,34,37,40,43,46,49,52,55,58,61]2123; AVX512BW-NEXT: vpermi2w %zmm2, %zmm3, %zmm42124; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [1,4,7,10,13,16,19,22,25,28,31,34,37,40,43,46,49,52,55,58,61,0,0,0,0,0,0,0,0,0,0,0]2125; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm32126; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm5 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,32,35,38,41,44,47,50,53,56,59,62]2127; AVX512BW-NEXT: vpermi2w %zmm2, %zmm3, %zmm52128; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [34,37,40,43,46,49,52,55,58,61,0,3,6,9,12,15,18,21,24,27,30,0,0,0,0,0,0,0,0,0,0,0]2129; AVX512BW-NEXT: vpermi2w %zmm0, %zmm1, %zmm32130; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm0 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,33,36,39,42,45,48,51,54,57,60,63]2131; AVX512BW-NEXT: vpermi2w %zmm2, %zmm3, %zmm02132; AVX512BW-NEXT: vmovdqa64 %zmm4, (%rsi)2133; AVX512BW-NEXT: vmovdqa64 %zmm5, (%rdx)2134; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rcx)2135; AVX512BW-NEXT: vzeroupper2136; AVX512BW-NEXT: retq2137;2138; AVX512BW-FCP-LABEL: load_i16_stride3_vf32:2139; AVX512BW-FCP: # %bb.0:2140; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm02141; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm12142; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %zmm22143; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,0,0,0,0,0,0,0,0,0,0]2144; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm32145; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm4 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,34,37,40,43,46,49,52,55,58,61]2146; AVX512BW-FCP-NEXT: vpermi2w %zmm2, %zmm3, %zmm42147; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [1,4,7,10,13,16,19,22,25,28,31,34,37,40,43,46,49,52,55,58,61,0,0,0,0,0,0,0,0,0,0,0]2148; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm32149; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm5 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,32,35,38,41,44,47,50,53,56,59,62]2150; AVX512BW-FCP-NEXT: vpermi2w %zmm2, %zmm3, %zmm52151; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [34,37,40,43,46,49,52,55,58,61,0,3,6,9,12,15,18,21,24,27,30,0,0,0,0,0,0,0,0,0,0,0]2152; AVX512BW-FCP-NEXT: vpermi2w %zmm0, %zmm1, %zmm32153; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm0 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,33,36,39,42,45,48,51,54,57,60,63]2154; AVX512BW-FCP-NEXT: vpermi2w %zmm2, %zmm3, %zmm02155; AVX512BW-FCP-NEXT: vmovdqa64 %zmm4, (%rsi)2156; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, (%rdx)2157; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, (%rcx)2158; AVX512BW-FCP-NEXT: vzeroupper2159; AVX512BW-FCP-NEXT: retq2160;2161; AVX512DQ-BW-LABEL: load_i16_stride3_vf32:2162; AVX512DQ-BW: # %bb.0:2163; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm02164; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %zmm12165; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %zmm22166; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,0,0,0,0,0,0,0,0,0,0]2167; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm32168; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm4 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,34,37,40,43,46,49,52,55,58,61]2169; AVX512DQ-BW-NEXT: vpermi2w %zmm2, %zmm3, %zmm42170; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [1,4,7,10,13,16,19,22,25,28,31,34,37,40,43,46,49,52,55,58,61,0,0,0,0,0,0,0,0,0,0,0]2171; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm32172; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm5 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,32,35,38,41,44,47,50,53,56,59,62]2173; AVX512DQ-BW-NEXT: vpermi2w %zmm2, %zmm3, %zmm52174; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [34,37,40,43,46,49,52,55,58,61,0,3,6,9,12,15,18,21,24,27,30,0,0,0,0,0,0,0,0,0,0,0]2175; AVX512DQ-BW-NEXT: vpermi2w %zmm0, %zmm1, %zmm32176; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm0 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,33,36,39,42,45,48,51,54,57,60,63]2177; AVX512DQ-BW-NEXT: vpermi2w %zmm2, %zmm3, %zmm02178; AVX512DQ-BW-NEXT: vmovdqa64 %zmm4, (%rsi)2179; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, (%rdx)2180; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, (%rcx)2181; AVX512DQ-BW-NEXT: vzeroupper2182; AVX512DQ-BW-NEXT: retq2183;2184; AVX512DQ-BW-FCP-LABEL: load_i16_stride3_vf32:2185; AVX512DQ-BW-FCP: # %bb.0:2186; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm02187; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm12188; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %zmm22189; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,0,0,0,0,0,0,0,0,0,0]2190; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm32191; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm4 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,34,37,40,43,46,49,52,55,58,61]2192; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm2, %zmm3, %zmm42193; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [1,4,7,10,13,16,19,22,25,28,31,34,37,40,43,46,49,52,55,58,61,0,0,0,0,0,0,0,0,0,0,0]2194; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm32195; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm5 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,32,35,38,41,44,47,50,53,56,59,62]2196; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm2, %zmm3, %zmm52197; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [34,37,40,43,46,49,52,55,58,61,0,3,6,9,12,15,18,21,24,27,30,0,0,0,0,0,0,0,0,0,0,0]2198; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm0, %zmm1, %zmm32199; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm0 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,33,36,39,42,45,48,51,54,57,60,63]2200; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm2, %zmm3, %zmm02201; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm4, (%rsi)2202; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, (%rdx)2203; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, (%rcx)2204; AVX512DQ-BW-FCP-NEXT: vzeroupper2205; AVX512DQ-BW-FCP-NEXT: retq2206 %wide.vec = load <96 x i16>, ptr %in.vec, align 642207 %strided.vec0 = shufflevector <96 x i16> %wide.vec, <96 x i16> poison, <32 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45, i32 48, i32 51, i32 54, i32 57, i32 60, i32 63, i32 66, i32 69, i32 72, i32 75, i32 78, i32 81, i32 84, i32 87, i32 90, i32 93>2208 %strided.vec1 = shufflevector <96 x i16> %wide.vec, <96 x i16> poison, <32 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46, i32 49, i32 52, i32 55, i32 58, i32 61, i32 64, i32 67, i32 70, i32 73, i32 76, i32 79, i32 82, i32 85, i32 88, i32 91, i32 94>2209 %strided.vec2 = shufflevector <96 x i16> %wide.vec, <96 x i16> poison, <32 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47, i32 50, i32 53, i32 56, i32 59, i32 62, i32 65, i32 68, i32 71, i32 74, i32 77, i32 80, i32 83, i32 86, i32 89, i32 92, i32 95>2210 store <32 x i16> %strided.vec0, ptr %out.vec0, align 642211 store <32 x i16> %strided.vec1, ptr %out.vec1, align 642212 store <32 x i16> %strided.vec2, ptr %out.vec2, align 642213 ret void2214}2215 2216define void @load_i16_stride3_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2) nounwind {2217; SSE-LABEL: load_i16_stride3_vf64:2218; SSE: # %bb.0:2219; SSE-NEXT: subq $440, %rsp # imm = 0x1B82220; SSE-NEXT: movdqa 192(%rdi), %xmm142221; SSE-NEXT: movdqa 272(%rdi), %xmm62222; SSE-NEXT: movdqa 240(%rdi), %xmm52223; SSE-NEXT: movdqa 256(%rdi), %xmm72224; SSE-NEXT: movdqa 80(%rdi), %xmm102225; SSE-NEXT: movdqa (%rdi), %xmm152226; SSE-NEXT: movdqa 16(%rdi), %xmm92227; SSE-NEXT: movdqa 32(%rdi), %xmm82228; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2229; SSE-NEXT: movdqa 48(%rdi), %xmm02230; SSE-NEXT: movdqa 64(%rdi), %xmm112231; SSE-NEXT: movdqa {{.*#+}} xmm1 = [65535,0,65535,65535,0,65535,65535,0]2232; SSE-NEXT: movdqa %xmm1, %xmm22233; SSE-NEXT: pandn %xmm11, %xmm22234; SSE-NEXT: movdqa %xmm0, %xmm32235; SSE-NEXT: pand %xmm1, %xmm32236; SSE-NEXT: por %xmm2, %xmm32237; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,2,1,3]2238; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,6,7]2239; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3]2240; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm2[0,3,2,1,4,5,6,7]2241; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm11[0,1,2,3,4,7,6,7]2242; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2243; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm10[0,1,2,1]2244; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2245; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,6,5]2246; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,0],xmm2[2,0]2247; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,0]2248; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2249; SSE-NEXT: movdqa %xmm1, %xmm22250; SSE-NEXT: pandn %xmm7, %xmm22251; SSE-NEXT: movdqa %xmm5, %xmm32252; SSE-NEXT: pand %xmm1, %xmm32253; SSE-NEXT: por %xmm2, %xmm32254; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,2,1,3]2255; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,6,7]2256; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3]2257; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm2[0,3,2,1,4,5,6,7]2258; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm7[0,1,2,3,4,7,6,7]2259; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2260; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm6[0,1,2,1]2261; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2262; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,6,5]2263; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,0],xmm2[2,0]2264; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,0]2265; SSE-NEXT: movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2266; SSE-NEXT: movdqa %xmm1, %xmm22267; SSE-NEXT: pandn %xmm9, %xmm22268; SSE-NEXT: movdqa %xmm15, %xmm32269; SSE-NEXT: pand %xmm1, %xmm32270; SSE-NEXT: por %xmm2, %xmm32271; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm9[0,1,2,3,4,7,6,7]2272; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2273; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm8[0,1,2,1]2274; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6,5]2275; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[3,0],xmm2[2,0]2276; SSE-NEXT: movdqa 208(%rdi), %xmm82277; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,2,1,3]2278; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,6,7]2279; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3]2280; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,3,2,1,4,5,6,7]2281; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,0]2282; SSE-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2283; SSE-NEXT: movdqa %xmm1, %xmm22284; SSE-NEXT: pandn %xmm8, %xmm22285; SSE-NEXT: movdqa %xmm8, (%rsp) # 16-byte Spill2286; SSE-NEXT: movdqa %xmm14, %xmm32287; SSE-NEXT: pand %xmm1, %xmm32288; SSE-NEXT: por %xmm2, %xmm32289; SSE-NEXT: movdqa 224(%rdi), %xmm22290; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2291; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,1]2292; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,6,5]2293; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm8[0,1,2,3,4,7,6,7]2294; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,0],xmm4[2,0]2295; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,1,3]2296; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,6,7]2297; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,1,3]2298; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,3,2,1,4,5,6,7]2299; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,0]2300; SSE-NEXT: movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2301; SSE-NEXT: movdqa 160(%rdi), %xmm42302; SSE-NEXT: movdqa %xmm1, %xmm22303; SSE-NEXT: pandn %xmm4, %xmm22304; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2305; SSE-NEXT: movdqa 144(%rdi), %xmm32306; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2307; SSE-NEXT: pand %xmm1, %xmm32308; SSE-NEXT: por %xmm2, %xmm32309; SSE-NEXT: movdqa 176(%rdi), %xmm22310; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2311; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,1]2312; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,6,5]2313; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,7,6,7]2314; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,0],xmm4[2,0]2315; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,1,3]2316; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,6,7]2317; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,1,3]2318; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,3,2,1,4,5,6,7]2319; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,0]2320; SSE-NEXT: movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2321; SSE-NEXT: movdqa 352(%rdi), %xmm42322; SSE-NEXT: movdqa %xmm1, %xmm22323; SSE-NEXT: pandn %xmm4, %xmm22324; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2325; SSE-NEXT: movdqa 336(%rdi), %xmm32326; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2327; SSE-NEXT: pand %xmm1, %xmm32328; SSE-NEXT: por %xmm2, %xmm32329; SSE-NEXT: movdqa 368(%rdi), %xmm22330; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2331; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,1]2332; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,6,5]2333; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,7,6,7]2334; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,0],xmm4[2,0]2335; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,1,3]2336; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,6,7]2337; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,1,3]2338; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,3,2,1,4,5,6,7]2339; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,0]2340; SSE-NEXT: movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2341; SSE-NEXT: movdqa 112(%rdi), %xmm42342; SSE-NEXT: movdqa %xmm1, %xmm22343; SSE-NEXT: pandn %xmm4, %xmm22344; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2345; SSE-NEXT: movdqa 96(%rdi), %xmm32346; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2347; SSE-NEXT: pand %xmm1, %xmm32348; SSE-NEXT: por %xmm2, %xmm32349; SSE-NEXT: movdqa 128(%rdi), %xmm22350; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2351; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,1]2352; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,6,5]2353; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,7,6,7]2354; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,0],xmm4[2,0]2355; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,1,3]2356; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,6,7]2357; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,1,3]2358; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,3,2,1,4,5,6,7]2359; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,0]2360; SSE-NEXT: movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2361; SSE-NEXT: movdqa 288(%rdi), %xmm22362; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2363; SSE-NEXT: pand %xmm1, %xmm22364; SSE-NEXT: movdqa 304(%rdi), %xmm32365; SSE-NEXT: pandn %xmm3, %xmm12366; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2367; SSE-NEXT: por %xmm2, %xmm12368; SSE-NEXT: movdqa 320(%rdi), %xmm22369; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2370; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,1]2371; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,6,5]2372; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,7]2373; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,0],xmm3[2,0]2374; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]2375; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7]2376; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]2377; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,1,4,5,6,7]2378; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,0]2379; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2380; SSE-NEXT: movdqa {{.*#+}} xmm12 = [65535,65535,0,65535,65535,0,65535,65535]2381; SSE-NEXT: movdqa %xmm12, %xmm12382; SSE-NEXT: pandn %xmm0, %xmm12383; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2384; SSE-NEXT: movdqa %xmm0, %xmm22385; SSE-NEXT: movdqa %xmm12, %xmm02386; SSE-NEXT: pandn %xmm11, %xmm02387; SSE-NEXT: pand %xmm12, %xmm22388; SSE-NEXT: por %xmm0, %xmm22389; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm10[0,3,2,3,4,5,6,7]2390; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]2391; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm0[0,1,2,3,4,4,5,6]2392; SSE-NEXT: movdqa {{.*#+}} xmm13 = [65535,65535,65535,65535,65535,0,0,0]2393; SSE-NEXT: movdqa %xmm13, %xmm02394; SSE-NEXT: pandn %xmm3, %xmm02395; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,2,3,4,5,6,7]2396; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7]2397; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]2398; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,3,0,4,5,6,7]2399; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5]2400; SSE-NEXT: pand %xmm13, %xmm22401; SSE-NEXT: por %xmm2, %xmm02402; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2403; SSE-NEXT: movdqa %xmm12, %xmm02404; SSE-NEXT: pandn %xmm5, %xmm02405; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2406; SSE-NEXT: movdqa %xmm12, %xmm32407; SSE-NEXT: pandn %xmm7, %xmm32408; SSE-NEXT: pand %xmm12, %xmm52409; SSE-NEXT: por %xmm3, %xmm52410; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm6[0,3,2,3,4,5,6,7]2411; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,3]2412; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,5,6]2413; SSE-NEXT: movdqa %xmm13, %xmm02414; SSE-NEXT: pandn %xmm3, %xmm02415; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm5[2,1,2,3,4,5,6,7]2416; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7]2417; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]2418; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,3,0,4,5,6,7]2419; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5]2420; SSE-NEXT: pand %xmm13, %xmm22421; SSE-NEXT: por %xmm2, %xmm02422; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2423; SSE-NEXT: movdqa %xmm12, %xmm82424; SSE-NEXT: pandn %xmm15, %xmm82425; SSE-NEXT: movdqa %xmm12, %xmm32426; SSE-NEXT: pandn %xmm9, %xmm32427; SSE-NEXT: pand %xmm12, %xmm152428; SSE-NEXT: por %xmm3, %xmm152429; SSE-NEXT: pshuflw $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload2430; SSE-NEXT: # xmm3 = mem[0,3,2,3,4,5,6,7]2431; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,3]2432; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,5,6]2433; SSE-NEXT: movdqa %xmm13, %xmm02434; SSE-NEXT: pandn %xmm3, %xmm02435; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm15[2,1,2,3,4,5,6,7]2436; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7]2437; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]2438; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,3,0,4,5,6,7]2439; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5]2440; SSE-NEXT: pand %xmm13, %xmm22441; SSE-NEXT: por %xmm2, %xmm02442; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2443; SSE-NEXT: movdqa %xmm12, %xmm02444; SSE-NEXT: pandn %xmm14, %xmm02445; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2446; SSE-NEXT: movdqa %xmm12, %xmm32447; SSE-NEXT: pandn (%rsp), %xmm3 # 16-byte Folded Reload2448; SSE-NEXT: pand %xmm12, %xmm142449; SSE-NEXT: por %xmm3, %xmm142450; SSE-NEXT: pshuflw $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload2451; SSE-NEXT: # xmm3 = mem[0,3,2,3,4,5,6,7]2452; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,3]2453; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,5,6]2454; SSE-NEXT: movdqa %xmm13, %xmm02455; SSE-NEXT: pandn %xmm3, %xmm02456; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm14[2,1,2,3,4,5,6,7]2457; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7]2458; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]2459; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,3,0,4,5,6,7]2460; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5]2461; SSE-NEXT: pand %xmm13, %xmm22462; SSE-NEXT: por %xmm2, %xmm02463; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2464; SSE-NEXT: movdqa %xmm12, %xmm52465; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload2466; SSE-NEXT: pandn %xmm2, %xmm52467; SSE-NEXT: movdqa %xmm12, %xmm32468; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload2469; SSE-NEXT: pandn %xmm14, %xmm32470; SSE-NEXT: pand %xmm12, %xmm22471; SSE-NEXT: por %xmm3, %xmm22472; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload2473; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm11[0,3,2,3,4,5,6,7]2474; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,3]2475; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,5,6]2476; SSE-NEXT: movdqa %xmm13, %xmm152477; SSE-NEXT: pandn %xmm3, %xmm152478; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,2,3,4,5,6,7]2479; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7]2480; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]2481; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,3,0,4,5,6,7]2482; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5]2483; SSE-NEXT: pand %xmm13, %xmm22484; SSE-NEXT: por %xmm2, %xmm152485; SSE-NEXT: movdqa %xmm12, %xmm02486; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload2487; SSE-NEXT: pandn %xmm2, %xmm02488; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2489; SSE-NEXT: movdqa %xmm12, %xmm32490; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload2491; SSE-NEXT: pand %xmm12, %xmm22492; SSE-NEXT: por %xmm3, %xmm22493; SSE-NEXT: pshuflw $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload2494; SSE-NEXT: # xmm3 = mem[0,3,2,3,4,5,6,7]2495; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,3]2496; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,5,6]2497; SSE-NEXT: movdqa %xmm13, %xmm102498; SSE-NEXT: pandn %xmm3, %xmm102499; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,2,3,4,5,6,7]2500; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7]2501; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]2502; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,3,0,4,5,6,7]2503; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5]2504; SSE-NEXT: pand %xmm13, %xmm22505; SSE-NEXT: por %xmm2, %xmm102506; SSE-NEXT: movdqa %xmm12, %xmm22507; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload2508; SSE-NEXT: pandn %xmm3, %xmm22509; SSE-NEXT: movdqa %xmm12, %xmm42510; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload2511; SSE-NEXT: pandn %xmm9, %xmm42512; SSE-NEXT: pand %xmm12, %xmm32513; SSE-NEXT: por %xmm4, %xmm32514; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload2515; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm7[0,3,2,3,4,5,6,7]2516; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,3]2517; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm4[0,1,2,3,4,4,5,6]2518; SSE-NEXT: movdqa %xmm13, %xmm62519; SSE-NEXT: pandn %xmm1, %xmm62520; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm3[2,1,2,3,4,5,6,7]2521; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,7]2522; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3]2523; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,0,4,5,6,7]2524; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5]2525; SSE-NEXT: pand %xmm13, %xmm12526; SSE-NEXT: por %xmm1, %xmm62527; SSE-NEXT: movdqa %xmm12, %xmm12528; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload2529; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2530; SSE-NEXT: pand %xmm12, %xmm02531; SSE-NEXT: por %xmm1, %xmm02532; SSE-NEXT: pshuflw $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload2533; SSE-NEXT: # xmm1 = mem[0,3,2,3,4,5,6,7]2534; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3]2535; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,5,6]2536; SSE-NEXT: movdqa %xmm13, %xmm32537; SSE-NEXT: pandn %xmm1, %xmm32538; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]2539; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,4,7]2540; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3]2541; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,2,3,0,4,5,6,7]2542; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5]2543; SSE-NEXT: pand %xmm13, %xmm02544; SSE-NEXT: por %xmm0, %xmm32545; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2546; SSE-NEXT: pand %xmm12, %xmm02547; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload2548; SSE-NEXT: movdqa %xmm0, %xmm12549; SSE-NEXT: pshufhw $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload2550; SSE-NEXT: # xmm0 = mem[0,1,2,3,4,7,6,7]2551; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,2]2552; SSE-NEXT: movdqa %xmm13, %xmm42553; SSE-NEXT: pandn %xmm0, %xmm42554; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,1,2,0]2555; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7]2556; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3]2557; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]2558; SSE-NEXT: pand %xmm13, %xmm02559; SSE-NEXT: por %xmm0, %xmm42560; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2561; SSE-NEXT: pand %xmm12, %xmm02562; SSE-NEXT: por %xmm8, %xmm02563; SSE-NEXT: movdqa %xmm0, %xmm12564; SSE-NEXT: pshufhw $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload2565; SSE-NEXT: # xmm0 = mem[0,1,2,3,4,7,6,7]2566; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,2]2567; SSE-NEXT: movdqa %xmm13, %xmm82568; SSE-NEXT: pandn %xmm0, %xmm82569; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,1,2,0]2570; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7]2571; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3]2572; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]2573; SSE-NEXT: pand %xmm13, %xmm02574; SSE-NEXT: por %xmm0, %xmm82575; SSE-NEXT: pand %xmm12, %xmm142576; SSE-NEXT: por %xmm5, %xmm142577; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm11[0,1,2,3,4,7,6,7]2578; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,2]2579; SSE-NEXT: movdqa %xmm13, %xmm52580; SSE-NEXT: pandn %xmm0, %xmm52581; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm14[3,1,2,0]2582; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7]2583; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3]2584; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]2585; SSE-NEXT: pand %xmm13, %xmm02586; SSE-NEXT: por %xmm0, %xmm52587; SSE-NEXT: pand %xmm12, %xmm92588; SSE-NEXT: por %xmm2, %xmm92589; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm7[0,1,2,3,4,7,6,7]2590; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,2]2591; SSE-NEXT: movdqa %xmm13, %xmm22592; SSE-NEXT: pandn %xmm0, %xmm22593; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm9[3,1,2,0]2594; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7]2595; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3]2596; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]2597; SSE-NEXT: pand %xmm13, %xmm02598; SSE-NEXT: por %xmm0, %xmm22599; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload2600; SSE-NEXT: pand %xmm12, %xmm12601; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload2602; SSE-NEXT: pshufhw $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload2603; SSE-NEXT: # xmm0 = mem[0,1,2,3,4,7,6,7]2604; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,2]2605; SSE-NEXT: movdqa %xmm13, %xmm142606; SSE-NEXT: pandn %xmm0, %xmm142607; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,1,2,0]2608; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7]2609; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3]2610; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]2611; SSE-NEXT: pand %xmm13, %xmm02612; SSE-NEXT: por %xmm0, %xmm142613; SSE-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload2614; SSE-NEXT: pand %xmm12, %xmm02615; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload2616; SSE-NEXT: movdqa %xmm0, %xmm12617; SSE-NEXT: pshufhw $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload2618; SSE-NEXT: # xmm0 = mem[0,1,2,3,4,7,6,7]2619; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,2]2620; SSE-NEXT: movdqa %xmm13, %xmm112621; SSE-NEXT: pandn %xmm0, %xmm112622; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,1,2,0]2623; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7]2624; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3]2625; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]2626; SSE-NEXT: pand %xmm13, %xmm02627; SSE-NEXT: por %xmm0, %xmm112628; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2629; SSE-NEXT: pand %xmm12, %xmm02630; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload2631; SSE-NEXT: movdqa %xmm0, %xmm12632; SSE-NEXT: pshufhw $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload2633; SSE-NEXT: # xmm0 = mem[0,1,2,3,4,7,6,7]2634; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,2]2635; SSE-NEXT: movdqa %xmm13, %xmm92636; SSE-NEXT: pandn %xmm0, %xmm92637; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,1,2,0]2638; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7]2639; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3]2640; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]2641; SSE-NEXT: pand %xmm13, %xmm02642; SSE-NEXT: por %xmm0, %xmm92643; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2644; SSE-NEXT: pand %xmm12, %xmm02645; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload2646; SSE-NEXT: por %xmm0, %xmm122647; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm12[3,1,2,0]2648; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7]2649; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3]2650; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]2651; SSE-NEXT: pand %xmm13, %xmm02652; SSE-NEXT: pshufhw $236, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload2653; SSE-NEXT: # xmm12 = mem[0,1,2,3,4,7,6,7]2654; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm12[0,1,0,2]2655; SSE-NEXT: pandn %xmm12, %xmm132656; SSE-NEXT: por %xmm0, %xmm132657; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2658; SSE-NEXT: movaps %xmm0, 96(%rsi)2659; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2660; SSE-NEXT: movaps %xmm0, 32(%rsi)2661; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2662; SSE-NEXT: movaps %xmm0, 112(%rsi)2663; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2664; SSE-NEXT: movaps %xmm0, 48(%rsi)2665; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2666; SSE-NEXT: movaps %xmm0, 64(%rsi)2667; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2668; SSE-NEXT: movaps %xmm0, (%rsi)2669; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2670; SSE-NEXT: movaps %xmm0, 80(%rsi)2671; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2672; SSE-NEXT: movaps %xmm0, 16(%rsi)2673; SSE-NEXT: movdqa %xmm3, 96(%rdx)2674; SSE-NEXT: movdqa %xmm6, 32(%rdx)2675; SSE-NEXT: movdqa %xmm10, 112(%rdx)2676; SSE-NEXT: movdqa %xmm15, 48(%rdx)2677; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2678; SSE-NEXT: movaps %xmm0, 64(%rdx)2679; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2680; SSE-NEXT: movaps %xmm0, (%rdx)2681; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2682; SSE-NEXT: movaps %xmm0, 80(%rdx)2683; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2684; SSE-NEXT: movaps %xmm0, 16(%rdx)2685; SSE-NEXT: movdqa %xmm13, 96(%rcx)2686; SSE-NEXT: movdqa %xmm9, 112(%rcx)2687; SSE-NEXT: movdqa %xmm11, 64(%rcx)2688; SSE-NEXT: movdqa %xmm14, 80(%rcx)2689; SSE-NEXT: movdqa %xmm2, 32(%rcx)2690; SSE-NEXT: movdqa %xmm5, 48(%rcx)2691; SSE-NEXT: movdqa %xmm8, (%rcx)2692; SSE-NEXT: movdqa %xmm4, 16(%rcx)2693; SSE-NEXT: addq $440, %rsp # imm = 0x1B82694; SSE-NEXT: retq2695;2696; AVX-LABEL: load_i16_stride3_vf64:2697; AVX: # %bb.0:2698; AVX-NEXT: subq $408, %rsp # imm = 0x1982699; AVX-NEXT: vmovdqa 176(%rdi), %xmm62700; AVX-NEXT: vmovdqa 160(%rdi), %xmm52701; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm5[0,1],xmm6[2],xmm5[3,4],xmm6[5],xmm5[6,7]2702; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]2703; AVX-NEXT: vpshufb %xmm1, %xmm0, %xmm02704; AVX-NEXT: vmovdqa 144(%rdi), %xmm22705; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2706; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,3,2,3,4,5,6,7]2707; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]2708; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm0[3,4,5,6,7]2709; AVX-NEXT: vmovdqa 112(%rdi), %xmm102710; AVX-NEXT: vmovdqa 96(%rdi), %xmm72711; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm7[0],xmm10[1],xmm7[2,3],xmm10[4],xmm7[5,6],xmm10[7]2712; AVX-NEXT: vmovdqa {{.*#+}} xmm8 = [0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]2713; AVX-NEXT: vpshufb %xmm8, %xmm3, %xmm32714; AVX-NEXT: vmovdqa 128(%rdi), %xmm02715; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2716; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[0,1,2,1]2717; AVX-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6,5]2718; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5],xmm4[6,7]2719; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm02720; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2721; AVX-NEXT: vmovdqa 368(%rdi), %xmm142722; AVX-NEXT: vmovdqa 352(%rdi), %xmm22723; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2724; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm14[2],xmm2[3,4],xmm14[5],xmm2[6,7]2725; AVX-NEXT: vmovdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2726; AVX-NEXT: vpshufb %xmm1, %xmm2, %xmm22727; AVX-NEXT: vmovdqa 336(%rdi), %xmm92728; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm9[0,3,2,3,4,5,6,7]2729; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,3,2,3]2730; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3,4,5,6,7]2731; AVX-NEXT: vmovdqa 304(%rdi), %xmm02732; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2733; AVX-NEXT: vmovdqa 288(%rdi), %xmm32734; AVX-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2735; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm0[1],xmm3[2,3],xmm0[4],xmm3[5,6],xmm0[7]2736; AVX-NEXT: vpshufb %xmm8, %xmm3, %xmm32737; AVX-NEXT: vmovdqa 320(%rdi), %xmm02738; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2739; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[0,1,2,1]2740; AVX-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6,5]2741; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5],xmm4[6,7]2742; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm02743; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2744; AVX-NEXT: vmovdqa 272(%rdi), %xmm02745; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2746; AVX-NEXT: vmovdqa 256(%rdi), %xmm22747; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2748; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]2749; AVX-NEXT: vpshufb %xmm1, %xmm2, %xmm22750; AVX-NEXT: vmovdqa 240(%rdi), %xmm02751; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2752; AVX-NEXT: vpshuflw {{.*#+}} xmm3 = xmm0[0,3,2,3,4,5,6,7]2753; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,3,2,3]2754; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3,4,5,6,7]2755; AVX-NEXT: vmovdqa 208(%rdi), %xmm02756; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2757; AVX-NEXT: vmovdqa 192(%rdi), %xmm32758; AVX-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2759; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm0[1],xmm3[2,3],xmm0[4],xmm3[5,6],xmm0[7]2760; AVX-NEXT: vpshufb %xmm8, %xmm3, %xmm32761; AVX-NEXT: vmovdqa 224(%rdi), %xmm02762; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2763; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[0,1,2,1]2764; AVX-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,6,5]2765; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5],xmm4[6,7]2766; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm02767; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2768; AVX-NEXT: vmovdqa 80(%rdi), %xmm132769; AVX-NEXT: vmovdqa 64(%rdi), %xmm112770; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm11[0,1],xmm13[2],xmm11[3,4],xmm13[5],xmm11[6,7]2771; AVX-NEXT: vmovdqa %xmm11, (%rsp) # 16-byte Spill2772; AVX-NEXT: vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2773; AVX-NEXT: vpshufb %xmm1, %xmm2, %xmm12774; AVX-NEXT: vmovdqa 48(%rdi), %xmm32775; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm3[0,3,2,3,4,5,6,7]2776; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]2777; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm1[3,4,5,6,7]2778; AVX-NEXT: vmovdqa (%rdi), %xmm152779; AVX-NEXT: vmovdqa 16(%rdi), %xmm42780; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm15[0],xmm4[1],xmm15[2,3],xmm4[4],xmm15[5,6],xmm4[7]2781; AVX-NEXT: vmovdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2782; AVX-NEXT: vpshufb %xmm8, %xmm1, %xmm02783; AVX-NEXT: vmovdqa 32(%rdi), %xmm12784; AVX-NEXT: vpshufd {{.*#+}} xmm12 = xmm1[0,1,2,1]2785; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2786; AVX-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,5,6,5]2787; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm12[6,7]2788; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm02789; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2790; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm6[0,1],xmm5[2],xmm6[3,4],xmm5[5],xmm6[6,7]2791; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload2792; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm8[1],xmm0[2,3],xmm8[4],xmm0[5,6],xmm8[7]2793; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13]2794; AVX-NEXT: vpshufb %xmm2, %xmm0, %xmm02795; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm7[0,1],xmm10[2],xmm7[3,4],xmm10[5],xmm7[6,7]2796; AVX-NEXT: vpblendw $73, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12, %xmm12 # 16-byte Folded Reload2797; AVX-NEXT: # xmm12 = mem[0],xmm12[1,2],mem[3],xmm12[4,5],mem[6],xmm12[7]2798; AVX-NEXT: vpshufb %xmm2, %xmm12, %xmm122799; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm12, %ymm02800; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2801; AVX-NEXT: vpblendw $36, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14, %xmm0 # 16-byte Folded Reload2802; AVX-NEXT: # xmm0 = xmm14[0,1],mem[2],xmm14[3,4],mem[5],xmm14[6,7]2803; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm9[1],xmm0[2,3],xmm9[4],xmm0[5,6],xmm9[7]2804; AVX-NEXT: vpshufb %xmm2, %xmm0, %xmm02805; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload2806; AVX-NEXT: vpblendw $36, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12, %xmm12 # 16-byte Folded Reload2807; AVX-NEXT: # xmm12 = xmm12[0,1],mem[2],xmm12[3,4],mem[5],xmm12[6,7]2808; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload2809; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm14[0],xmm12[1,2],xmm14[3],xmm12[4,5],xmm14[6],xmm12[7]2810; AVX-NEXT: vpshufb %xmm2, %xmm12, %xmm122811; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm12, %ymm02812; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2813; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm13[0,1],xmm11[2],xmm13[3,4],xmm11[5],xmm13[6,7]2814; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2,3],xmm3[4],xmm0[5,6],xmm3[7]2815; AVX-NEXT: vpshufb %xmm2, %xmm0, %xmm02816; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm15[0,1],xmm4[2],xmm15[3,4],xmm4[5],xmm15[6,7]2817; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm1[0],xmm12[1,2],xmm1[3],xmm12[4,5],xmm1[6],xmm12[7]2818; AVX-NEXT: vpshufb %xmm2, %xmm12, %xmm122819; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm12, %ymm02820; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2821; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload2822; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload2823; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm11[0,1],xmm13[2],xmm11[3,4],xmm13[5],xmm11[6,7]2824; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload2825; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm15[1],xmm0[2,3],xmm15[4],xmm0[5,6],xmm15[7]2826; AVX-NEXT: vpshufb %xmm2, %xmm0, %xmm02827; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload2828; AVX-NEXT: vpblendw $219, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12, %xmm12 # 16-byte Folded Reload2829; AVX-NEXT: # xmm12 = mem[0,1],xmm12[2],mem[3,4],xmm12[5],mem[6,7]2830; AVX-NEXT: vpblendw $73, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12, %xmm12 # 16-byte Folded Reload2831; AVX-NEXT: # xmm12 = mem[0],xmm12[1,2],mem[3],xmm12[4,5],mem[6],xmm12[7]2832; AVX-NEXT: vpshufb %xmm2, %xmm12, %xmm22833; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm02834; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2835; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm5[0],xmm6[1],xmm5[2,3],xmm6[4],xmm5[5,6],xmm6[7]2836; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [u,u,u,u,0,1,6,7,12,13,2,3,8,9,14,15]2837; AVX-NEXT: vpshufb %xmm2, %xmm0, %xmm02838; AVX-NEXT: vpshufd {{.*#+}} xmm12 = xmm8[2,1,2,3]2839; AVX-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[2,1,2,3,4,5,6,7]2840; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm12[0,1],xmm0[2,3,4,5,6,7]2841; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm10[0,1],xmm7[2],xmm10[3,4],xmm7[5],xmm10[6,7]2842; AVX-NEXT: vpblendw $146, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12, %xmm12 # 16-byte Folded Reload2843; AVX-NEXT: # xmm12 = xmm12[0],mem[1],xmm12[2,3],mem[4],xmm12[5,6],mem[7]2844; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15]2845; AVX-NEXT: vpshufb %xmm1, %xmm12, %xmm122846; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm12, %ymm82847; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2848; AVX-NEXT: vpblendw $109, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload2849; AVX-NEXT: # xmm0 = mem[0],xmm0[1],mem[2,3],xmm0[4],mem[5,6],xmm0[7]2850; AVX-NEXT: vpshufb %xmm2, %xmm0, %xmm02851; AVX-NEXT: vpshufd {{.*#+}} xmm12 = xmm9[2,1,2,3]2852; AVX-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[2,1,2,3,4,5,6,7]2853; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm12[0,1],xmm0[2,3,4,5,6,7]2854; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload2855; AVX-NEXT: vpblendw $219, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm10 # 16-byte Folded Reload2856; AVX-NEXT: # xmm10 = mem[0,1],xmm5[2],mem[3,4],xmm5[5],mem[6,7]2857; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm10[0],xmm14[1],xmm10[2,3],xmm14[4],xmm10[5,6],xmm14[7]2858; AVX-NEXT: vpshufb %xmm1, %xmm9, %xmm92859; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm9, %ymm02860; AVX-NEXT: vmovdqa (%rsp), %xmm5 # 16-byte Reload2861; AVX-NEXT: vpblendw $146, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm7 # 16-byte Folded Reload2862; AVX-NEXT: # xmm7 = xmm5[0],mem[1],xmm5[2,3],mem[4],xmm5[5,6],mem[7]2863; AVX-NEXT: vpshufb %xmm2, %xmm7, %xmm72864; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[2,1,2,3]2865; AVX-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[2,1,2,3,4,5,6,7]2866; AVX-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm7[2,3,4,5,6,7]2867; AVX-NEXT: vpblendw $36, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload2868; AVX-NEXT: # xmm4 = xmm4[0,1],mem[2],xmm4[3,4],mem[5],xmm4[6,7]2869; AVX-NEXT: vpblendw $146, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm3 # 16-byte Folded Reload2870; AVX-NEXT: # xmm3 = xmm4[0],mem[1],xmm4[2,3],mem[4],xmm4[5,6],mem[7]2871; AVX-NEXT: vpshufb %xmm1, %xmm3, %xmm32872; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm3, %ymm32873; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm13[0],xmm11[1],xmm13[2,3],xmm11[4],xmm13[5,6],xmm11[7]2874; AVX-NEXT: vpshufb %xmm2, %xmm4, %xmm22875; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm15[2,1,2,3]2876; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[2,1,2,3,4,5,6,7]2877; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0,1],xmm2[2,3,4,5,6,7]2878; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload2879; AVX-NEXT: vpblendw $36, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload2880; AVX-NEXT: # xmm4 = xmm4[0,1],mem[2],xmm4[3,4],mem[5],xmm4[6,7]2881; AVX-NEXT: vpblendw $146, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload2882; AVX-NEXT: # xmm4 = xmm4[0],mem[1],xmm4[2,3],mem[4],xmm4[5,6],mem[7]2883; AVX-NEXT: vpshufb %xmm1, %xmm4, %xmm12884; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm12885; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload2886; AVX-NEXT: vmovaps %ymm2, (%rsi)2887; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload2888; AVX-NEXT: vmovaps %ymm2, 64(%rsi)2889; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload2890; AVX-NEXT: vmovaps %ymm2, 96(%rsi)2891; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload2892; AVX-NEXT: vmovaps %ymm2, 32(%rsi)2893; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload2894; AVX-NEXT: vmovaps %ymm2, 64(%rdx)2895; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload2896; AVX-NEXT: vmovaps %ymm2, (%rdx)2897; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload2898; AVX-NEXT: vmovaps %ymm2, 96(%rdx)2899; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload2900; AVX-NEXT: vmovaps %ymm2, 32(%rdx)2901; AVX-NEXT: vmovaps %ymm1, 64(%rcx)2902; AVX-NEXT: vmovaps %ymm3, (%rcx)2903; AVX-NEXT: vmovaps %ymm0, 96(%rcx)2904; AVX-NEXT: vmovaps %ymm8, 32(%rcx)2905; AVX-NEXT: addq $408, %rsp # imm = 0x1982906; AVX-NEXT: vzeroupper2907; AVX-NEXT: retq2908;2909; AVX2-LABEL: load_i16_stride3_vf64:2910; AVX2: # %bb.0:2911; AVX2-NEXT: subq $136, %rsp2912; AVX2-NEXT: vmovdqa (%rdi), %ymm12913; AVX2-NEXT: vmovdqa 32(%rdi), %ymm22914; AVX2-NEXT: vmovdqa 192(%rdi), %ymm42915; AVX2-NEXT: vmovdqa 224(%rdi), %ymm52916; AVX2-NEXT: vmovdqa 288(%rdi), %ymm92917; AVX2-NEXT: vmovdqa 320(%rdi), %ymm102918; AVX2-NEXT: vmovdqa 96(%rdi), %ymm122919; AVX2-NEXT: vmovdqa 128(%rdi), %ymm132920; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm11 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]2921; AVX2-NEXT: vpblendvb %ymm11, %ymm12, %ymm13, %ymm02922; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm0[2,3,0,1]2923; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm3[1],ymm0[2,3],ymm3[4],ymm0[5,6],ymm3[7],ymm0[8],ymm3[9],ymm0[10,11],ymm3[12],ymm0[13,14],ymm3[15]2924; AVX2-NEXT: vpshufb {{.*#+}} ymm15 = ymm0[0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]2925; AVX2-NEXT: vpblendvb %ymm11, %ymm9, %ymm10, %ymm32926; AVX2-NEXT: vpblendvb %ymm11, %ymm4, %ymm5, %ymm82927; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm0 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]2928; AVX2-NEXT: vpblendvb %ymm0, %ymm13, %ymm12, %ymm62929; AVX2-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2930; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm7 = [0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0]2931; AVX2-NEXT: vpblendvb %ymm7, %ymm13, %ymm12, %ymm62932; AVX2-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2933; AVX2-NEXT: vpblendvb %ymm0, %ymm10, %ymm9, %ymm132934; AVX2-NEXT: vpblendvb %ymm7, %ymm10, %ymm9, %ymm62935; AVX2-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2936; AVX2-NEXT: vpblendvb %ymm0, %ymm5, %ymm4, %ymm102937; AVX2-NEXT: vpblendvb %ymm7, %ymm5, %ymm4, %ymm42938; AVX2-NEXT: vmovdqu %ymm4, (%rsp) # 32-byte Spill2939; AVX2-NEXT: vmovdqa 176(%rdi), %xmm52940; AVX2-NEXT: vpblendvb %ymm11, %ymm1, %ymm2, %ymm112941; AVX2-NEXT: vpblendvb %ymm0, %ymm2, %ymm1, %ymm142942; AVX2-NEXT: vpblendvb %ymm7, %ymm2, %ymm1, %ymm62943; AVX2-NEXT: vmovdqa 160(%rdi), %xmm72944; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm7[0,1],xmm5[2],xmm7[3,4],xmm5[5],xmm7[6,7]2945; AVX2-NEXT: vmovdqa {{.*#+}} xmm0 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]2946; AVX2-NEXT: vpshufb %xmm0, %xmm1, %xmm12947; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm12948; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm15[0,1,2],ymm1[3,4,5,6,7],ymm15[8,9,10],ymm1[11,12,13,14,15]2949; AVX2-NEXT: vpshufhw {{.*#+}} xmm2 = xmm15[0,1,2,3,6,5,4,7]2950; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]2951; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2952; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm3[2,3,0,1]2953; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm3[0],ymm1[1],ymm3[2,3],ymm1[4],ymm3[5,6],ymm1[7],ymm3[8],ymm1[9],ymm3[10,11],ymm1[12],ymm3[13,14],ymm1[15]2954; AVX2-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]2955; AVX2-NEXT: vpshufb %ymm3, %ymm1, %ymm12956; AVX2-NEXT: vmovdqa 368(%rdi), %xmm152957; AVX2-NEXT: vmovdqa 352(%rdi), %xmm42958; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0,1],xmm15[2],xmm4[3,4],xmm15[5],xmm4[6,7]2959; AVX2-NEXT: vpshufb %xmm0, %xmm2, %xmm22960; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm22961; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0,1,2],ymm2[3,4,5,6,7],ymm1[8,9,10],ymm2[11,12,13,14,15]2962; AVX2-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,4,7]2963; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]2964; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2965; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm8[2,3,0,1]2966; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm8[0],ymm1[1],ymm8[2,3],ymm1[4],ymm8[5,6],ymm1[7],ymm8[8],ymm1[9],ymm8[10,11],ymm1[12],ymm8[13,14],ymm1[15]2967; AVX2-NEXT: vpshufb %ymm3, %ymm1, %ymm12968; AVX2-NEXT: vmovdqa %ymm3, %ymm22969; AVX2-NEXT: vmovdqa 272(%rdi), %xmm82970; AVX2-NEXT: vmovdqa 256(%rdi), %xmm32971; AVX2-NEXT: vpblendw {{.*#+}} xmm12 = xmm3[0,1],xmm8[2],xmm3[3,4],xmm8[5],xmm3[6,7]2972; AVX2-NEXT: vpshufb %xmm0, %xmm12, %xmm122973; AVX2-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm122974; AVX2-NEXT: vpblendw {{.*#+}} ymm12 = ymm1[0,1,2],ymm12[3,4,5,6,7],ymm1[8,9,10],ymm12[11,12,13,14,15]2975; AVX2-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,4,7]2976; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm12[4,5,6,7]2977; AVX2-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2978; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm11[2,3,0,1]2979; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm11[0],ymm1[1],ymm11[2,3],ymm1[4],ymm11[5,6],ymm1[7],ymm11[8],ymm1[9],ymm11[10,11],ymm1[12],ymm11[13,14],ymm1[15]2980; AVX2-NEXT: vpshufb %ymm2, %ymm1, %ymm122981; AVX2-NEXT: vmovdqa 80(%rdi), %xmm22982; AVX2-NEXT: vmovdqa 64(%rdi), %xmm112983; AVX2-NEXT: vpblendw {{.*#+}} xmm9 = xmm11[0,1],xmm2[2],xmm11[3,4],xmm2[5],xmm11[6,7]2984; AVX2-NEXT: vpshufb %xmm0, %xmm9, %xmm02985; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm02986; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm12[0,1,2],ymm0[3,4,5,6,7],ymm12[8,9,10],ymm0[11,12,13,14,15]2987; AVX2-NEXT: vpshufhw {{.*#+}} xmm9 = xmm12[0,1,2,3,6,5,4,7]2988; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3],ymm0[4,5,6,7]2989; AVX2-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2990; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload2991; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm1[2,3,0,1]2992; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7,8,9],ymm0[10],ymm1[11,12],ymm0[13],ymm1[14,15]2993; AVX2-NEXT: vmovdqa {{.*#+}} ymm9 = [2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]2994; AVX2-NEXT: vpshufb %ymm9, %ymm0, %ymm02995; AVX2-NEXT: vpblendw {{.*#+}} xmm12 = xmm5[0,1],xmm7[2],xmm5[3,4],xmm7[5],xmm5[6,7]2996; AVX2-NEXT: vmovdqa {{.*#+}} xmm1 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]2997; AVX2-NEXT: vpshufb %xmm1, %xmm12, %xmm122998; AVX2-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm122999; AVX2-NEXT: vpblendw {{.*#+}} ymm12 = ymm0[0,1,2],ymm12[3,4,5,6,7],ymm0[8,9,10],ymm12[11,12,13,14,15]3000; AVX2-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,6,7,4]3001; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm12[4,5,6,7]3002; AVX2-NEXT: vpermq {{.*#+}} ymm12 = ymm13[2,3,0,1]3003; AVX2-NEXT: vpblendw {{.*#+}} ymm12 = ymm13[0,1],ymm12[2],ymm13[3,4],ymm12[5],ymm13[6,7,8,9],ymm12[10],ymm13[11,12],ymm12[13],ymm13[14,15]3004; AVX2-NEXT: vpshufb %ymm9, %ymm12, %ymm123005; AVX2-NEXT: vpblendw {{.*#+}} xmm13 = xmm15[0,1],xmm4[2],xmm15[3,4],xmm4[5],xmm15[6,7]3006; AVX2-NEXT: vpshufb %xmm1, %xmm13, %xmm133007; AVX2-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm133008; AVX2-NEXT: vpblendw {{.*#+}} ymm13 = ymm12[0,1,2],ymm13[3,4,5,6,7],ymm12[8,9,10],ymm13[11,12,13,14,15]3009; AVX2-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,5,6,7,4]3010; AVX2-NEXT: vpblendd {{.*#+}} ymm13 = ymm12[0,1,2,3],ymm13[4,5,6,7]3011; AVX2-NEXT: vpermq {{.*#+}} ymm12 = ymm10[2,3,0,1]3012; AVX2-NEXT: vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm12[2],ymm10[3,4],ymm12[5],ymm10[6,7,8,9],ymm12[10],ymm10[11,12],ymm12[13],ymm10[14,15]3013; AVX2-NEXT: vpshufb %ymm9, %ymm10, %ymm103014; AVX2-NEXT: vpblendw {{.*#+}} xmm12 = xmm8[0,1],xmm3[2],xmm8[3,4],xmm3[5],xmm8[6,7]3015; AVX2-NEXT: vpshufb %xmm1, %xmm12, %xmm123016; AVX2-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm123017; AVX2-NEXT: vpblendw {{.*#+}} ymm12 = ymm10[0,1,2],ymm12[3,4,5,6,7],ymm10[8,9,10],ymm12[11,12,13,14,15]3018; AVX2-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,6,7,4]3019; AVX2-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm12[4,5,6,7]3020; AVX2-NEXT: vpermq {{.*#+}} ymm12 = ymm14[2,3,0,1]3021; AVX2-NEXT: vpblendw {{.*#+}} ymm12 = ymm14[0,1],ymm12[2],ymm14[3,4],ymm12[5],ymm14[6,7,8,9],ymm12[10],ymm14[11,12],ymm12[13],ymm14[14,15]3022; AVX2-NEXT: vpshufb %ymm9, %ymm12, %ymm93023; AVX2-NEXT: vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm11[2],xmm2[3,4],xmm11[5],xmm2[6,7]3024; AVX2-NEXT: vpshufb %xmm1, %xmm12, %xmm13025; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm13026; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm9[0,1,2],ymm1[3,4,5,6,7],ymm9[8,9,10],ymm1[11,12,13,14,15]3027; AVX2-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,5,6,7,4]3028; AVX2-NEXT: vpblendd {{.*#+}} ymm14 = ymm9[0,1,2,3],ymm1[4,5,6,7]3029; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload3030; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm9[2,3,0,1]3031; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm9[1,2],ymm1[3],ymm9[4,5],ymm1[6],ymm9[7],ymm1[8],ymm9[9,10],ymm1[11],ymm9[12,13],ymm1[14],ymm9[15]3032; AVX2-NEXT: vpblendw {{.*#+}} xmm5 = xmm7[0],xmm5[1],xmm7[2,3],xmm5[4],xmm7[5,6],xmm5[7]3033; AVX2-NEXT: vmovdqa {{.*#+}} ymm7 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]3034; AVX2-NEXT: vpshufb %ymm7, %ymm1, %ymm13035; AVX2-NEXT: vmovdqa {{.*#+}} xmm9 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]3036; AVX2-NEXT: vpshufb %xmm9, %xmm5, %xmm53037; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm53038; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm1[0,1,2,3,4],ymm5[5,6,7]3039; AVX2-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload3040; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm12[2,3,0,1]3041; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm12[1,2],ymm1[3],ymm12[4,5],ymm1[6],ymm12[7],ymm1[8],ymm12[9,10],ymm1[11],ymm12[12,13],ymm1[14],ymm12[15]3042; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm15[1],xmm4[2,3],xmm15[4],xmm4[5,6],xmm15[7]3043; AVX2-NEXT: vpshufb %ymm7, %ymm1, %ymm13044; AVX2-NEXT: vpshufb %xmm9, %xmm4, %xmm43045; AVX2-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm43046; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm4[5,6,7]3047; AVX2-NEXT: vmovdqu (%rsp), %ymm12 # 32-byte Reload3048; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm12[2,3,0,1]3049; AVX2-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm12[1,2],ymm4[3],ymm12[4,5],ymm4[6],ymm12[7],ymm4[8],ymm12[9,10],ymm4[11],ymm12[12,13],ymm4[14],ymm12[15]3050; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm8[1],xmm3[2,3],xmm8[4],xmm3[5,6],xmm8[7]3051; AVX2-NEXT: vpshufb %ymm7, %ymm4, %ymm43052; AVX2-NEXT: vpshufb %xmm9, %xmm3, %xmm33053; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm33054; AVX2-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4],ymm3[5,6,7]3055; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm6[2,3,0,1]3056; AVX2-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm6[1,2],ymm4[3],ymm6[4,5],ymm4[6],ymm6[7],ymm4[8],ymm6[9,10],ymm4[11],ymm6[12,13],ymm4[14],ymm6[15]3057; AVX2-NEXT: vpshufb %ymm7, %ymm4, %ymm43058; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm11[0],xmm2[1],xmm11[2,3],xmm2[4],xmm11[5,6],xmm2[7]3059; AVX2-NEXT: vpshufb %xmm9, %xmm2, %xmm23060; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm23061; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4],ymm2[5,6,7]3062; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload3063; AVX2-NEXT: vmovaps %ymm4, (%rsi)3064; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload3065; AVX2-NEXT: vmovaps %ymm4, 64(%rsi)3066; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload3067; AVX2-NEXT: vmovaps %ymm4, 96(%rsi)3068; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload3069; AVX2-NEXT: vmovaps %ymm4, 32(%rsi)3070; AVX2-NEXT: vmovdqa %ymm10, 64(%rdx)3071; AVX2-NEXT: vmovdqa %ymm14, (%rdx)3072; AVX2-NEXT: vmovdqa %ymm13, 96(%rdx)3073; AVX2-NEXT: vmovdqa %ymm0, 32(%rdx)3074; AVX2-NEXT: vmovdqa %ymm3, 64(%rcx)3075; AVX2-NEXT: vmovdqa %ymm2, (%rcx)3076; AVX2-NEXT: vmovdqa %ymm1, 96(%rcx)3077; AVX2-NEXT: vmovdqa %ymm5, 32(%rcx)3078; AVX2-NEXT: addq $136, %rsp3079; AVX2-NEXT: vzeroupper3080; AVX2-NEXT: retq3081;3082; AVX2-FP-LABEL: load_i16_stride3_vf64:3083; AVX2-FP: # %bb.0:3084; AVX2-FP-NEXT: subq $136, %rsp3085; AVX2-FP-NEXT: vmovdqa (%rdi), %ymm13086; AVX2-FP-NEXT: vmovdqa 32(%rdi), %ymm23087; AVX2-FP-NEXT: vmovdqa 192(%rdi), %ymm43088; AVX2-FP-NEXT: vmovdqa 224(%rdi), %ymm53089; AVX2-FP-NEXT: vmovdqa 288(%rdi), %ymm93090; AVX2-FP-NEXT: vmovdqa 320(%rdi), %ymm103091; AVX2-FP-NEXT: vmovdqa 96(%rdi), %ymm123092; AVX2-FP-NEXT: vmovdqa 128(%rdi), %ymm133093; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm11 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]3094; AVX2-FP-NEXT: vpblendvb %ymm11, %ymm12, %ymm13, %ymm03095; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm0[2,3,0,1]3096; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm3[1],ymm0[2,3],ymm3[4],ymm0[5,6],ymm3[7],ymm0[8],ymm3[9],ymm0[10,11],ymm3[12],ymm0[13,14],ymm3[15]3097; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm15 = ymm0[0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]3098; AVX2-FP-NEXT: vpblendvb %ymm11, %ymm9, %ymm10, %ymm33099; AVX2-FP-NEXT: vpblendvb %ymm11, %ymm4, %ymm5, %ymm83100; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm0 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]3101; AVX2-FP-NEXT: vpblendvb %ymm0, %ymm13, %ymm12, %ymm63102; AVX2-FP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3103; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm7 = [0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0]3104; AVX2-FP-NEXT: vpblendvb %ymm7, %ymm13, %ymm12, %ymm63105; AVX2-FP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3106; AVX2-FP-NEXT: vpblendvb %ymm0, %ymm10, %ymm9, %ymm133107; AVX2-FP-NEXT: vpblendvb %ymm7, %ymm10, %ymm9, %ymm63108; AVX2-FP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3109; AVX2-FP-NEXT: vpblendvb %ymm0, %ymm5, %ymm4, %ymm103110; AVX2-FP-NEXT: vpblendvb %ymm7, %ymm5, %ymm4, %ymm43111; AVX2-FP-NEXT: vmovdqu %ymm4, (%rsp) # 32-byte Spill3112; AVX2-FP-NEXT: vmovdqa 176(%rdi), %xmm53113; AVX2-FP-NEXT: vpblendvb %ymm11, %ymm1, %ymm2, %ymm113114; AVX2-FP-NEXT: vpblendvb %ymm0, %ymm2, %ymm1, %ymm143115; AVX2-FP-NEXT: vpblendvb %ymm7, %ymm2, %ymm1, %ymm63116; AVX2-FP-NEXT: vmovdqa 160(%rdi), %xmm73117; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm1 = xmm7[0,1],xmm5[2],xmm7[3,4],xmm5[5],xmm7[6,7]3118; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm0 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]3119; AVX2-FP-NEXT: vpshufb %xmm0, %xmm1, %xmm13120; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm13121; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = ymm15[0,1,2],ymm1[3,4,5,6,7],ymm15[8,9,10],ymm1[11,12,13,14,15]3122; AVX2-FP-NEXT: vpshufhw {{.*#+}} xmm2 = xmm15[0,1,2,3,6,5,4,7]3123; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]3124; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3125; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm3[2,3,0,1]3126; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = ymm3[0],ymm1[1],ymm3[2,3],ymm1[4],ymm3[5,6],ymm1[7],ymm3[8],ymm1[9],ymm3[10,11],ymm1[12],ymm3[13,14],ymm1[15]3127; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]3128; AVX2-FP-NEXT: vpshufb %ymm3, %ymm1, %ymm13129; AVX2-FP-NEXT: vmovdqa 368(%rdi), %xmm153130; AVX2-FP-NEXT: vmovdqa 352(%rdi), %xmm43131; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0,1],xmm15[2],xmm4[3,4],xmm15[5],xmm4[6,7]3132; AVX2-FP-NEXT: vpshufb %xmm0, %xmm2, %xmm23133; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm23134; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0,1,2],ymm2[3,4,5,6,7],ymm1[8,9,10],ymm2[11,12,13,14,15]3135; AVX2-FP-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,4,7]3136; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]3137; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3138; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm8[2,3,0,1]3139; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = ymm8[0],ymm1[1],ymm8[2,3],ymm1[4],ymm8[5,6],ymm1[7],ymm8[8],ymm1[9],ymm8[10,11],ymm1[12],ymm8[13,14],ymm1[15]3140; AVX2-FP-NEXT: vpshufb %ymm3, %ymm1, %ymm13141; AVX2-FP-NEXT: vmovdqa %ymm3, %ymm23142; AVX2-FP-NEXT: vmovdqa 272(%rdi), %xmm83143; AVX2-FP-NEXT: vmovdqa 256(%rdi), %xmm33144; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm12 = xmm3[0,1],xmm8[2],xmm3[3,4],xmm8[5],xmm3[6,7]3145; AVX2-FP-NEXT: vpshufb %xmm0, %xmm12, %xmm123146; AVX2-FP-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm123147; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm12 = ymm1[0,1,2],ymm12[3,4,5,6,7],ymm1[8,9,10],ymm12[11,12,13,14,15]3148; AVX2-FP-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,4,7]3149; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm12[4,5,6,7]3150; AVX2-FP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3151; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm11[2,3,0,1]3152; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = ymm11[0],ymm1[1],ymm11[2,3],ymm1[4],ymm11[5,6],ymm1[7],ymm11[8],ymm1[9],ymm11[10,11],ymm1[12],ymm11[13,14],ymm1[15]3153; AVX2-FP-NEXT: vpshufb %ymm2, %ymm1, %ymm123154; AVX2-FP-NEXT: vmovdqa 80(%rdi), %xmm23155; AVX2-FP-NEXT: vmovdqa 64(%rdi), %xmm113156; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm9 = xmm11[0,1],xmm2[2],xmm11[3,4],xmm2[5],xmm11[6,7]3157; AVX2-FP-NEXT: vpshufb %xmm0, %xmm9, %xmm03158; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm03159; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm12[0,1,2],ymm0[3,4,5,6,7],ymm12[8,9,10],ymm0[11,12,13,14,15]3160; AVX2-FP-NEXT: vpshufhw {{.*#+}} xmm9 = xmm12[0,1,2,3,6,5,4,7]3161; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3],ymm0[4,5,6,7]3162; AVX2-FP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3163; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload3164; AVX2-FP-NEXT: vpermq {{.*#+}} ymm0 = ymm1[2,3,0,1]3165; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7,8,9],ymm0[10],ymm1[11,12],ymm0[13],ymm1[14,15]3166; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm9 = [2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]3167; AVX2-FP-NEXT: vpshufb %ymm9, %ymm0, %ymm03168; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm12 = xmm5[0,1],xmm7[2],xmm5[3,4],xmm7[5],xmm5[6,7]3169; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm1 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]3170; AVX2-FP-NEXT: vpshufb %xmm1, %xmm12, %xmm123171; AVX2-FP-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm123172; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm12 = ymm0[0,1,2],ymm12[3,4,5,6,7],ymm0[8,9,10],ymm12[11,12,13,14,15]3173; AVX2-FP-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,6,7,4]3174; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm12[4,5,6,7]3175; AVX2-FP-NEXT: vpermq {{.*#+}} ymm12 = ymm13[2,3,0,1]3176; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm12 = ymm13[0,1],ymm12[2],ymm13[3,4],ymm12[5],ymm13[6,7,8,9],ymm12[10],ymm13[11,12],ymm12[13],ymm13[14,15]3177; AVX2-FP-NEXT: vpshufb %ymm9, %ymm12, %ymm123178; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm13 = xmm15[0,1],xmm4[2],xmm15[3,4],xmm4[5],xmm15[6,7]3179; AVX2-FP-NEXT: vpshufb %xmm1, %xmm13, %xmm133180; AVX2-FP-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm133181; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm13 = ymm12[0,1,2],ymm13[3,4,5,6,7],ymm12[8,9,10],ymm13[11,12,13,14,15]3182; AVX2-FP-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,5,6,7,4]3183; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm13 = ymm12[0,1,2,3],ymm13[4,5,6,7]3184; AVX2-FP-NEXT: vpermq {{.*#+}} ymm12 = ymm10[2,3,0,1]3185; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm12[2],ymm10[3,4],ymm12[5],ymm10[6,7,8,9],ymm12[10],ymm10[11,12],ymm12[13],ymm10[14,15]3186; AVX2-FP-NEXT: vpshufb %ymm9, %ymm10, %ymm103187; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm12 = xmm8[0,1],xmm3[2],xmm8[3,4],xmm3[5],xmm8[6,7]3188; AVX2-FP-NEXT: vpshufb %xmm1, %xmm12, %xmm123189; AVX2-FP-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm123190; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm12 = ymm10[0,1,2],ymm12[3,4,5,6,7],ymm10[8,9,10],ymm12[11,12,13,14,15]3191; AVX2-FP-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,6,7,4]3192; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm12[4,5,6,7]3193; AVX2-FP-NEXT: vpermq {{.*#+}} ymm12 = ymm14[2,3,0,1]3194; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm12 = ymm14[0,1],ymm12[2],ymm14[3,4],ymm12[5],ymm14[6,7,8,9],ymm12[10],ymm14[11,12],ymm12[13],ymm14[14,15]3195; AVX2-FP-NEXT: vpshufb %ymm9, %ymm12, %ymm93196; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm11[2],xmm2[3,4],xmm11[5],xmm2[6,7]3197; AVX2-FP-NEXT: vpshufb %xmm1, %xmm12, %xmm13198; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm13199; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = ymm9[0,1,2],ymm1[3,4,5,6,7],ymm9[8,9,10],ymm1[11,12,13,14,15]3200; AVX2-FP-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,5,6,7,4]3201; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm14 = ymm9[0,1,2,3],ymm1[4,5,6,7]3202; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload3203; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm9[2,3,0,1]3204; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm9[1,2],ymm1[3],ymm9[4,5],ymm1[6],ymm9[7],ymm1[8],ymm9[9,10],ymm1[11],ymm9[12,13],ymm1[14],ymm9[15]3205; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm5 = xmm7[0],xmm5[1],xmm7[2,3],xmm5[4],xmm7[5,6],xmm5[7]3206; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm7 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]3207; AVX2-FP-NEXT: vpshufb %ymm7, %ymm1, %ymm13208; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm9 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]3209; AVX2-FP-NEXT: vpshufb %xmm9, %xmm5, %xmm53210; AVX2-FP-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm53211; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm5 = ymm1[0,1,2,3,4],ymm5[5,6,7]3212; AVX2-FP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload3213; AVX2-FP-NEXT: vpermq {{.*#+}} ymm1 = ymm12[2,3,0,1]3214; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm12[1,2],ymm1[3],ymm12[4,5],ymm1[6],ymm12[7],ymm1[8],ymm12[9,10],ymm1[11],ymm12[12,13],ymm1[14],ymm12[15]3215; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm15[1],xmm4[2,3],xmm15[4],xmm4[5,6],xmm15[7]3216; AVX2-FP-NEXT: vpshufb %ymm7, %ymm1, %ymm13217; AVX2-FP-NEXT: vpshufb %xmm9, %xmm4, %xmm43218; AVX2-FP-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm43219; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm4[5,6,7]3220; AVX2-FP-NEXT: vmovdqu (%rsp), %ymm12 # 32-byte Reload3221; AVX2-FP-NEXT: vpermq {{.*#+}} ymm4 = ymm12[2,3,0,1]3222; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm12[1,2],ymm4[3],ymm12[4,5],ymm4[6],ymm12[7],ymm4[8],ymm12[9,10],ymm4[11],ymm12[12,13],ymm4[14],ymm12[15]3223; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm8[1],xmm3[2,3],xmm8[4],xmm3[5,6],xmm8[7]3224; AVX2-FP-NEXT: vpshufb %ymm7, %ymm4, %ymm43225; AVX2-FP-NEXT: vpshufb %xmm9, %xmm3, %xmm33226; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm33227; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4],ymm3[5,6,7]3228; AVX2-FP-NEXT: vpermq {{.*#+}} ymm4 = ymm6[2,3,0,1]3229; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm6[1,2],ymm4[3],ymm6[4,5],ymm4[6],ymm6[7],ymm4[8],ymm6[9,10],ymm4[11],ymm6[12,13],ymm4[14],ymm6[15]3230; AVX2-FP-NEXT: vpshufb %ymm7, %ymm4, %ymm43231; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm2 = xmm11[0],xmm2[1],xmm11[2,3],xmm2[4],xmm11[5,6],xmm2[7]3232; AVX2-FP-NEXT: vpshufb %xmm9, %xmm2, %xmm23233; AVX2-FP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm23234; AVX2-FP-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4],ymm2[5,6,7]3235; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload3236; AVX2-FP-NEXT: vmovaps %ymm4, (%rsi)3237; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload3238; AVX2-FP-NEXT: vmovaps %ymm4, 64(%rsi)3239; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload3240; AVX2-FP-NEXT: vmovaps %ymm4, 96(%rsi)3241; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload3242; AVX2-FP-NEXT: vmovaps %ymm4, 32(%rsi)3243; AVX2-FP-NEXT: vmovdqa %ymm10, 64(%rdx)3244; AVX2-FP-NEXT: vmovdqa %ymm14, (%rdx)3245; AVX2-FP-NEXT: vmovdqa %ymm13, 96(%rdx)3246; AVX2-FP-NEXT: vmovdqa %ymm0, 32(%rdx)3247; AVX2-FP-NEXT: vmovdqa %ymm3, 64(%rcx)3248; AVX2-FP-NEXT: vmovdqa %ymm2, (%rcx)3249; AVX2-FP-NEXT: vmovdqa %ymm1, 96(%rcx)3250; AVX2-FP-NEXT: vmovdqa %ymm5, 32(%rcx)3251; AVX2-FP-NEXT: addq $136, %rsp3252; AVX2-FP-NEXT: vzeroupper3253; AVX2-FP-NEXT: retq3254;3255; AVX2-FCP-LABEL: load_i16_stride3_vf64:3256; AVX2-FCP: # %bb.0:3257; AVX2-FCP-NEXT: subq $136, %rsp3258; AVX2-FCP-NEXT: vmovdqa (%rdi), %ymm13259; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %ymm23260; AVX2-FCP-NEXT: vmovdqa 192(%rdi), %ymm43261; AVX2-FCP-NEXT: vmovdqa 224(%rdi), %ymm53262; AVX2-FCP-NEXT: vmovdqa 288(%rdi), %ymm93263; AVX2-FCP-NEXT: vmovdqa 320(%rdi), %ymm103264; AVX2-FCP-NEXT: vmovdqa 96(%rdi), %ymm123265; AVX2-FCP-NEXT: vmovdqa 128(%rdi), %ymm133266; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm11 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]3267; AVX2-FCP-NEXT: vpblendvb %ymm11, %ymm12, %ymm13, %ymm03268; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm0[2,3,0,1]3269; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm3[1],ymm0[2,3],ymm3[4],ymm0[5,6],ymm3[7],ymm0[8],ymm3[9],ymm0[10,11],ymm3[12],ymm0[13,14],ymm3[15]3270; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm15 = ymm0[0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]3271; AVX2-FCP-NEXT: vpblendvb %ymm11, %ymm9, %ymm10, %ymm33272; AVX2-FCP-NEXT: vpblendvb %ymm11, %ymm4, %ymm5, %ymm83273; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm0 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]3274; AVX2-FCP-NEXT: vpblendvb %ymm0, %ymm13, %ymm12, %ymm63275; AVX2-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3276; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm7 = [0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0]3277; AVX2-FCP-NEXT: vpblendvb %ymm7, %ymm13, %ymm12, %ymm63278; AVX2-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3279; AVX2-FCP-NEXT: vpblendvb %ymm0, %ymm10, %ymm9, %ymm133280; AVX2-FCP-NEXT: vpblendvb %ymm7, %ymm10, %ymm9, %ymm63281; AVX2-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3282; AVX2-FCP-NEXT: vpblendvb %ymm0, %ymm5, %ymm4, %ymm103283; AVX2-FCP-NEXT: vpblendvb %ymm7, %ymm5, %ymm4, %ymm43284; AVX2-FCP-NEXT: vmovdqu %ymm4, (%rsp) # 32-byte Spill3285; AVX2-FCP-NEXT: vmovdqa 176(%rdi), %xmm53286; AVX2-FCP-NEXT: vpblendvb %ymm11, %ymm1, %ymm2, %ymm113287; AVX2-FCP-NEXT: vpblendvb %ymm0, %ymm2, %ymm1, %ymm143288; AVX2-FCP-NEXT: vpblendvb %ymm7, %ymm2, %ymm1, %ymm63289; AVX2-FCP-NEXT: vmovdqa 160(%rdi), %xmm73290; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm7[0,1],xmm5[2],xmm7[3,4],xmm5[5],xmm7[6,7]3291; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]3292; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm1, %xmm13293; AVX2-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm13294; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm15[0,1,2],ymm1[3,4,5,6,7],ymm15[8,9,10],ymm1[11,12,13,14,15]3295; AVX2-FCP-NEXT: vpshufhw {{.*#+}} xmm2 = xmm15[0,1,2,3,6,5,4,7]3296; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]3297; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3298; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm3[2,3,0,1]3299; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm3[0],ymm1[1],ymm3[2,3],ymm1[4],ymm3[5,6],ymm1[7],ymm3[8],ymm1[9],ymm3[10,11],ymm1[12],ymm3[13,14],ymm1[15]3300; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm3 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]3301; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm1, %ymm13302; AVX2-FCP-NEXT: vmovdqa 368(%rdi), %xmm153303; AVX2-FCP-NEXT: vmovdqa 352(%rdi), %xmm43304; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm4[0,1],xmm15[2],xmm4[3,4],xmm15[5],xmm4[6,7]3305; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm2, %xmm23306; AVX2-FCP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm23307; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0,1,2],ymm2[3,4,5,6,7],ymm1[8,9,10],ymm2[11,12,13,14,15]3308; AVX2-FCP-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,4,7]3309; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]3310; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3311; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm8[2,3,0,1]3312; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm8[0],ymm1[1],ymm8[2,3],ymm1[4],ymm8[5,6],ymm1[7],ymm8[8],ymm1[9],ymm8[10,11],ymm1[12],ymm8[13,14],ymm1[15]3313; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm1, %ymm13314; AVX2-FCP-NEXT: vmovdqa %ymm3, %ymm23315; AVX2-FCP-NEXT: vmovdqa 272(%rdi), %xmm83316; AVX2-FCP-NEXT: vmovdqa 256(%rdi), %xmm33317; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm3[0,1],xmm8[2],xmm3[3,4],xmm8[5],xmm3[6,7]3318; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm12, %xmm123319; AVX2-FCP-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm123320; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm1[0,1,2],ymm12[3,4,5,6,7],ymm1[8,9,10],ymm12[11,12,13,14,15]3321; AVX2-FCP-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,4,7]3322; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm12[4,5,6,7]3323; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3324; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm11[2,3,0,1]3325; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm11[0],ymm1[1],ymm11[2,3],ymm1[4],ymm11[5,6],ymm1[7],ymm11[8],ymm1[9],ymm11[10,11],ymm1[12],ymm11[13,14],ymm1[15]3326; AVX2-FCP-NEXT: vpshufb %ymm2, %ymm1, %ymm123327; AVX2-FCP-NEXT: vmovdqa 80(%rdi), %xmm23328; AVX2-FCP-NEXT: vmovdqa 64(%rdi), %xmm113329; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm9 = xmm11[0,1],xmm2[2],xmm11[3,4],xmm2[5],xmm11[6,7]3330; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm9, %xmm03331; AVX2-FCP-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm03332; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm12[0,1,2],ymm0[3,4,5,6,7],ymm12[8,9,10],ymm0[11,12,13,14,15]3333; AVX2-FCP-NEXT: vpshufhw {{.*#+}} xmm9 = xmm12[0,1,2,3,6,5,4,7]3334; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3],ymm0[4,5,6,7]3335; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3336; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload3337; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm0 = ymm1[2,3,0,1]3338; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7,8,9],ymm0[10],ymm1[11,12],ymm0[13],ymm1[14,15]3339; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm9 = [2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]3340; AVX2-FCP-NEXT: vpshufb %ymm9, %ymm0, %ymm03341; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm5[0,1],xmm7[2],xmm5[3,4],xmm7[5],xmm5[6,7]3342; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm1 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]3343; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm12, %xmm123344; AVX2-FCP-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm123345; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm0[0,1,2],ymm12[3,4,5,6,7],ymm0[8,9,10],ymm12[11,12,13,14,15]3346; AVX2-FCP-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,6,7,4]3347; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm12[4,5,6,7]3348; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm12 = ymm13[2,3,0,1]3349; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm13[0,1],ymm12[2],ymm13[3,4],ymm12[5],ymm13[6,7,8,9],ymm12[10],ymm13[11,12],ymm12[13],ymm13[14,15]3350; AVX2-FCP-NEXT: vpshufb %ymm9, %ymm12, %ymm123351; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm13 = xmm15[0,1],xmm4[2],xmm15[3,4],xmm4[5],xmm15[6,7]3352; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm13, %xmm133353; AVX2-FCP-NEXT: vinserti128 $1, %xmm13, %ymm0, %ymm133354; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm13 = ymm12[0,1,2],ymm13[3,4,5,6,7],ymm12[8,9,10],ymm13[11,12,13,14,15]3355; AVX2-FCP-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,5,6,7,4]3356; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm13 = ymm12[0,1,2,3],ymm13[4,5,6,7]3357; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm12 = ymm10[2,3,0,1]3358; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm12[2],ymm10[3,4],ymm12[5],ymm10[6,7,8,9],ymm12[10],ymm10[11,12],ymm12[13],ymm10[14,15]3359; AVX2-FCP-NEXT: vpshufb %ymm9, %ymm10, %ymm103360; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm8[0,1],xmm3[2],xmm8[3,4],xmm3[5],xmm8[6,7]3361; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm12, %xmm123362; AVX2-FCP-NEXT: vinserti128 $1, %xmm12, %ymm0, %ymm123363; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm10[0,1,2],ymm12[3,4,5,6,7],ymm10[8,9,10],ymm12[11,12,13,14,15]3364; AVX2-FCP-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,6,7,4]3365; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm12[4,5,6,7]3366; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm12 = ymm14[2,3,0,1]3367; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm12 = ymm14[0,1],ymm12[2],ymm14[3,4],ymm12[5],ymm14[6,7,8,9],ymm12[10],ymm14[11,12],ymm12[13],ymm14[14,15]3368; AVX2-FCP-NEXT: vpshufb %ymm9, %ymm12, %ymm93369; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm11[2],xmm2[3,4],xmm11[5],xmm2[6,7]3370; AVX2-FCP-NEXT: vpshufb %xmm1, %xmm12, %xmm13371; AVX2-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm13372; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm9[0,1,2],ymm1[3,4,5,6,7],ymm9[8,9,10],ymm1[11,12,13,14,15]3373; AVX2-FCP-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,5,6,7,4]3374; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm14 = ymm9[0,1,2,3],ymm1[4,5,6,7]3375; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload3376; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm9[2,3,0,1]3377; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm9[1,2],ymm1[3],ymm9[4,5],ymm1[6],ymm9[7],ymm1[8],ymm9[9,10],ymm1[11],ymm9[12,13],ymm1[14],ymm9[15]3378; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm5 = xmm7[0],xmm5[1],xmm7[2,3],xmm5[4],xmm7[5,6],xmm5[7]3379; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm7 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]3380; AVX2-FCP-NEXT: vpshufb %ymm7, %ymm1, %ymm13381; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm9 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]3382; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm5, %xmm53383; AVX2-FCP-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm53384; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm1[0,1,2,3,4],ymm5[5,6,7]3385; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload3386; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm12[2,3,0,1]3387; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm12[1,2],ymm1[3],ymm12[4,5],ymm1[6],ymm12[7],ymm1[8],ymm12[9,10],ymm1[11],ymm12[12,13],ymm1[14],ymm12[15]3388; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm15[1],xmm4[2,3],xmm15[4],xmm4[5,6],xmm15[7]3389; AVX2-FCP-NEXT: vpshufb %ymm7, %ymm1, %ymm13390; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm4, %xmm43391; AVX2-FCP-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm43392; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm4[5,6,7]3393; AVX2-FCP-NEXT: vmovdqu (%rsp), %ymm12 # 32-byte Reload3394; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm12[2,3,0,1]3395; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm12[1,2],ymm4[3],ymm12[4,5],ymm4[6],ymm12[7],ymm4[8],ymm12[9,10],ymm4[11],ymm12[12,13],ymm4[14],ymm12[15]3396; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm8[1],xmm3[2,3],xmm8[4],xmm3[5,6],xmm8[7]3397; AVX2-FCP-NEXT: vpshufb %ymm7, %ymm4, %ymm43398; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm3, %xmm33399; AVX2-FCP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm33400; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4],ymm3[5,6,7]3401; AVX2-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm6[2,3,0,1]3402; AVX2-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm6[1,2],ymm4[3],ymm6[4,5],ymm4[6],ymm6[7],ymm4[8],ymm6[9,10],ymm4[11],ymm6[12,13],ymm4[14],ymm6[15]3403; AVX2-FCP-NEXT: vpshufb %ymm7, %ymm4, %ymm43404; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm11[0],xmm2[1],xmm11[2,3],xmm2[4],xmm11[5,6],xmm2[7]3405; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm2, %xmm23406; AVX2-FCP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm23407; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4],ymm2[5,6,7]3408; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload3409; AVX2-FCP-NEXT: vmovaps %ymm4, (%rsi)3410; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload3411; AVX2-FCP-NEXT: vmovaps %ymm4, 64(%rsi)3412; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload3413; AVX2-FCP-NEXT: vmovaps %ymm4, 96(%rsi)3414; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload3415; AVX2-FCP-NEXT: vmovaps %ymm4, 32(%rsi)3416; AVX2-FCP-NEXT: vmovdqa %ymm10, 64(%rdx)3417; AVX2-FCP-NEXT: vmovdqa %ymm14, (%rdx)3418; AVX2-FCP-NEXT: vmovdqa %ymm13, 96(%rdx)3419; AVX2-FCP-NEXT: vmovdqa %ymm0, 32(%rdx)3420; AVX2-FCP-NEXT: vmovdqa %ymm3, 64(%rcx)3421; AVX2-FCP-NEXT: vmovdqa %ymm2, (%rcx)3422; AVX2-FCP-NEXT: vmovdqa %ymm1, 96(%rcx)3423; AVX2-FCP-NEXT: vmovdqa %ymm5, 32(%rcx)3424; AVX2-FCP-NEXT: addq $136, %rsp3425; AVX2-FCP-NEXT: vzeroupper3426; AVX2-FCP-NEXT: retq3427;3428; AVX512-LABEL: load_i16_stride3_vf64:3429; AVX512: # %bb.0:3430; AVX512-NEXT: vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]3431; AVX512-NEXT: vmovdqa64 224(%rdi), %ymm183432; AVX512-NEXT: vmovdqa64 192(%rdi), %ymm203433; AVX512-NEXT: vmovdqa %ymm0, %ymm13434; AVX512-NEXT: vpternlogq {{.*#+}} ymm1 = ymm18 ^ (ymm1 & (ymm20 ^ ymm18))3435; AVX512-NEXT: vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]3436; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]3437; AVX512-NEXT: vmovdqa {{.*#+}} ymm7 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]3438; AVX512-NEXT: vpshufb %ymm7, %ymm2, %ymm53439; AVX512-NEXT: vmovdqa 272(%rdi), %xmm13440; AVX512-NEXT: vmovdqa 256(%rdi), %xmm33441; AVX512-NEXT: vpblendw {{.*#+}} xmm6 = xmm3[0,1],xmm1[2],xmm3[3,4],xmm1[5],xmm3[6,7]3442; AVX512-NEXT: vmovdqa64 %xmm1, %xmm193443; AVX512-NEXT: vmovdqa {{.*#+}} xmm13 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]3444; AVX512-NEXT: vpshufb %xmm13, %xmm6, %xmm63445; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm63446; AVX512-NEXT: vpblendw {{.*#+}} ymm6 = ymm5[0,1,2],ymm6[3,4,5,6,7],ymm5[8,9,10],ymm6[11,12,13,14,15]3447; AVX512-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,5,4,7]3448; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]3449; AVX512-NEXT: vmovdqa64 320(%rdi), %ymm213450; AVX512-NEXT: vmovdqa64 352(%rdi), %ymm223451; AVX512-NEXT: vmovdqa %ymm0, %ymm83452; AVX512-NEXT: vpternlogq {{.*#+}} ymm8 = ymm21 ^ (ymm8 & (ymm22 ^ ymm21))3453; AVX512-NEXT: vpermq {{.*#+}} ymm9 = ymm8[2,3,0,1]3454; AVX512-NEXT: vpblendw {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5,6],ymm9[7],ymm8[8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13,14],ymm9[15]3455; AVX512-NEXT: vmovdqa {{.*#+}} ymm10 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]3456; AVX512-NEXT: vpshufb %ymm10, %ymm8, %ymm113457; AVX512-NEXT: vmovdqa 304(%rdi), %xmm83458; AVX512-NEXT: vmovdqa 288(%rdi), %xmm43459; AVX512-NEXT: vpblendw {{.*#+}} xmm12 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]3460; AVX512-NEXT: vmovdqa {{.*#+}} xmm14 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]3461; AVX512-NEXT: vpshufb %xmm14, %xmm12, %xmm123462; AVX512-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3,4,5,6,7]3463; AVX512-NEXT: vinserti64x4 $1, %ymm11, %zmm5, %zmm163464; AVX512-NEXT: vmovdqa64 128(%rdi), %ymm233465; AVX512-NEXT: vmovdqa 160(%rdi), %ymm113466; AVX512-NEXT: vmovdqa %ymm0, %ymm53467; AVX512-NEXT: vpternlogq {{.*#+}} ymm5 = ymm23 ^ (ymm5 & (ymm11 ^ ymm23))3468; AVX512-NEXT: vpermq {{.*#+}} ymm12 = ymm5[2,3,0,1]3469; AVX512-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1],ymm5[2,3],ymm12[4],ymm5[5,6],ymm12[7],ymm5[8],ymm12[9],ymm5[10,11],ymm12[12],ymm5[13,14],ymm12[15]3470; AVX512-NEXT: vpshufb %ymm10, %ymm5, %ymm103471; AVX512-NEXT: vmovdqa 112(%rdi), %xmm153472; AVX512-NEXT: vmovdqa 96(%rdi), %xmm53473; AVX512-NEXT: vpblendw {{.*#+}} xmm12 = xmm5[0],xmm15[1],xmm5[2,3],xmm15[4],xmm5[5,6],xmm15[7]3474; AVX512-NEXT: vpshufb %xmm14, %xmm12, %xmm123475; AVX512-NEXT: vpblendd {{.*#+}} ymm6 = ymm12[0,1,2],ymm10[3,4,5,6,7]3476; AVX512-NEXT: vmovdqa64 (%rdi), %ymm243477; AVX512-NEXT: vmovdqa 32(%rdi), %ymm123478; AVX512-NEXT: vmovdqa %ymm0, %ymm103479; AVX512-NEXT: vpternlogq {{.*#+}} ymm10 = ymm12 ^ (ymm10 & (ymm24 ^ ymm12))3480; AVX512-NEXT: vpermq {{.*#+}} ymm1 = ymm10[2,3,0,1]3481; AVX512-NEXT: vpblendw {{.*#+}} ymm1 = ymm10[0],ymm1[1],ymm10[2,3],ymm1[4],ymm10[5,6],ymm1[7],ymm10[8],ymm1[9],ymm10[10,11],ymm1[12],ymm10[13,14],ymm1[15]3482; AVX512-NEXT: vpshufb %ymm7, %ymm1, %ymm73483; AVX512-NEXT: vmovdqa 80(%rdi), %xmm103484; AVX512-NEXT: vmovdqa 64(%rdi), %xmm13485; AVX512-NEXT: vpblendw {{.*#+}} xmm2 = xmm1[0,1],xmm10[2],xmm1[3,4],xmm10[5],xmm1[6,7]3486; AVX512-NEXT: vpshufb %xmm13, %xmm2, %xmm23487; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm23488; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm7[0,1,2],ymm2[3,4,5,6,7],ymm7[8,9,10],ymm2[11,12,13,14,15]3489; AVX512-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,6,5,4,7]3490; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]3491; AVX512-NEXT: vinserti64x4 $1, %ymm6, %zmm2, %zmm173492; AVX512-NEXT: vmovdqa %ymm0, %ymm23493; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm22 ^ (ymm2 & (ymm21 ^ ymm22))3494; AVX512-NEXT: vpermq {{.*#+}} ymm6 = ymm2[2,3,0,1]3495; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm6[2],ymm2[3,4],ymm6[5],ymm2[6,7,8,9],ymm6[10],ymm2[11,12],ymm6[13],ymm2[14,15]3496; AVX512-NEXT: vmovdqa {{.*#+}} ymm9 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]3497; AVX512-NEXT: vpshufb %ymm9, %ymm2, %ymm23498; AVX512-NEXT: vpblendw {{.*#+}} xmm7 = xmm4[0,1],xmm8[2],xmm4[3,4],xmm8[5],xmm4[6,7]3499; AVX512-NEXT: vmovdqa64 %xmm8, %xmm253500; AVX512-NEXT: vmovdqa64 %xmm4, %xmm263501; AVX512-NEXT: vmovdqa {{.*#+}} xmm6 = [2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]3502; AVX512-NEXT: vpshufb %xmm6, %xmm7, %xmm73503; AVX512-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4],xmm2[5,6,7]3504; AVX512-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm2[4,5,6,7]3505; AVX512-NEXT: vmovdqa {{.*#+}} ymm13 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]3506; AVX512-NEXT: vmovdqa %ymm13, %ymm23507; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm20 ^ (ymm2 & (ymm18 ^ ymm20))3508; AVX512-NEXT: vpermq {{.*#+}} ymm4 = ymm2[2,3,0,1]3509; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]3510; AVX512-NEXT: vmovdqa {{.*#+}} ymm4 = [2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]3511; AVX512-NEXT: vpshufb %ymm4, %ymm2, %ymm23512; AVX512-NEXT: vmovdqa64 %xmm19, %xmm83513; AVX512-NEXT: vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm3[2],xmm8[3,4],xmm3[5],xmm8[6,7]3514; AVX512-NEXT: vmovdqa64 %xmm3, %xmm273515; AVX512-NEXT: vmovdqa {{.*#+}} xmm3 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]3516; AVX512-NEXT: vpshufb %xmm3, %xmm14, %xmm143517; AVX512-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm143518; AVX512-NEXT: vpblendw {{.*#+}} ymm14 = ymm2[0,1,2],ymm14[3,4,5,6,7],ymm2[8,9,10],ymm14[11,12,13,14,15]3519; AVX512-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,6,7,4]3520; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm14[4,5,6,7]3521; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm2, %zmm193522; AVX512-NEXT: vmovdqa %ymm0, %ymm23523; AVX512-NEXT: vpternlogq {{.*#+}} ymm2 = ymm11 ^ (ymm2 & (ymm23 ^ ymm11))3524; AVX512-NEXT: vpermq {{.*#+}} ymm7 = ymm2[2,3,0,1]3525; AVX512-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm7[2],ymm2[3,4],ymm7[5],ymm2[6,7,8,9],ymm7[10],ymm2[11,12],ymm7[13],ymm2[14,15]3526; AVX512-NEXT: vpshufb %ymm9, %ymm2, %ymm23527; AVX512-NEXT: vpblendw {{.*#+}} xmm7 = xmm5[0,1],xmm15[2],xmm5[3,4],xmm15[5],xmm5[6,7]3528; AVX512-NEXT: vpshufb %xmm6, %xmm7, %xmm63529; AVX512-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm2[5,6,7]3530; AVX512-NEXT: vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]3531; AVX512-NEXT: vmovdqa %ymm13, %ymm63532; AVX512-NEXT: vpternlogq {{.*#+}} ymm6 = ymm24 ^ (ymm6 & (ymm12 ^ ymm24))3533; AVX512-NEXT: vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]3534; AVX512-NEXT: vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7,8,9],ymm7[10],ymm6[11,12],ymm7[13],ymm6[14,15]3535; AVX512-NEXT: vpshufb %ymm4, %ymm6, %ymm43536; AVX512-NEXT: vpblendw {{.*#+}} xmm6 = xmm10[0,1],xmm1[2],xmm10[3,4],xmm1[5],xmm10[6,7]3537; AVX512-NEXT: vpshufb %xmm3, %xmm6, %xmm33538; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm33539; AVX512-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7],ymm4[8,9,10],ymm3[11,12,13,14,15]3540; AVX512-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,7,4]3541; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]3542; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm23543; AVX512-NEXT: vpternlogq {{.*#+}} ymm11 = ymm23 ^ (ymm13 & (ymm11 ^ ymm23))3544; AVX512-NEXT: vpermq {{.*#+}} ymm3 = ymm11[2,3,0,1]3545; AVX512-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm11[1,2],ymm3[3],ymm11[4,5],ymm3[6],ymm11[7],ymm3[8],ymm11[9,10],ymm3[11],ymm11[12,13],ymm3[14],ymm11[15]3546; AVX512-NEXT: vmovdqa {{.*#+}} ymm11 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]3547; AVX512-NEXT: vpshufb %ymm11, %ymm3, %ymm33548; AVX512-NEXT: vpblendw {{.*#+}} xmm4 = xmm15[0,1],xmm5[2],xmm15[3,4],xmm5[5],xmm15[6,7]3549; AVX512-NEXT: vmovdqa {{.*#+}} xmm5 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]3550; AVX512-NEXT: vpshufb %xmm5, %xmm4, %xmm43551; AVX512-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]3552; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]3553; AVX512-NEXT: vpternlogq {{.*#+}} ymm12 = ymm24 ^ (ymm0 & (ymm12 ^ ymm24))3554; AVX512-NEXT: vpermq {{.*#+}} ymm4 = ymm12[2,3,0,1]3555; AVX512-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm12[1,2],ymm4[3],ymm12[4,5],ymm4[6],ymm12[7],ymm4[8],ymm12[9,10],ymm4[11],ymm12[12,13],ymm4[14],ymm12[15]3556; AVX512-NEXT: vpshufb %ymm11, %ymm4, %ymm43557; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm10[1],xmm1[2,3],xmm10[4],xmm1[5,6],xmm10[7]3558; AVX512-NEXT: vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]3559; AVX512-NEXT: vpshufb %xmm6, %xmm1, %xmm13560; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm13561; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4],ymm1[5,6,7]3562; AVX512-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm13563; AVX512-NEXT: vpternlogq {{.*#+}} ymm13 = ymm21 ^ (ymm13 & (ymm22 ^ ymm21))3564; AVX512-NEXT: vpermq {{.*#+}} ymm3 = ymm13[2,3,0,1]3565; AVX512-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm13[1,2],ymm3[3],ymm13[4,5],ymm3[6],ymm13[7],ymm3[8],ymm13[9,10],ymm3[11],ymm13[12,13],ymm3[14],ymm13[15]3566; AVX512-NEXT: vmovdqa64 %xmm25, %xmm43567; AVX512-NEXT: vmovdqa64 %xmm26, %xmm73568; AVX512-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm7[2],xmm4[3,4],xmm7[5],xmm4[6,7]3569; AVX512-NEXT: vpshufb %xmm5, %xmm4, %xmm43570; AVX512-NEXT: vpshufb %ymm11, %ymm3, %ymm33571; AVX512-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]3572; AVX512-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]3573; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm20 ^ (ymm0 & (ymm18 ^ ymm20))3574; AVX512-NEXT: vpermq {{.*#+}} ymm4 = ymm0[2,3,0,1]3575; AVX512-NEXT: vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1,2],ymm4[3],ymm0[4,5],ymm4[6],ymm0[7],ymm4[8],ymm0[9,10],ymm4[11],ymm0[12,13],ymm4[14],ymm0[15]3576; AVX512-NEXT: vpshufb %ymm11, %ymm0, %ymm03577; AVX512-NEXT: vmovdqa64 %xmm27, %xmm43578; AVX512-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]3579; AVX512-NEXT: vpshufb %xmm6, %xmm4, %xmm43580; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm43581; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7]3582; AVX512-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm03583; AVX512-NEXT: vmovdqa64 %zmm17, (%rsi)3584; AVX512-NEXT: vmovdqa64 %zmm16, 64(%rsi)3585; AVX512-NEXT: vmovdqa64 %zmm19, 64(%rdx)3586; AVX512-NEXT: vmovdqa64 %zmm2, (%rdx)3587; AVX512-NEXT: vmovdqa64 %zmm0, 64(%rcx)3588; AVX512-NEXT: vmovdqa64 %zmm1, (%rcx)3589; AVX512-NEXT: vzeroupper3590; AVX512-NEXT: retq3591;3592; AVX512-FCP-LABEL: load_i16_stride3_vf64:3593; AVX512-FCP: # %bb.0:3594; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]3595; AVX512-FCP-NEXT: vmovdqa64 224(%rdi), %ymm183596; AVX512-FCP-NEXT: vmovdqa64 192(%rdi), %ymm203597; AVX512-FCP-NEXT: vmovdqa %ymm0, %ymm13598; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm18 ^ (ymm1 & (ymm20 ^ ymm18))3599; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]3600; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]3601; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm7 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]3602; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm2, %ymm53603; AVX512-FCP-NEXT: vmovdqa 272(%rdi), %xmm13604; AVX512-FCP-NEXT: vmovdqa 256(%rdi), %xmm33605; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm3[0,1],xmm1[2],xmm3[3,4],xmm1[5],xmm3[6,7]3606; AVX512-FCP-NEXT: vmovdqa64 %xmm1, %xmm193607; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm13 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]3608; AVX512-FCP-NEXT: vpshufb %xmm13, %xmm6, %xmm63609; AVX512-FCP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm63610; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm5[0,1,2],ymm6[3,4,5,6,7],ymm5[8,9,10],ymm6[11,12,13,14,15]3611; AVX512-FCP-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,5,4,7]3612; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]3613; AVX512-FCP-NEXT: vmovdqa64 320(%rdi), %ymm213614; AVX512-FCP-NEXT: vmovdqa64 352(%rdi), %ymm223615; AVX512-FCP-NEXT: vmovdqa %ymm0, %ymm83616; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm8 = ymm21 ^ (ymm8 & (ymm22 ^ ymm21))3617; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm9 = ymm8[2,3,0,1]3618; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5,6],ymm9[7],ymm8[8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13,14],ymm9[15]3619; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm10 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]3620; AVX512-FCP-NEXT: vpshufb %ymm10, %ymm8, %ymm113621; AVX512-FCP-NEXT: vmovdqa 304(%rdi), %xmm83622; AVX512-FCP-NEXT: vmovdqa 288(%rdi), %xmm43623; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]3624; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm14 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]3625; AVX512-FCP-NEXT: vpshufb %xmm14, %xmm12, %xmm123626; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3,4,5,6,7]3627; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm11, %zmm5, %zmm163628; AVX512-FCP-NEXT: vmovdqa64 128(%rdi), %ymm233629; AVX512-FCP-NEXT: vmovdqa 160(%rdi), %ymm113630; AVX512-FCP-NEXT: vmovdqa %ymm0, %ymm53631; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm5 = ymm23 ^ (ymm5 & (ymm11 ^ ymm23))3632; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm12 = ymm5[2,3,0,1]3633; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1],ymm5[2,3],ymm12[4],ymm5[5,6],ymm12[7],ymm5[8],ymm12[9],ymm5[10,11],ymm12[12],ymm5[13,14],ymm12[15]3634; AVX512-FCP-NEXT: vpshufb %ymm10, %ymm5, %ymm103635; AVX512-FCP-NEXT: vmovdqa 112(%rdi), %xmm153636; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %xmm53637; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm5[0],xmm15[1],xmm5[2,3],xmm15[4],xmm5[5,6],xmm15[7]3638; AVX512-FCP-NEXT: vpshufb %xmm14, %xmm12, %xmm123639; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm12[0,1,2],ymm10[3,4,5,6,7]3640; AVX512-FCP-NEXT: vmovdqa64 (%rdi), %ymm243641; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm123642; AVX512-FCP-NEXT: vmovdqa %ymm0, %ymm103643; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm10 = ymm12 ^ (ymm10 & (ymm24 ^ ymm12))3644; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm10[2,3,0,1]3645; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm10[0],ymm1[1],ymm10[2,3],ymm1[4],ymm10[5,6],ymm1[7],ymm10[8],ymm1[9],ymm10[10,11],ymm1[12],ymm10[13,14],ymm1[15]3646; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm1, %ymm73647; AVX512-FCP-NEXT: vmovdqa 80(%rdi), %xmm103648; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %xmm13649; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm1[0,1],xmm10[2],xmm1[3,4],xmm10[5],xmm1[6,7]3650; AVX512-FCP-NEXT: vpshufb %xmm13, %xmm2, %xmm23651; AVX512-FCP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm23652; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm7[0,1,2],ymm2[3,4,5,6,7],ymm7[8,9,10],ymm2[11,12,13,14,15]3653; AVX512-FCP-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,6,5,4,7]3654; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]3655; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm6, %zmm2, %zmm173656; AVX512-FCP-NEXT: vmovdqa %ymm0, %ymm23657; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm22 ^ (ymm2 & (ymm21 ^ ymm22))3658; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm6 = ymm2[2,3,0,1]3659; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm6[2],ymm2[3,4],ymm6[5],ymm2[6,7,8,9],ymm6[10],ymm2[11,12],ymm6[13],ymm2[14,15]3660; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm9 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]3661; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm2, %ymm23662; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm7 = xmm4[0,1],xmm8[2],xmm4[3,4],xmm8[5],xmm4[6,7]3663; AVX512-FCP-NEXT: vmovdqa64 %xmm8, %xmm253664; AVX512-FCP-NEXT: vmovdqa64 %xmm4, %xmm263665; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm6 = [2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]3666; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm7, %xmm73667; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4],xmm2[5,6,7]3668; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm2[4,5,6,7]3669; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm13 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]3670; AVX512-FCP-NEXT: vmovdqa %ymm13, %ymm23671; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm20 ^ (ymm2 & (ymm18 ^ ymm20))3672; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm2[2,3,0,1]3673; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]3674; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]3675; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm2, %ymm23676; AVX512-FCP-NEXT: vmovdqa64 %xmm19, %xmm83677; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm3[2],xmm8[3,4],xmm3[5],xmm8[6,7]3678; AVX512-FCP-NEXT: vmovdqa64 %xmm3, %xmm273679; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm3 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]3680; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm14, %xmm143681; AVX512-FCP-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm143682; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm14 = ymm2[0,1,2],ymm14[3,4,5,6,7],ymm2[8,9,10],ymm14[11,12,13,14,15]3683; AVX512-FCP-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,6,7,4]3684; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm14[4,5,6,7]3685; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm2, %zmm193686; AVX512-FCP-NEXT: vmovdqa %ymm0, %ymm23687; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm11 ^ (ymm2 & (ymm23 ^ ymm11))3688; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm7 = ymm2[2,3,0,1]3689; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm7[2],ymm2[3,4],ymm7[5],ymm2[6,7,8,9],ymm7[10],ymm2[11,12],ymm7[13],ymm2[14,15]3690; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm2, %ymm23691; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm7 = xmm5[0,1],xmm15[2],xmm5[3,4],xmm15[5],xmm5[6,7]3692; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm7, %xmm63693; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm2[5,6,7]3694; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]3695; AVX512-FCP-NEXT: vmovdqa %ymm13, %ymm63696; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm6 = ymm24 ^ (ymm6 & (ymm12 ^ ymm24))3697; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]3698; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7,8,9],ymm7[10],ymm6[11,12],ymm7[13],ymm6[14,15]3699; AVX512-FCP-NEXT: vpshufb %ymm4, %ymm6, %ymm43700; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm10[0,1],xmm1[2],xmm10[3,4],xmm1[5],xmm10[6,7]3701; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm6, %xmm33702; AVX512-FCP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm33703; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7],ymm4[8,9,10],ymm3[11,12,13,14,15]3704; AVX512-FCP-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,7,4]3705; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]3706; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm23707; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm23 ^ (ymm13 & (ymm11 ^ ymm23))3708; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm11[2,3,0,1]3709; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm11[1,2],ymm3[3],ymm11[4,5],ymm3[6],ymm11[7],ymm3[8],ymm11[9,10],ymm3[11],ymm11[12,13],ymm3[14],ymm11[15]3710; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm11 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]3711; AVX512-FCP-NEXT: vpshufb %ymm11, %ymm3, %ymm33712; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm15[0,1],xmm5[2],xmm15[3,4],xmm5[5],xmm15[6,7]3713; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm5 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]3714; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm4, %xmm43715; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]3716; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]3717; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm12 = ymm24 ^ (ymm0 & (ymm12 ^ ymm24))3718; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm12[2,3,0,1]3719; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm12[1,2],ymm4[3],ymm12[4,5],ymm4[6],ymm12[7],ymm4[8],ymm12[9,10],ymm4[11],ymm12[12,13],ymm4[14],ymm12[15]3720; AVX512-FCP-NEXT: vpshufb %ymm11, %ymm4, %ymm43721; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm10[1],xmm1[2,3],xmm10[4],xmm1[5,6],xmm10[7]3722; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]3723; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm1, %xmm13724; AVX512-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm13725; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4],ymm1[5,6,7]3726; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm13727; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm13 = ymm21 ^ (ymm13 & (ymm22 ^ ymm21))3728; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm13[2,3,0,1]3729; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm13[1,2],ymm3[3],ymm13[4,5],ymm3[6],ymm13[7],ymm3[8],ymm13[9,10],ymm3[11],ymm13[12,13],ymm3[14],ymm13[15]3730; AVX512-FCP-NEXT: vmovdqa64 %xmm25, %xmm43731; AVX512-FCP-NEXT: vmovdqa64 %xmm26, %xmm73732; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm7[2],xmm4[3,4],xmm7[5],xmm4[6,7]3733; AVX512-FCP-NEXT: vpshufb %xmm5, %xmm4, %xmm43734; AVX512-FCP-NEXT: vpshufb %ymm11, %ymm3, %ymm33735; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]3736; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]3737; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm20 ^ (ymm0 & (ymm18 ^ ymm20))3738; AVX512-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm0[2,3,0,1]3739; AVX512-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1,2],ymm4[3],ymm0[4,5],ymm4[6],ymm0[7],ymm4[8],ymm0[9,10],ymm4[11],ymm0[12,13],ymm4[14],ymm0[15]3740; AVX512-FCP-NEXT: vpshufb %ymm11, %ymm0, %ymm03741; AVX512-FCP-NEXT: vmovdqa64 %xmm27, %xmm43742; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]3743; AVX512-FCP-NEXT: vpshufb %xmm6, %xmm4, %xmm43744; AVX512-FCP-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm43745; AVX512-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7]3746; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm03747; AVX512-FCP-NEXT: vmovdqa64 %zmm17, (%rsi)3748; AVX512-FCP-NEXT: vmovdqa64 %zmm16, 64(%rsi)3749; AVX512-FCP-NEXT: vmovdqa64 %zmm19, 64(%rdx)3750; AVX512-FCP-NEXT: vmovdqa64 %zmm2, (%rdx)3751; AVX512-FCP-NEXT: vmovdqa64 %zmm0, 64(%rcx)3752; AVX512-FCP-NEXT: vmovdqa64 %zmm1, (%rcx)3753; AVX512-FCP-NEXT: vzeroupper3754; AVX512-FCP-NEXT: retq3755;3756; AVX512DQ-LABEL: load_i16_stride3_vf64:3757; AVX512DQ: # %bb.0:3758; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]3759; AVX512DQ-NEXT: vmovdqa64 224(%rdi), %ymm183760; AVX512DQ-NEXT: vmovdqa64 192(%rdi), %ymm203761; AVX512DQ-NEXT: vmovdqa %ymm0, %ymm13762; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm1 = ymm18 ^ (ymm1 & (ymm20 ^ ymm18))3763; AVX512DQ-NEXT: vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]3764; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]3765; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm7 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]3766; AVX512DQ-NEXT: vpshufb %ymm7, %ymm2, %ymm53767; AVX512DQ-NEXT: vmovdqa 272(%rdi), %xmm13768; AVX512DQ-NEXT: vmovdqa 256(%rdi), %xmm33769; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm6 = xmm3[0,1],xmm1[2],xmm3[3,4],xmm1[5],xmm3[6,7]3770; AVX512DQ-NEXT: vmovdqa64 %xmm1, %xmm193771; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm13 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]3772; AVX512DQ-NEXT: vpshufb %xmm13, %xmm6, %xmm63773; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm63774; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm6 = ymm5[0,1,2],ymm6[3,4,5,6,7],ymm5[8,9,10],ymm6[11,12,13,14,15]3775; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,5,4,7]3776; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]3777; AVX512DQ-NEXT: vmovdqa64 320(%rdi), %ymm213778; AVX512DQ-NEXT: vmovdqa64 352(%rdi), %ymm223779; AVX512DQ-NEXT: vmovdqa %ymm0, %ymm83780; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm8 = ymm21 ^ (ymm8 & (ymm22 ^ ymm21))3781; AVX512DQ-NEXT: vpermq {{.*#+}} ymm9 = ymm8[2,3,0,1]3782; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5,6],ymm9[7],ymm8[8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13,14],ymm9[15]3783; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm10 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]3784; AVX512DQ-NEXT: vpshufb %ymm10, %ymm8, %ymm113785; AVX512DQ-NEXT: vmovdqa 304(%rdi), %xmm83786; AVX512DQ-NEXT: vmovdqa 288(%rdi), %xmm43787; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm12 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]3788; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm14 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]3789; AVX512DQ-NEXT: vpshufb %xmm14, %xmm12, %xmm123790; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3,4,5,6,7]3791; AVX512DQ-NEXT: vinserti64x4 $1, %ymm11, %zmm5, %zmm163792; AVX512DQ-NEXT: vmovdqa64 128(%rdi), %ymm233793; AVX512DQ-NEXT: vmovdqa 160(%rdi), %ymm113794; AVX512DQ-NEXT: vmovdqa %ymm0, %ymm53795; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm5 = ymm23 ^ (ymm5 & (ymm11 ^ ymm23))3796; AVX512DQ-NEXT: vpermq {{.*#+}} ymm12 = ymm5[2,3,0,1]3797; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1],ymm5[2,3],ymm12[4],ymm5[5,6],ymm12[7],ymm5[8],ymm12[9],ymm5[10,11],ymm12[12],ymm5[13,14],ymm12[15]3798; AVX512DQ-NEXT: vpshufb %ymm10, %ymm5, %ymm103799; AVX512DQ-NEXT: vmovdqa 112(%rdi), %xmm153800; AVX512DQ-NEXT: vmovdqa 96(%rdi), %xmm53801; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm12 = xmm5[0],xmm15[1],xmm5[2,3],xmm15[4],xmm5[5,6],xmm15[7]3802; AVX512DQ-NEXT: vpshufb %xmm14, %xmm12, %xmm123803; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm6 = ymm12[0,1,2],ymm10[3,4,5,6,7]3804; AVX512DQ-NEXT: vmovdqa64 (%rdi), %ymm243805; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm123806; AVX512DQ-NEXT: vmovdqa %ymm0, %ymm103807; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm10 = ymm12 ^ (ymm10 & (ymm24 ^ ymm12))3808; AVX512DQ-NEXT: vpermq {{.*#+}} ymm1 = ymm10[2,3,0,1]3809; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm10[0],ymm1[1],ymm10[2,3],ymm1[4],ymm10[5,6],ymm1[7],ymm10[8],ymm1[9],ymm10[10,11],ymm1[12],ymm10[13,14],ymm1[15]3810; AVX512DQ-NEXT: vpshufb %ymm7, %ymm1, %ymm73811; AVX512DQ-NEXT: vmovdqa 80(%rdi), %xmm103812; AVX512DQ-NEXT: vmovdqa 64(%rdi), %xmm13813; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm1[0,1],xmm10[2],xmm1[3,4],xmm10[5],xmm1[6,7]3814; AVX512DQ-NEXT: vpshufb %xmm13, %xmm2, %xmm23815; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm23816; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm7[0,1,2],ymm2[3,4,5,6,7],ymm7[8,9,10],ymm2[11,12,13,14,15]3817; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,6,5,4,7]3818; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]3819; AVX512DQ-NEXT: vinserti64x4 $1, %ymm6, %zmm2, %zmm173820; AVX512DQ-NEXT: vmovdqa %ymm0, %ymm23821; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = ymm22 ^ (ymm2 & (ymm21 ^ ymm22))3822; AVX512DQ-NEXT: vpermq {{.*#+}} ymm6 = ymm2[2,3,0,1]3823; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm6[2],ymm2[3,4],ymm6[5],ymm2[6,7,8,9],ymm6[10],ymm2[11,12],ymm6[13],ymm2[14,15]3824; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm9 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]3825; AVX512DQ-NEXT: vpshufb %ymm9, %ymm2, %ymm23826; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm7 = xmm4[0,1],xmm8[2],xmm4[3,4],xmm8[5],xmm4[6,7]3827; AVX512DQ-NEXT: vmovdqa64 %xmm8, %xmm253828; AVX512DQ-NEXT: vmovdqa64 %xmm4, %xmm263829; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm6 = [2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]3830; AVX512DQ-NEXT: vpshufb %xmm6, %xmm7, %xmm73831; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4],xmm2[5,6,7]3832; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm2[4,5,6,7]3833; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm13 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]3834; AVX512DQ-NEXT: vmovdqa %ymm13, %ymm23835; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = ymm20 ^ (ymm2 & (ymm18 ^ ymm20))3836; AVX512DQ-NEXT: vpermq {{.*#+}} ymm4 = ymm2[2,3,0,1]3837; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]3838; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm4 = [2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]3839; AVX512DQ-NEXT: vpshufb %ymm4, %ymm2, %ymm23840; AVX512DQ-NEXT: vmovdqa64 %xmm19, %xmm83841; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm3[2],xmm8[3,4],xmm3[5],xmm8[6,7]3842; AVX512DQ-NEXT: vmovdqa64 %xmm3, %xmm273843; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm3 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]3844; AVX512DQ-NEXT: vpshufb %xmm3, %xmm14, %xmm143845; AVX512DQ-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm143846; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm14 = ymm2[0,1,2],ymm14[3,4,5,6,7],ymm2[8,9,10],ymm14[11,12,13,14,15]3847; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,6,7,4]3848; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm14[4,5,6,7]3849; AVX512DQ-NEXT: vinserti64x4 $1, %ymm7, %zmm2, %zmm193850; AVX512DQ-NEXT: vmovdqa %ymm0, %ymm23851; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm2 = ymm11 ^ (ymm2 & (ymm23 ^ ymm11))3852; AVX512DQ-NEXT: vpermq {{.*#+}} ymm7 = ymm2[2,3,0,1]3853; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm7[2],ymm2[3,4],ymm7[5],ymm2[6,7,8,9],ymm7[10],ymm2[11,12],ymm7[13],ymm2[14,15]3854; AVX512DQ-NEXT: vpshufb %ymm9, %ymm2, %ymm23855; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm7 = xmm5[0,1],xmm15[2],xmm5[3,4],xmm15[5],xmm5[6,7]3856; AVX512DQ-NEXT: vpshufb %xmm6, %xmm7, %xmm63857; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm2[5,6,7]3858; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]3859; AVX512DQ-NEXT: vmovdqa %ymm13, %ymm63860; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm6 = ymm24 ^ (ymm6 & (ymm12 ^ ymm24))3861; AVX512DQ-NEXT: vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]3862; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7,8,9],ymm7[10],ymm6[11,12],ymm7[13],ymm6[14,15]3863; AVX512DQ-NEXT: vpshufb %ymm4, %ymm6, %ymm43864; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm6 = xmm10[0,1],xmm1[2],xmm10[3,4],xmm1[5],xmm10[6,7]3865; AVX512DQ-NEXT: vpshufb %xmm3, %xmm6, %xmm33866; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm33867; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7],ymm4[8,9,10],ymm3[11,12,13,14,15]3868; AVX512DQ-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,7,4]3869; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]3870; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm23871; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm11 = ymm23 ^ (ymm13 & (ymm11 ^ ymm23))3872; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = ymm11[2,3,0,1]3873; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm11[1,2],ymm3[3],ymm11[4,5],ymm3[6],ymm11[7],ymm3[8],ymm11[9,10],ymm3[11],ymm11[12,13],ymm3[14],ymm11[15]3874; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm11 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]3875; AVX512DQ-NEXT: vpshufb %ymm11, %ymm3, %ymm33876; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm4 = xmm15[0,1],xmm5[2],xmm15[3,4],xmm5[5],xmm15[6,7]3877; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm5 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]3878; AVX512DQ-NEXT: vpshufb %xmm5, %xmm4, %xmm43879; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]3880; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]3881; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm12 = ymm24 ^ (ymm0 & (ymm12 ^ ymm24))3882; AVX512DQ-NEXT: vpermq {{.*#+}} ymm4 = ymm12[2,3,0,1]3883; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm12[1,2],ymm4[3],ymm12[4,5],ymm4[6],ymm12[7],ymm4[8],ymm12[9,10],ymm4[11],ymm12[12,13],ymm4[14],ymm12[15]3884; AVX512DQ-NEXT: vpshufb %ymm11, %ymm4, %ymm43885; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm10[1],xmm1[2,3],xmm10[4],xmm1[5,6],xmm10[7]3886; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]3887; AVX512DQ-NEXT: vpshufb %xmm6, %xmm1, %xmm13888; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm13889; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4],ymm1[5,6,7]3890; AVX512DQ-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm13891; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm13 = ymm21 ^ (ymm13 & (ymm22 ^ ymm21))3892; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = ymm13[2,3,0,1]3893; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm13[1,2],ymm3[3],ymm13[4,5],ymm3[6],ymm13[7],ymm3[8],ymm13[9,10],ymm3[11],ymm13[12,13],ymm3[14],ymm13[15]3894; AVX512DQ-NEXT: vmovdqa64 %xmm25, %xmm43895; AVX512DQ-NEXT: vmovdqa64 %xmm26, %xmm73896; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm7[2],xmm4[3,4],xmm7[5],xmm4[6,7]3897; AVX512DQ-NEXT: vpshufb %xmm5, %xmm4, %xmm43898; AVX512DQ-NEXT: vpshufb %ymm11, %ymm3, %ymm33899; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]3900; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]3901; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm0 = ymm20 ^ (ymm0 & (ymm18 ^ ymm20))3902; AVX512DQ-NEXT: vpermq {{.*#+}} ymm4 = ymm0[2,3,0,1]3903; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1,2],ymm4[3],ymm0[4,5],ymm4[6],ymm0[7],ymm4[8],ymm0[9,10],ymm4[11],ymm0[12,13],ymm4[14],ymm0[15]3904; AVX512DQ-NEXT: vpshufb %ymm11, %ymm0, %ymm03905; AVX512DQ-NEXT: vmovdqa64 %xmm27, %xmm43906; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]3907; AVX512DQ-NEXT: vpshufb %xmm6, %xmm4, %xmm43908; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm43909; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7]3910; AVX512DQ-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm03911; AVX512DQ-NEXT: vmovdqa64 %zmm17, (%rsi)3912; AVX512DQ-NEXT: vmovdqa64 %zmm16, 64(%rsi)3913; AVX512DQ-NEXT: vmovdqa64 %zmm19, 64(%rdx)3914; AVX512DQ-NEXT: vmovdqa64 %zmm2, (%rdx)3915; AVX512DQ-NEXT: vmovdqa64 %zmm0, 64(%rcx)3916; AVX512DQ-NEXT: vmovdqa64 %zmm1, (%rcx)3917; AVX512DQ-NEXT: vzeroupper3918; AVX512DQ-NEXT: retq3919;3920; AVX512DQ-FCP-LABEL: load_i16_stride3_vf64:3921; AVX512DQ-FCP: # %bb.0:3922; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]3923; AVX512DQ-FCP-NEXT: vmovdqa64 224(%rdi), %ymm183924; AVX512DQ-FCP-NEXT: vmovdqa64 192(%rdi), %ymm203925; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, %ymm13926; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm1 = ymm18 ^ (ymm1 & (ymm20 ^ ymm18))3927; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]3928; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]3929; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm7 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]3930; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm2, %ymm53931; AVX512DQ-FCP-NEXT: vmovdqa 272(%rdi), %xmm13932; AVX512DQ-FCP-NEXT: vmovdqa 256(%rdi), %xmm33933; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm3[0,1],xmm1[2],xmm3[3,4],xmm1[5],xmm3[6,7]3934; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, %xmm193935; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm13 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]3936; AVX512DQ-FCP-NEXT: vpshufb %xmm13, %xmm6, %xmm63937; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm6, %ymm0, %ymm63938; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm5[0,1,2],ymm6[3,4,5,6,7],ymm5[8,9,10],ymm6[11,12,13,14,15]3939; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,5,4,7]3940; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]3941; AVX512DQ-FCP-NEXT: vmovdqa64 320(%rdi), %ymm213942; AVX512DQ-FCP-NEXT: vmovdqa64 352(%rdi), %ymm223943; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, %ymm83944; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm8 = ymm21 ^ (ymm8 & (ymm22 ^ ymm21))3945; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm9 = ymm8[2,3,0,1]3946; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5,6],ymm9[7],ymm8[8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13,14],ymm9[15]3947; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm10 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]3948; AVX512DQ-FCP-NEXT: vpshufb %ymm10, %ymm8, %ymm113949; AVX512DQ-FCP-NEXT: vmovdqa 304(%rdi), %xmm83950; AVX512DQ-FCP-NEXT: vmovdqa 288(%rdi), %xmm43951; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]3952; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm14 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]3953; AVX512DQ-FCP-NEXT: vpshufb %xmm14, %xmm12, %xmm123954; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3,4,5,6,7]3955; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm11, %zmm5, %zmm163956; AVX512DQ-FCP-NEXT: vmovdqa64 128(%rdi), %ymm233957; AVX512DQ-FCP-NEXT: vmovdqa 160(%rdi), %ymm113958; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, %ymm53959; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm5 = ymm23 ^ (ymm5 & (ymm11 ^ ymm23))3960; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm12 = ymm5[2,3,0,1]3961; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1],ymm5[2,3],ymm12[4],ymm5[5,6],ymm12[7],ymm5[8],ymm12[9],ymm5[10,11],ymm12[12],ymm5[13,14],ymm12[15]3962; AVX512DQ-FCP-NEXT: vpshufb %ymm10, %ymm5, %ymm103963; AVX512DQ-FCP-NEXT: vmovdqa 112(%rdi), %xmm153964; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %xmm53965; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm5[0],xmm15[1],xmm5[2,3],xmm15[4],xmm5[5,6],xmm15[7]3966; AVX512DQ-FCP-NEXT: vpshufb %xmm14, %xmm12, %xmm123967; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm12[0,1,2],ymm10[3,4,5,6,7]3968; AVX512DQ-FCP-NEXT: vmovdqa64 (%rdi), %ymm243969; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm123970; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, %ymm103971; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm10 = ymm12 ^ (ymm10 & (ymm24 ^ ymm12))3972; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm1 = ymm10[2,3,0,1]3973; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm1 = ymm10[0],ymm1[1],ymm10[2,3],ymm1[4],ymm10[5,6],ymm1[7],ymm10[8],ymm1[9],ymm10[10,11],ymm1[12],ymm10[13,14],ymm1[15]3974; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm1, %ymm73975; AVX512DQ-FCP-NEXT: vmovdqa 80(%rdi), %xmm103976; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %xmm13977; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm1[0,1],xmm10[2],xmm1[3,4],xmm10[5],xmm1[6,7]3978; AVX512DQ-FCP-NEXT: vpshufb %xmm13, %xmm2, %xmm23979; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm23980; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm7[0,1,2],ymm2[3,4,5,6,7],ymm7[8,9,10],ymm2[11,12,13,14,15]3981; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,6,5,4,7]3982; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]3983; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm6, %zmm2, %zmm173984; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, %ymm23985; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm22 ^ (ymm2 & (ymm21 ^ ymm22))3986; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm6 = ymm2[2,3,0,1]3987; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm6[2],ymm2[3,4],ymm6[5],ymm2[6,7,8,9],ymm6[10],ymm2[11,12],ymm6[13],ymm2[14,15]3988; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm9 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]3989; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm2, %ymm23990; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm7 = xmm4[0,1],xmm8[2],xmm4[3,4],xmm8[5],xmm4[6,7]3991; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm8, %xmm253992; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm4, %xmm263993; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm6 = [2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]3994; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm7, %xmm73995; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4],xmm2[5,6,7]3996; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm2[4,5,6,7]3997; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm13 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]3998; AVX512DQ-FCP-NEXT: vmovdqa %ymm13, %ymm23999; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm20 ^ (ymm2 & (ymm18 ^ ymm20))4000; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm2[2,3,0,1]4001; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]4002; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]4003; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm2, %ymm24004; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm19, %xmm84005; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm3[2],xmm8[3,4],xmm3[5],xmm8[6,7]4006; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm3, %xmm274007; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm3 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]4008; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm14, %xmm144009; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm14, %ymm0, %ymm144010; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm14 = ymm2[0,1,2],ymm14[3,4,5,6,7],ymm2[8,9,10],ymm14[11,12,13,14,15]4011; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,6,7,4]4012; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm14[4,5,6,7]4013; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm2, %zmm194014; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, %ymm24015; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm2 = ymm11 ^ (ymm2 & (ymm23 ^ ymm11))4016; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm7 = ymm2[2,3,0,1]4017; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm7[2],ymm2[3,4],ymm7[5],ymm2[6,7,8,9],ymm7[10],ymm2[11,12],ymm7[13],ymm2[14,15]4018; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm2, %ymm24019; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm7 = xmm5[0,1],xmm15[2],xmm5[3,4],xmm15[5],xmm5[6,7]4020; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm7, %xmm64021; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm2[5,6,7]4022; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]4023; AVX512DQ-FCP-NEXT: vmovdqa %ymm13, %ymm64024; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm6 = ymm24 ^ (ymm6 & (ymm12 ^ ymm24))4025; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]4026; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7,8,9],ymm7[10],ymm6[11,12],ymm7[13],ymm6[14,15]4027; AVX512DQ-FCP-NEXT: vpshufb %ymm4, %ymm6, %ymm44028; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm10[0,1],xmm1[2],xmm10[3,4],xmm1[5],xmm10[6,7]4029; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm6, %xmm34030; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm34031; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7],ymm4[8,9,10],ymm3[11,12,13,14,15]4032; AVX512DQ-FCP-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,7,4]4033; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]4034; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm24035; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm11 = ymm23 ^ (ymm13 & (ymm11 ^ ymm23))4036; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm11[2,3,0,1]4037; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm11[1,2],ymm3[3],ymm11[4,5],ymm3[6],ymm11[7],ymm3[8],ymm11[9,10],ymm3[11],ymm11[12,13],ymm3[14],ymm11[15]4038; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm11 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]4039; AVX512DQ-FCP-NEXT: vpshufb %ymm11, %ymm3, %ymm34040; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm15[0,1],xmm5[2],xmm15[3,4],xmm5[5],xmm15[6,7]4041; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm5 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]4042; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm4, %xmm44043; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]4044; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]4045; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm12 = ymm24 ^ (ymm0 & (ymm12 ^ ymm24))4046; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm12[2,3,0,1]4047; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm12[1,2],ymm4[3],ymm12[4,5],ymm4[6],ymm12[7],ymm4[8],ymm12[9,10],ymm4[11],ymm12[12,13],ymm4[14],ymm12[15]4048; AVX512DQ-FCP-NEXT: vpshufb %ymm11, %ymm4, %ymm44049; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm10[1],xmm1[2,3],xmm10[4],xmm1[5,6],xmm10[7]4050; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]4051; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm1, %xmm14052; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm14053; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4],ymm1[5,6,7]4054; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm14055; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm13 = ymm21 ^ (ymm13 & (ymm22 ^ ymm21))4056; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm13[2,3,0,1]4057; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm13[1,2],ymm3[3],ymm13[4,5],ymm3[6],ymm13[7],ymm3[8],ymm13[9,10],ymm3[11],ymm13[12,13],ymm3[14],ymm13[15]4058; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm25, %xmm44059; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm26, %xmm74060; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm7[2],xmm4[3,4],xmm7[5],xmm4[6,7]4061; AVX512DQ-FCP-NEXT: vpshufb %xmm5, %xmm4, %xmm44062; AVX512DQ-FCP-NEXT: vpshufb %ymm11, %ymm3, %ymm34063; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]4064; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]4065; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm20 ^ (ymm0 & (ymm18 ^ ymm20))4066; AVX512DQ-FCP-NEXT: vpermq {{.*#+}} ymm4 = ymm0[2,3,0,1]4067; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1,2],ymm4[3],ymm0[4,5],ymm4[6],ymm0[7],ymm4[8],ymm0[9,10],ymm4[11],ymm0[12,13],ymm4[14],ymm0[15]4068; AVX512DQ-FCP-NEXT: vpshufb %ymm11, %ymm0, %ymm04069; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm27, %xmm44070; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]4071; AVX512DQ-FCP-NEXT: vpshufb %xmm6, %xmm4, %xmm44072; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm44073; AVX512DQ-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7]4074; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm04075; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm17, (%rsi)4076; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm16, 64(%rsi)4077; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm19, 64(%rdx)4078; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, (%rdx)4079; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, 64(%rcx)4080; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, (%rcx)4081; AVX512DQ-FCP-NEXT: vzeroupper4082; AVX512DQ-FCP-NEXT: retq4083;4084; AVX512BW-LABEL: load_i16_stride3_vf64:4085; AVX512BW: # %bb.0:4086; AVX512BW-NEXT: vmovdqa64 320(%rdi), %zmm04087; AVX512BW-NEXT: vmovdqa64 256(%rdi), %zmm14088; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm24089; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm34090; AVX512BW-NEXT: vmovdqa64 128(%rdi), %zmm44091; AVX512BW-NEXT: vmovdqa64 192(%rdi), %zmm54092; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm6 = [0,3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,0,0,0,0,0,0,0,0,0,0]4093; AVX512BW-NEXT: vmovdqa64 %zmm5, %zmm74094; AVX512BW-NEXT: vpermt2w %zmm1, %zmm6, %zmm74095; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm8 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,34,37,40,43,46,49,52,55,58,61]4096; AVX512BW-NEXT: vpermt2w %zmm0, %zmm8, %zmm74097; AVX512BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm64098; AVX512BW-NEXT: vpermt2w %zmm4, %zmm8, %zmm64099; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm8 = [1,4,7,10,13,16,19,22,25,28,31,34,37,40,43,46,49,52,55,58,61,0,0,0,0,0,0,0,0,0,0,0]4100; AVX512BW-NEXT: vmovdqa64 %zmm5, %zmm94101; AVX512BW-NEXT: vpermt2w %zmm1, %zmm8, %zmm94102; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm10 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,32,35,38,41,44,47,50,53,56,59,62]4103; AVX512BW-NEXT: vpermt2w %zmm0, %zmm10, %zmm94104; AVX512BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm84105; AVX512BW-NEXT: vpermt2w %zmm4, %zmm10, %zmm84106; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm10 = [34,37,40,43,46,49,52,55,58,61,0,3,6,9,12,15,18,21,24,27,30,0,0,0,0,0,0,0,0,0,0,0]4107; AVX512BW-NEXT: vpermt2w %zmm5, %zmm10, %zmm14108; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm5 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,33,36,39,42,45,48,51,54,57,60,63]4109; AVX512BW-NEXT: vpermt2w %zmm0, %zmm5, %zmm14110; AVX512BW-NEXT: vpermt2w %zmm2, %zmm10, %zmm34111; AVX512BW-NEXT: vpermt2w %zmm4, %zmm5, %zmm34112; AVX512BW-NEXT: vmovdqa64 %zmm7, 64(%rsi)4113; AVX512BW-NEXT: vmovdqa64 %zmm6, (%rsi)4114; AVX512BW-NEXT: vmovdqa64 %zmm9, 64(%rdx)4115; AVX512BW-NEXT: vmovdqa64 %zmm8, (%rdx)4116; AVX512BW-NEXT: vmovdqa64 %zmm1, 64(%rcx)4117; AVX512BW-NEXT: vmovdqa64 %zmm3, (%rcx)4118; AVX512BW-NEXT: vzeroupper4119; AVX512BW-NEXT: retq4120;4121; AVX512BW-FCP-LABEL: load_i16_stride3_vf64:4122; AVX512BW-FCP: # %bb.0:4123; AVX512BW-FCP-NEXT: vmovdqa64 320(%rdi), %zmm04124; AVX512BW-FCP-NEXT: vmovdqa64 256(%rdi), %zmm14125; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm24126; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm34127; AVX512BW-FCP-NEXT: vmovdqa64 128(%rdi), %zmm44128; AVX512BW-FCP-NEXT: vmovdqa64 192(%rdi), %zmm54129; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm6 = [0,3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,0,0,0,0,0,0,0,0,0,0]4130; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm74131; AVX512BW-FCP-NEXT: vpermt2w %zmm1, %zmm6, %zmm74132; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm8 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,34,37,40,43,46,49,52,55,58,61]4133; AVX512BW-FCP-NEXT: vpermt2w %zmm0, %zmm8, %zmm74134; AVX512BW-FCP-NEXT: vpermi2w %zmm3, %zmm2, %zmm64135; AVX512BW-FCP-NEXT: vpermt2w %zmm4, %zmm8, %zmm64136; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm8 = [1,4,7,10,13,16,19,22,25,28,31,34,37,40,43,46,49,52,55,58,61,0,0,0,0,0,0,0,0,0,0,0]4137; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm94138; AVX512BW-FCP-NEXT: vpermt2w %zmm1, %zmm8, %zmm94139; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm10 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,32,35,38,41,44,47,50,53,56,59,62]4140; AVX512BW-FCP-NEXT: vpermt2w %zmm0, %zmm10, %zmm94141; AVX512BW-FCP-NEXT: vpermi2w %zmm3, %zmm2, %zmm84142; AVX512BW-FCP-NEXT: vpermt2w %zmm4, %zmm10, %zmm84143; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm10 = [34,37,40,43,46,49,52,55,58,61,0,3,6,9,12,15,18,21,24,27,30,0,0,0,0,0,0,0,0,0,0,0]4144; AVX512BW-FCP-NEXT: vpermt2w %zmm5, %zmm10, %zmm14145; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm5 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,33,36,39,42,45,48,51,54,57,60,63]4146; AVX512BW-FCP-NEXT: vpermt2w %zmm0, %zmm5, %zmm14147; AVX512BW-FCP-NEXT: vpermt2w %zmm2, %zmm10, %zmm34148; AVX512BW-FCP-NEXT: vpermt2w %zmm4, %zmm5, %zmm34149; AVX512BW-FCP-NEXT: vmovdqa64 %zmm7, 64(%rsi)4150; AVX512BW-FCP-NEXT: vmovdqa64 %zmm6, (%rsi)4151; AVX512BW-FCP-NEXT: vmovdqa64 %zmm9, 64(%rdx)4152; AVX512BW-FCP-NEXT: vmovdqa64 %zmm8, (%rdx)4153; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, 64(%rcx)4154; AVX512BW-FCP-NEXT: vmovdqa64 %zmm3, (%rcx)4155; AVX512BW-FCP-NEXT: vzeroupper4156; AVX512BW-FCP-NEXT: retq4157;4158; AVX512DQ-BW-LABEL: load_i16_stride3_vf64:4159; AVX512DQ-BW: # %bb.0:4160; AVX512DQ-BW-NEXT: vmovdqa64 320(%rdi), %zmm04161; AVX512DQ-BW-NEXT: vmovdqa64 256(%rdi), %zmm14162; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm24163; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %zmm34164; AVX512DQ-BW-NEXT: vmovdqa64 128(%rdi), %zmm44165; AVX512DQ-BW-NEXT: vmovdqa64 192(%rdi), %zmm54166; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm6 = [0,3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,0,0,0,0,0,0,0,0,0,0]4167; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, %zmm74168; AVX512DQ-BW-NEXT: vpermt2w %zmm1, %zmm6, %zmm74169; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm8 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,34,37,40,43,46,49,52,55,58,61]4170; AVX512DQ-BW-NEXT: vpermt2w %zmm0, %zmm8, %zmm74171; AVX512DQ-BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm64172; AVX512DQ-BW-NEXT: vpermt2w %zmm4, %zmm8, %zmm64173; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm8 = [1,4,7,10,13,16,19,22,25,28,31,34,37,40,43,46,49,52,55,58,61,0,0,0,0,0,0,0,0,0,0,0]4174; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, %zmm94175; AVX512DQ-BW-NEXT: vpermt2w %zmm1, %zmm8, %zmm94176; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm10 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,32,35,38,41,44,47,50,53,56,59,62]4177; AVX512DQ-BW-NEXT: vpermt2w %zmm0, %zmm10, %zmm94178; AVX512DQ-BW-NEXT: vpermi2w %zmm3, %zmm2, %zmm84179; AVX512DQ-BW-NEXT: vpermt2w %zmm4, %zmm10, %zmm84180; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm10 = [34,37,40,43,46,49,52,55,58,61,0,3,6,9,12,15,18,21,24,27,30,0,0,0,0,0,0,0,0,0,0,0]4181; AVX512DQ-BW-NEXT: vpermt2w %zmm5, %zmm10, %zmm14182; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm5 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,33,36,39,42,45,48,51,54,57,60,63]4183; AVX512DQ-BW-NEXT: vpermt2w %zmm0, %zmm5, %zmm14184; AVX512DQ-BW-NEXT: vpermt2w %zmm2, %zmm10, %zmm34185; AVX512DQ-BW-NEXT: vpermt2w %zmm4, %zmm5, %zmm34186; AVX512DQ-BW-NEXT: vmovdqa64 %zmm7, 64(%rsi)4187; AVX512DQ-BW-NEXT: vmovdqa64 %zmm6, (%rsi)4188; AVX512DQ-BW-NEXT: vmovdqa64 %zmm9, 64(%rdx)4189; AVX512DQ-BW-NEXT: vmovdqa64 %zmm8, (%rdx)4190; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, 64(%rcx)4191; AVX512DQ-BW-NEXT: vmovdqa64 %zmm3, (%rcx)4192; AVX512DQ-BW-NEXT: vzeroupper4193; AVX512DQ-BW-NEXT: retq4194;4195; AVX512DQ-BW-FCP-LABEL: load_i16_stride3_vf64:4196; AVX512DQ-BW-FCP: # %bb.0:4197; AVX512DQ-BW-FCP-NEXT: vmovdqa64 320(%rdi), %zmm04198; AVX512DQ-BW-FCP-NEXT: vmovdqa64 256(%rdi), %zmm14199; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm24200; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm34201; AVX512DQ-BW-FCP-NEXT: vmovdqa64 128(%rdi), %zmm44202; AVX512DQ-BW-FCP-NEXT: vmovdqa64 192(%rdi), %zmm54203; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm6 = [0,3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,0,0,0,0,0,0,0,0,0,0]4204; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm74205; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm1, %zmm6, %zmm74206; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm8 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,34,37,40,43,46,49,52,55,58,61]4207; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm0, %zmm8, %zmm74208; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm3, %zmm2, %zmm64209; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm4, %zmm8, %zmm64210; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm8 = [1,4,7,10,13,16,19,22,25,28,31,34,37,40,43,46,49,52,55,58,61,0,0,0,0,0,0,0,0,0,0,0]4211; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, %zmm94212; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm1, %zmm8, %zmm94213; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm10 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,32,35,38,41,44,47,50,53,56,59,62]4214; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm0, %zmm10, %zmm94215; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm3, %zmm2, %zmm84216; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm4, %zmm10, %zmm84217; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm10 = [34,37,40,43,46,49,52,55,58,61,0,3,6,9,12,15,18,21,24,27,30,0,0,0,0,0,0,0,0,0,0,0]4218; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm5, %zmm10, %zmm14219; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm5 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,33,36,39,42,45,48,51,54,57,60,63]4220; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm0, %zmm5, %zmm14221; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm2, %zmm10, %zmm34222; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm4, %zmm5, %zmm34223; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm7, 64(%rsi)4224; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm6, (%rsi)4225; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm9, 64(%rdx)4226; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm8, (%rdx)4227; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, 64(%rcx)4228; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm3, (%rcx)4229; AVX512DQ-BW-FCP-NEXT: vzeroupper4230; AVX512DQ-BW-FCP-NEXT: retq4231 %wide.vec = load <192 x i16>, ptr %in.vec, align 644232 %strided.vec0 = shufflevector <192 x i16> %wide.vec, <192 x i16> poison, <64 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45, i32 48, i32 51, i32 54, i32 57, i32 60, i32 63, i32 66, i32 69, i32 72, i32 75, i32 78, i32 81, i32 84, i32 87, i32 90, i32 93, i32 96, i32 99, i32 102, i32 105, i32 108, i32 111, i32 114, i32 117, i32 120, i32 123, i32 126, i32 129, i32 132, i32 135, i32 138, i32 141, i32 144, i32 147, i32 150, i32 153, i32 156, i32 159, i32 162, i32 165, i32 168, i32 171, i32 174, i32 177, i32 180, i32 183, i32 186, i32 189>4233 %strided.vec1 = shufflevector <192 x i16> %wide.vec, <192 x i16> poison, <64 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46, i32 49, i32 52, i32 55, i32 58, i32 61, i32 64, i32 67, i32 70, i32 73, i32 76, i32 79, i32 82, i32 85, i32 88, i32 91, i32 94, i32 97, i32 100, i32 103, i32 106, i32 109, i32 112, i32 115, i32 118, i32 121, i32 124, i32 127, i32 130, i32 133, i32 136, i32 139, i32 142, i32 145, i32 148, i32 151, i32 154, i32 157, i32 160, i32 163, i32 166, i32 169, i32 172, i32 175, i32 178, i32 181, i32 184, i32 187, i32 190>4234 %strided.vec2 = shufflevector <192 x i16> %wide.vec, <192 x i16> poison, <64 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47, i32 50, i32 53, i32 56, i32 59, i32 62, i32 65, i32 68, i32 71, i32 74, i32 77, i32 80, i32 83, i32 86, i32 89, i32 92, i32 95, i32 98, i32 101, i32 104, i32 107, i32 110, i32 113, i32 116, i32 119, i32 122, i32 125, i32 128, i32 131, i32 134, i32 137, i32 140, i32 143, i32 146, i32 149, i32 152, i32 155, i32 158, i32 161, i32 164, i32 167, i32 170, i32 173, i32 176, i32 179, i32 182, i32 185, i32 188, i32 191>4235 store <64 x i16> %strided.vec0, ptr %out.vec0, align 644236 store <64 x i16> %strided.vec1, ptr %out.vec1, align 644237 store <64 x i16> %strided.vec2, ptr %out.vec2, align 644238 ret void4239}4240