2749 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE3; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX4; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX25; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FP6; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FCP7; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX5128; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512-FCP9; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ10; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ-FCP11; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW12; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW-FCP13; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512DQ-BW14; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ-BW-FCP15 16; These patterns are produced by LoopVectorizer for interleaved loads.17 18define void @load_i8_stride3_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2) nounwind {19; SSE-LABEL: load_i8_stride3_vf2:20; SSE: # %bb.0:21; SSE-NEXT: movq {{.*#+}} xmm0 = mem[0],zero22; SSE-NEXT: pxor %xmm1, %xmm123; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]24; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm0[0,3,2,3,4,5,6,7]25; SSE-NEXT: packuswb %xmm1, %xmm126; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]27; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,2,3,4,5,6,7]28; SSE-NEXT: packuswb %xmm2, %xmm229; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]30; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]31; SSE-NEXT: packuswb %xmm0, %xmm032; SSE-NEXT: movd %xmm1, %eax33; SSE-NEXT: movw %ax, (%rsi)34; SSE-NEXT: movd %xmm2, %eax35; SSE-NEXT: movw %ax, (%rdx)36; SSE-NEXT: movd %xmm0, %eax37; SSE-NEXT: movw %ax, (%rcx)38; SSE-NEXT: retq39;40; AVX-LABEL: load_i8_stride3_vf2:41; AVX: # %bb.0:42; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero43; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,u,u,u,u,u,u,u,u,u,u,u,u,u,u]44; AVX-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]45; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,u,u,u,u,u,u,u,u,u,u,u,u,u,u]46; AVX-NEXT: vpextrw $0, %xmm1, (%rsi)47; AVX-NEXT: vpextrw $0, %xmm2, (%rdx)48; AVX-NEXT: vpextrw $0, %xmm0, (%rcx)49; AVX-NEXT: retq50;51; AVX2-LABEL: load_i8_stride3_vf2:52; AVX2: # %bb.0:53; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero54; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,u,u,u,u,u,u,u,u,u,u,u,u,u,u]55; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]56; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,u,u,u,u,u,u,u,u,u,u,u,u,u,u]57; AVX2-NEXT: vpextrw $0, %xmm1, (%rsi)58; AVX2-NEXT: vpextrw $0, %xmm2, (%rdx)59; AVX2-NEXT: vpextrw $0, %xmm0, (%rcx)60; AVX2-NEXT: retq61;62; AVX2-FP-LABEL: load_i8_stride3_vf2:63; AVX2-FP: # %bb.0:64; AVX2-FP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero65; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,u,u,u,u,u,u,u,u,u,u,u,u,u,u]66; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]67; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,u,u,u,u,u,u,u,u,u,u,u,u,u,u]68; AVX2-FP-NEXT: vpextrw $0, %xmm1, (%rsi)69; AVX2-FP-NEXT: vpextrw $0, %xmm2, (%rdx)70; AVX2-FP-NEXT: vpextrw $0, %xmm0, (%rcx)71; AVX2-FP-NEXT: retq72;73; AVX2-FCP-LABEL: load_i8_stride3_vf2:74; AVX2-FCP: # %bb.0:75; AVX2-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero76; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,u,u,u,u,u,u,u,u,u,u,u,u,u,u]77; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]78; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,u,u,u,u,u,u,u,u,u,u,u,u,u,u]79; AVX2-FCP-NEXT: vpextrw $0, %xmm1, (%rsi)80; AVX2-FCP-NEXT: vpextrw $0, %xmm2, (%rdx)81; AVX2-FCP-NEXT: vpextrw $0, %xmm0, (%rcx)82; AVX2-FCP-NEXT: retq83;84; AVX512-LABEL: load_i8_stride3_vf2:85; AVX512: # %bb.0:86; AVX512-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero87; AVX512-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,u,u,u,u,u,u,u,u,u,u,u,u,u,u]88; AVX512-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]89; AVX512-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,u,u,u,u,u,u,u,u,u,u,u,u,u,u]90; AVX512-NEXT: vpextrw $0, %xmm1, (%rsi)91; AVX512-NEXT: vpextrw $0, %xmm2, (%rdx)92; AVX512-NEXT: vpextrw $0, %xmm0, (%rcx)93; AVX512-NEXT: retq94;95; AVX512-FCP-LABEL: load_i8_stride3_vf2:96; AVX512-FCP: # %bb.0:97; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero98; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,u,u,u,u,u,u,u,u,u,u,u,u,u,u]99; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]100; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,u,u,u,u,u,u,u,u,u,u,u,u,u,u]101; AVX512-FCP-NEXT: vpextrw $0, %xmm1, (%rsi)102; AVX512-FCP-NEXT: vpextrw $0, %xmm2, (%rdx)103; AVX512-FCP-NEXT: vpextrw $0, %xmm0, (%rcx)104; AVX512-FCP-NEXT: retq105;106; AVX512DQ-LABEL: load_i8_stride3_vf2:107; AVX512DQ: # %bb.0:108; AVX512DQ-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero109; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,u,u,u,u,u,u,u,u,u,u,u,u,u,u]110; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]111; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,u,u,u,u,u,u,u,u,u,u,u,u,u,u]112; AVX512DQ-NEXT: vpextrw $0, %xmm1, (%rsi)113; AVX512DQ-NEXT: vpextrw $0, %xmm2, (%rdx)114; AVX512DQ-NEXT: vpextrw $0, %xmm0, (%rcx)115; AVX512DQ-NEXT: retq116;117; AVX512DQ-FCP-LABEL: load_i8_stride3_vf2:118; AVX512DQ-FCP: # %bb.0:119; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero120; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,u,u,u,u,u,u,u,u,u,u,u,u,u,u]121; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]122; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,u,u,u,u,u,u,u,u,u,u,u,u,u,u]123; AVX512DQ-FCP-NEXT: vpextrw $0, %xmm1, (%rsi)124; AVX512DQ-FCP-NEXT: vpextrw $0, %xmm2, (%rdx)125; AVX512DQ-FCP-NEXT: vpextrw $0, %xmm0, (%rcx)126; AVX512DQ-FCP-NEXT: retq127;128; AVX512BW-LABEL: load_i8_stride3_vf2:129; AVX512BW: # %bb.0:130; AVX512BW-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero131; AVX512BW-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,u,u,u,u,u,u,u,u,u,u,u,u,u,u]132; AVX512BW-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]133; AVX512BW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,u,u,u,u,u,u,u,u,u,u,u,u,u,u]134; AVX512BW-NEXT: vpextrw $0, %xmm1, (%rsi)135; AVX512BW-NEXT: vpextrw $0, %xmm2, (%rdx)136; AVX512BW-NEXT: vpextrw $0, %xmm0, (%rcx)137; AVX512BW-NEXT: retq138;139; AVX512BW-FCP-LABEL: load_i8_stride3_vf2:140; AVX512BW-FCP: # %bb.0:141; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero142; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,u,u,u,u,u,u,u,u,u,u,u,u,u,u]143; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]144; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,u,u,u,u,u,u,u,u,u,u,u,u,u,u]145; AVX512BW-FCP-NEXT: vpextrw $0, %xmm1, (%rsi)146; AVX512BW-FCP-NEXT: vpextrw $0, %xmm2, (%rdx)147; AVX512BW-FCP-NEXT: vpextrw $0, %xmm0, (%rcx)148; AVX512BW-FCP-NEXT: retq149;150; AVX512DQ-BW-LABEL: load_i8_stride3_vf2:151; AVX512DQ-BW: # %bb.0:152; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero153; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,u,u,u,u,u,u,u,u,u,u,u,u,u,u]154; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]155; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,u,u,u,u,u,u,u,u,u,u,u,u,u,u]156; AVX512DQ-BW-NEXT: vpextrw $0, %xmm1, (%rsi)157; AVX512DQ-BW-NEXT: vpextrw $0, %xmm2, (%rdx)158; AVX512DQ-BW-NEXT: vpextrw $0, %xmm0, (%rcx)159; AVX512DQ-BW-NEXT: retq160;161; AVX512DQ-BW-FCP-LABEL: load_i8_stride3_vf2:162; AVX512DQ-BW-FCP: # %bb.0:163; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero164; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,u,u,u,u,u,u,u,u,u,u,u,u,u,u]165; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]166; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,u,u,u,u,u,u,u,u,u,u,u,u,u,u]167; AVX512DQ-BW-FCP-NEXT: vpextrw $0, %xmm1, (%rsi)168; AVX512DQ-BW-FCP-NEXT: vpextrw $0, %xmm2, (%rdx)169; AVX512DQ-BW-FCP-NEXT: vpextrw $0, %xmm0, (%rcx)170; AVX512DQ-BW-FCP-NEXT: retq171 %wide.vec = load <6 x i8>, ptr %in.vec, align 64172 %strided.vec0 = shufflevector <6 x i8> %wide.vec, <6 x i8> poison, <2 x i32> <i32 0, i32 3>173 %strided.vec1 = shufflevector <6 x i8> %wide.vec, <6 x i8> poison, <2 x i32> <i32 1, i32 4>174 %strided.vec2 = shufflevector <6 x i8> %wide.vec, <6 x i8> poison, <2 x i32> <i32 2, i32 5>175 store <2 x i8> %strided.vec0, ptr %out.vec0, align 64176 store <2 x i8> %strided.vec1, ptr %out.vec1, align 64177 store <2 x i8> %strided.vec2, ptr %out.vec2, align 64178 ret void179}180 181define void @load_i8_stride3_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2) nounwind {182; SSE-LABEL: load_i8_stride3_vf4:183; SSE: # %bb.0:184; SSE-NEXT: movdqa (%rdi), %xmm0185; SSE-NEXT: movdqa {{.*#+}} xmm1 = [65535,0,65535,65535,65535,65535,65535,65535]186; SSE-NEXT: pxor %xmm2, %xmm2187; SSE-NEXT: movdqa %xmm0, %xmm3188; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]189; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]190; SSE-NEXT: movdqa %xmm0, %xmm2191; SSE-NEXT: pand %xmm1, %xmm2192; SSE-NEXT: pandn %xmm3, %xmm1193; SSE-NEXT: por %xmm2, %xmm1194; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]195; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7]196; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]197; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,1,4,5,6,7]198; SSE-NEXT: packuswb %xmm1, %xmm1199; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm3[0,3,2,3,4,5,6,7]200; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,0],xmm0[0,0]201; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm0[2,3]202; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[2,1,2,3,4,5,6,7]203; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,7]204; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]205; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[1,2,3,0,4,5,6,7]206; SSE-NEXT: packuswb %xmm3, %xmm3207; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]208; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]209; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]210; SSE-NEXT: packuswb %xmm0, %xmm0211; SSE-NEXT: movd %xmm1, (%rsi)212; SSE-NEXT: movd %xmm3, (%rdx)213; SSE-NEXT: movd %xmm0, (%rcx)214; SSE-NEXT: retq215;216; AVX-LABEL: load_i8_stride3_vf4:217; AVX: # %bb.0:218; AVX-NEXT: vmovdqa (%rdi), %xmm0219; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,6,9,u,u,u,u,u,u,u,u,u,u,u,u]220; AVX-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,7,10,u,u,u,u,u,u,u,u,u,u,u,u]221; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,u,u,u,u,u,u,u,u,u,u,u,u]222; AVX-NEXT: vmovd %xmm1, (%rsi)223; AVX-NEXT: vmovd %xmm2, (%rdx)224; AVX-NEXT: vmovd %xmm0, (%rcx)225; AVX-NEXT: retq226;227; AVX2-LABEL: load_i8_stride3_vf4:228; AVX2: # %bb.0:229; AVX2-NEXT: vmovdqa (%rdi), %xmm0230; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,6,9,u,u,u,u,u,u,u,u,u,u,u,u]231; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,7,10,u,u,u,u,u,u,u,u,u,u,u,u]232; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,u,u,u,u,u,u,u,u,u,u,u,u]233; AVX2-NEXT: vmovd %xmm1, (%rsi)234; AVX2-NEXT: vmovd %xmm2, (%rdx)235; AVX2-NEXT: vmovd %xmm0, (%rcx)236; AVX2-NEXT: retq237;238; AVX2-FP-LABEL: load_i8_stride3_vf4:239; AVX2-FP: # %bb.0:240; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm0241; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,6,9,u,u,u,u,u,u,u,u,u,u,u,u]242; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,7,10,u,u,u,u,u,u,u,u,u,u,u,u]243; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,u,u,u,u,u,u,u,u,u,u,u,u]244; AVX2-FP-NEXT: vmovd %xmm1, (%rsi)245; AVX2-FP-NEXT: vmovd %xmm2, (%rdx)246; AVX2-FP-NEXT: vmovd %xmm0, (%rcx)247; AVX2-FP-NEXT: retq248;249; AVX2-FCP-LABEL: load_i8_stride3_vf4:250; AVX2-FCP: # %bb.0:251; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm0252; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,6,9,u,u,u,u,u,u,u,u,u,u,u,u]253; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,7,10,u,u,u,u,u,u,u,u,u,u,u,u]254; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,u,u,u,u,u,u,u,u,u,u,u,u]255; AVX2-FCP-NEXT: vmovd %xmm1, (%rsi)256; AVX2-FCP-NEXT: vmovd %xmm2, (%rdx)257; AVX2-FCP-NEXT: vmovd %xmm0, (%rcx)258; AVX2-FCP-NEXT: retq259;260; AVX512-LABEL: load_i8_stride3_vf4:261; AVX512: # %bb.0:262; AVX512-NEXT: vmovdqa (%rdi), %xmm0263; AVX512-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,6,9,u,u,u,u,u,u,u,u,u,u,u,u]264; AVX512-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,7,10,u,u,u,u,u,u,u,u,u,u,u,u]265; AVX512-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,u,u,u,u,u,u,u,u,u,u,u,u]266; AVX512-NEXT: vmovd %xmm1, (%rsi)267; AVX512-NEXT: vmovd %xmm2, (%rdx)268; AVX512-NEXT: vmovd %xmm0, (%rcx)269; AVX512-NEXT: retq270;271; AVX512-FCP-LABEL: load_i8_stride3_vf4:272; AVX512-FCP: # %bb.0:273; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm0274; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,6,9,u,u,u,u,u,u,u,u,u,u,u,u]275; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,7,10,u,u,u,u,u,u,u,u,u,u,u,u]276; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,u,u,u,u,u,u,u,u,u,u,u,u]277; AVX512-FCP-NEXT: vmovd %xmm1, (%rsi)278; AVX512-FCP-NEXT: vmovd %xmm2, (%rdx)279; AVX512-FCP-NEXT: vmovd %xmm0, (%rcx)280; AVX512-FCP-NEXT: retq281;282; AVX512DQ-LABEL: load_i8_stride3_vf4:283; AVX512DQ: # %bb.0:284; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0285; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,6,9,u,u,u,u,u,u,u,u,u,u,u,u]286; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,7,10,u,u,u,u,u,u,u,u,u,u,u,u]287; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,u,u,u,u,u,u,u,u,u,u,u,u]288; AVX512DQ-NEXT: vmovd %xmm1, (%rsi)289; AVX512DQ-NEXT: vmovd %xmm2, (%rdx)290; AVX512DQ-NEXT: vmovd %xmm0, (%rcx)291; AVX512DQ-NEXT: retq292;293; AVX512DQ-FCP-LABEL: load_i8_stride3_vf4:294; AVX512DQ-FCP: # %bb.0:295; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm0296; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,6,9,u,u,u,u,u,u,u,u,u,u,u,u]297; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,7,10,u,u,u,u,u,u,u,u,u,u,u,u]298; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,u,u,u,u,u,u,u,u,u,u,u,u]299; AVX512DQ-FCP-NEXT: vmovd %xmm1, (%rsi)300; AVX512DQ-FCP-NEXT: vmovd %xmm2, (%rdx)301; AVX512DQ-FCP-NEXT: vmovd %xmm0, (%rcx)302; AVX512DQ-FCP-NEXT: retq303;304; AVX512BW-LABEL: load_i8_stride3_vf4:305; AVX512BW: # %bb.0:306; AVX512BW-NEXT: vmovdqa (%rdi), %xmm0307; AVX512BW-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,6,9,u,u,u,u,u,u,u,u,u,u,u,u]308; AVX512BW-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,7,10,u,u,u,u,u,u,u,u,u,u,u,u]309; AVX512BW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,u,u,u,u,u,u,u,u,u,u,u,u]310; AVX512BW-NEXT: vmovd %xmm1, (%rsi)311; AVX512BW-NEXT: vmovd %xmm2, (%rdx)312; AVX512BW-NEXT: vmovd %xmm0, (%rcx)313; AVX512BW-NEXT: retq314;315; AVX512BW-FCP-LABEL: load_i8_stride3_vf4:316; AVX512BW-FCP: # %bb.0:317; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %xmm0318; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,6,9,u,u,u,u,u,u,u,u,u,u,u,u]319; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,7,10,u,u,u,u,u,u,u,u,u,u,u,u]320; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,u,u,u,u,u,u,u,u,u,u,u,u]321; AVX512BW-FCP-NEXT: vmovd %xmm1, (%rsi)322; AVX512BW-FCP-NEXT: vmovd %xmm2, (%rdx)323; AVX512BW-FCP-NEXT: vmovd %xmm0, (%rcx)324; AVX512BW-FCP-NEXT: retq325;326; AVX512DQ-BW-LABEL: load_i8_stride3_vf4:327; AVX512DQ-BW: # %bb.0:328; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %xmm0329; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,6,9,u,u,u,u,u,u,u,u,u,u,u,u]330; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,7,10,u,u,u,u,u,u,u,u,u,u,u,u]331; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,u,u,u,u,u,u,u,u,u,u,u,u]332; AVX512DQ-BW-NEXT: vmovd %xmm1, (%rsi)333; AVX512DQ-BW-NEXT: vmovd %xmm2, (%rdx)334; AVX512DQ-BW-NEXT: vmovd %xmm0, (%rcx)335; AVX512DQ-BW-NEXT: retq336;337; AVX512DQ-BW-FCP-LABEL: load_i8_stride3_vf4:338; AVX512DQ-BW-FCP: # %bb.0:339; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %xmm0340; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,3,6,9,u,u,u,u,u,u,u,u,u,u,u,u]341; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm0[1,4,7,10,u,u,u,u,u,u,u,u,u,u,u,u]342; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,u,u,u,u,u,u,u,u,u,u,u,u]343; AVX512DQ-BW-FCP-NEXT: vmovd %xmm1, (%rsi)344; AVX512DQ-BW-FCP-NEXT: vmovd %xmm2, (%rdx)345; AVX512DQ-BW-FCP-NEXT: vmovd %xmm0, (%rcx)346; AVX512DQ-BW-FCP-NEXT: retq347 %wide.vec = load <12 x i8>, ptr %in.vec, align 64348 %strided.vec0 = shufflevector <12 x i8> %wide.vec, <12 x i8> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>349 %strided.vec1 = shufflevector <12 x i8> %wide.vec, <12 x i8> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>350 %strided.vec2 = shufflevector <12 x i8> %wide.vec, <12 x i8> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>351 store <4 x i8> %strided.vec0, ptr %out.vec0, align 64352 store <4 x i8> %strided.vec1, ptr %out.vec1, align 64353 store <4 x i8> %strided.vec2, ptr %out.vec2, align 64354 ret void355}356 357define void @load_i8_stride3_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2) nounwind {358; SSE-LABEL: load_i8_stride3_vf8:359; SSE: # %bb.0:360; SSE-NEXT: movdqa (%rdi), %xmm0361; SSE-NEXT: movdqa 16(%rdi), %xmm1362; SSE-NEXT: movdqa {{.*#+}} xmm4 = [255,255,0,255,255,0,255,255,255,255,255,255,255,255,255,255]363; SSE-NEXT: movdqa %xmm0, %xmm2364; SSE-NEXT: pand %xmm4, %xmm2365; SSE-NEXT: pandn %xmm1, %xmm4366; SSE-NEXT: por %xmm2, %xmm4367; SSE-NEXT: pxor %xmm2, %xmm2368; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]369; SSE-NEXT: movdqa {{.*#+}} xmm5 = [65535,0,65535,65535,0,65535,65535,0]370; SSE-NEXT: pand %xmm5, %xmm4371; SSE-NEXT: movdqa %xmm0, %xmm3372; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]373; SSE-NEXT: pandn %xmm3, %xmm5374; SSE-NEXT: por %xmm4, %xmm5375; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm5[0,2,1,3]376; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,7,6,5]377; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,3,2,1]378; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,3,2,1,4,5,6,7]379; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,6,5,4,7]380; SSE-NEXT: packuswb %xmm4, %xmm4381; SSE-NEXT: movq %xmm4, (%rsi)382; SSE-NEXT: movdqa {{.*#+}} xmm4 = [0,255,255,0,255,255,0,255,255,255,255,255,255,255,255,255]383; SSE-NEXT: movdqa %xmm0, %xmm5384; SSE-NEXT: pand %xmm4, %xmm5385; SSE-NEXT: pandn %xmm1, %xmm4386; SSE-NEXT: por %xmm5, %xmm4387; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]388; SSE-NEXT: movdqa {{.*#+}} xmm5 = [65535,65535,0,65535,65535,0,65535,65535]389; SSE-NEXT: pand %xmm5, %xmm4390; SSE-NEXT: pandn %xmm3, %xmm5391; SSE-NEXT: por %xmm4, %xmm5392; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm5[2,1,0,3,4,5,6,7]393; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,6,5,4,7]394; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,3,2,1]395; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,2,3,0,4,5,6,7]396; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,7,4]397; SSE-NEXT: packuswb %xmm4, %xmm4398; SSE-NEXT: movq %xmm4, (%rdx)399; SSE-NEXT: movdqa {{.*#+}} xmm4 = [255,0,255,255,0,255,255,0,255,255,255,255,255,255,255,255]400; SSE-NEXT: pand %xmm4, %xmm0401; SSE-NEXT: pandn %xmm1, %xmm4402; SSE-NEXT: por %xmm0, %xmm4403; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]404; SSE-NEXT: movdqa {{.*#+}} xmm0 = [0,65535,65535,0,65535,65535,0,65535]405; SSE-NEXT: pand %xmm0, %xmm4406; SSE-NEXT: pandn %xmm3, %xmm0407; SSE-NEXT: por %xmm4, %xmm0408; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0]409; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5]410; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0]411; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]412; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]413; SSE-NEXT: packuswb %xmm0, %xmm0414; SSE-NEXT: movq %xmm0, (%rcx)415; SSE-NEXT: retq416;417; AVX-LABEL: load_i8_stride3_vf8:418; AVX: # %bb.0:419; AVX-NEXT: vmovdqa (%rdi), %xmm0420; AVX-NEXT: vmovdqa 16(%rdi), %xmm1421; AVX-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm1[2,5,u,u,u,u,u,u,u,u]422; AVX-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[0,3,6,9,12,15],zero,zero,xmm0[u,u,u,u,u,u,u,u]423; AVX-NEXT: vpor %xmm2, %xmm3, %xmm2424; AVX-NEXT: vmovq %xmm2, (%rsi)425; AVX-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,xmm1[0,3,6,u,u,u,u,u,u,u,u]426; AVX-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[1,4,7,10,13],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]427; AVX-NEXT: vpor %xmm2, %xmm3, %xmm2428; AVX-NEXT: vmovq %xmm2, (%rdx)429; AVX-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]430; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]431; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0432; AVX-NEXT: vmovq %xmm0, (%rcx)433; AVX-NEXT: retq434;435; AVX2-LABEL: load_i8_stride3_vf8:436; AVX2: # %bb.0:437; AVX2-NEXT: vmovdqa (%rdi), %xmm0438; AVX2-NEXT: vmovdqa 16(%rdi), %xmm1439; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm1[2,5,u,u,u,u,u,u,u,u]440; AVX2-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[0,3,6,9,12,15],zero,zero,xmm0[u,u,u,u,u,u,u,u]441; AVX2-NEXT: vpor %xmm2, %xmm3, %xmm2442; AVX2-NEXT: vmovq %xmm2, (%rsi)443; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,xmm1[0,3,6,u,u,u,u,u,u,u,u]444; AVX2-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[1,4,7,10,13],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]445; AVX2-NEXT: vpor %xmm2, %xmm3, %xmm2446; AVX2-NEXT: vmovq %xmm2, (%rdx)447; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]448; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]449; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0450; AVX2-NEXT: vmovq %xmm0, (%rcx)451; AVX2-NEXT: retq452;453; AVX2-FP-LABEL: load_i8_stride3_vf8:454; AVX2-FP: # %bb.0:455; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm0456; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm1457; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm1[2,5,u,u,u,u,u,u,u,u]458; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[0,3,6,9,12,15],zero,zero,xmm0[u,u,u,u,u,u,u,u]459; AVX2-FP-NEXT: vpor %xmm2, %xmm3, %xmm2460; AVX2-FP-NEXT: vmovq %xmm2, (%rsi)461; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,xmm1[0,3,6,u,u,u,u,u,u,u,u]462; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[1,4,7,10,13],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]463; AVX2-FP-NEXT: vpor %xmm2, %xmm3, %xmm2464; AVX2-FP-NEXT: vmovq %xmm2, (%rdx)465; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]466; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]467; AVX2-FP-NEXT: vpor %xmm1, %xmm0, %xmm0468; AVX2-FP-NEXT: vmovq %xmm0, (%rcx)469; AVX2-FP-NEXT: retq470;471; AVX2-FCP-LABEL: load_i8_stride3_vf8:472; AVX2-FCP: # %bb.0:473; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm0474; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm1475; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm1[2,5,u,u,u,u,u,u,u,u]476; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[0,3,6,9,12,15],zero,zero,xmm0[u,u,u,u,u,u,u,u]477; AVX2-FCP-NEXT: vpor %xmm2, %xmm3, %xmm2478; AVX2-FCP-NEXT: vmovq %xmm2, (%rsi)479; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,xmm1[0,3,6,u,u,u,u,u,u,u,u]480; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[1,4,7,10,13],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]481; AVX2-FCP-NEXT: vpor %xmm2, %xmm3, %xmm2482; AVX2-FCP-NEXT: vmovq %xmm2, (%rdx)483; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]484; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]485; AVX2-FCP-NEXT: vpor %xmm1, %xmm0, %xmm0486; AVX2-FCP-NEXT: vmovq %xmm0, (%rcx)487; AVX2-FCP-NEXT: retq488;489; AVX512-LABEL: load_i8_stride3_vf8:490; AVX512: # %bb.0:491; AVX512-NEXT: vmovdqa (%rdi), %xmm0492; AVX512-NEXT: vmovdqa 16(%rdi), %xmm1493; AVX512-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm1[2,5,u,u,u,u,u,u,u,u]494; AVX512-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[0,3,6,9,12,15],zero,zero,xmm0[u,u,u,u,u,u,u,u]495; AVX512-NEXT: vpor %xmm2, %xmm3, %xmm2496; AVX512-NEXT: vmovq %xmm2, (%rsi)497; AVX512-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,xmm1[0,3,6,u,u,u,u,u,u,u,u]498; AVX512-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[1,4,7,10,13],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]499; AVX512-NEXT: vpor %xmm2, %xmm3, %xmm2500; AVX512-NEXT: vmovq %xmm2, (%rdx)501; AVX512-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]502; AVX512-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]503; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0504; AVX512-NEXT: vmovq %xmm0, (%rcx)505; AVX512-NEXT: retq506;507; AVX512-FCP-LABEL: load_i8_stride3_vf8:508; AVX512-FCP: # %bb.0:509; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm0510; AVX512-FCP-NEXT: vmovdqa 16(%rdi), %xmm1511; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm1[2,5,u,u,u,u,u,u,u,u]512; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[0,3,6,9,12,15],zero,zero,xmm0[u,u,u,u,u,u,u,u]513; AVX512-FCP-NEXT: vpor %xmm2, %xmm3, %xmm2514; AVX512-FCP-NEXT: vmovq %xmm2, (%rsi)515; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,xmm1[0,3,6,u,u,u,u,u,u,u,u]516; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[1,4,7,10,13],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]517; AVX512-FCP-NEXT: vpor %xmm2, %xmm3, %xmm2518; AVX512-FCP-NEXT: vmovq %xmm2, (%rdx)519; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]520; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]521; AVX512-FCP-NEXT: vpor %xmm1, %xmm0, %xmm0522; AVX512-FCP-NEXT: vmovq %xmm0, (%rcx)523; AVX512-FCP-NEXT: retq524;525; AVX512DQ-LABEL: load_i8_stride3_vf8:526; AVX512DQ: # %bb.0:527; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0528; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm1529; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm1[2,5,u,u,u,u,u,u,u,u]530; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[0,3,6,9,12,15],zero,zero,xmm0[u,u,u,u,u,u,u,u]531; AVX512DQ-NEXT: vpor %xmm2, %xmm3, %xmm2532; AVX512DQ-NEXT: vmovq %xmm2, (%rsi)533; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,xmm1[0,3,6,u,u,u,u,u,u,u,u]534; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[1,4,7,10,13],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]535; AVX512DQ-NEXT: vpor %xmm2, %xmm3, %xmm2536; AVX512DQ-NEXT: vmovq %xmm2, (%rdx)537; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]538; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]539; AVX512DQ-NEXT: vpor %xmm1, %xmm0, %xmm0540; AVX512DQ-NEXT: vmovq %xmm0, (%rcx)541; AVX512DQ-NEXT: retq542;543; AVX512DQ-FCP-LABEL: load_i8_stride3_vf8:544; AVX512DQ-FCP: # %bb.0:545; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm0546; AVX512DQ-FCP-NEXT: vmovdqa 16(%rdi), %xmm1547; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm1[2,5,u,u,u,u,u,u,u,u]548; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[0,3,6,9,12,15],zero,zero,xmm0[u,u,u,u,u,u,u,u]549; AVX512DQ-FCP-NEXT: vpor %xmm2, %xmm3, %xmm2550; AVX512DQ-FCP-NEXT: vmovq %xmm2, (%rsi)551; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,xmm1[0,3,6,u,u,u,u,u,u,u,u]552; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[1,4,7,10,13],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]553; AVX512DQ-FCP-NEXT: vpor %xmm2, %xmm3, %xmm2554; AVX512DQ-FCP-NEXT: vmovq %xmm2, (%rdx)555; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]556; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]557; AVX512DQ-FCP-NEXT: vpor %xmm1, %xmm0, %xmm0558; AVX512DQ-FCP-NEXT: vmovq %xmm0, (%rcx)559; AVX512DQ-FCP-NEXT: retq560;561; AVX512BW-LABEL: load_i8_stride3_vf8:562; AVX512BW: # %bb.0:563; AVX512BW-NEXT: vmovdqa (%rdi), %xmm0564; AVX512BW-NEXT: vmovdqa 16(%rdi), %xmm1565; AVX512BW-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm1[2,5,u,u,u,u,u,u,u,u]566; AVX512BW-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[0,3,6,9,12,15],zero,zero,xmm0[u,u,u,u,u,u,u,u]567; AVX512BW-NEXT: vpor %xmm2, %xmm3, %xmm2568; AVX512BW-NEXT: vmovq %xmm2, (%rsi)569; AVX512BW-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,xmm1[0,3,6,u,u,u,u,u,u,u,u]570; AVX512BW-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[1,4,7,10,13],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]571; AVX512BW-NEXT: vpor %xmm2, %xmm3, %xmm2572; AVX512BW-NEXT: vmovq %xmm2, (%rdx)573; AVX512BW-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]574; AVX512BW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]575; AVX512BW-NEXT: vpor %xmm1, %xmm0, %xmm0576; AVX512BW-NEXT: vmovq %xmm0, (%rcx)577; AVX512BW-NEXT: retq578;579; AVX512BW-FCP-LABEL: load_i8_stride3_vf8:580; AVX512BW-FCP: # %bb.0:581; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %xmm0582; AVX512BW-FCP-NEXT: vmovdqa 16(%rdi), %xmm1583; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm1[2,5,u,u,u,u,u,u,u,u]584; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[0,3,6,9,12,15],zero,zero,xmm0[u,u,u,u,u,u,u,u]585; AVX512BW-FCP-NEXT: vpor %xmm2, %xmm3, %xmm2586; AVX512BW-FCP-NEXT: vmovq %xmm2, (%rsi)587; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,xmm1[0,3,6,u,u,u,u,u,u,u,u]588; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[1,4,7,10,13],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]589; AVX512BW-FCP-NEXT: vpor %xmm2, %xmm3, %xmm2590; AVX512BW-FCP-NEXT: vmovq %xmm2, (%rdx)591; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]592; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]593; AVX512BW-FCP-NEXT: vpor %xmm1, %xmm0, %xmm0594; AVX512BW-FCP-NEXT: vmovq %xmm0, (%rcx)595; AVX512BW-FCP-NEXT: retq596;597; AVX512DQ-BW-LABEL: load_i8_stride3_vf8:598; AVX512DQ-BW: # %bb.0:599; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %xmm0600; AVX512DQ-BW-NEXT: vmovdqa 16(%rdi), %xmm1601; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm1[2,5,u,u,u,u,u,u,u,u]602; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[0,3,6,9,12,15],zero,zero,xmm0[u,u,u,u,u,u,u,u]603; AVX512DQ-BW-NEXT: vpor %xmm2, %xmm3, %xmm2604; AVX512DQ-BW-NEXT: vmovq %xmm2, (%rsi)605; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,xmm1[0,3,6,u,u,u,u,u,u,u,u]606; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[1,4,7,10,13],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]607; AVX512DQ-BW-NEXT: vpor %xmm2, %xmm3, %xmm2608; AVX512DQ-BW-NEXT: vmovq %xmm2, (%rdx)609; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]610; AVX512DQ-BW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]611; AVX512DQ-BW-NEXT: vpor %xmm1, %xmm0, %xmm0612; AVX512DQ-BW-NEXT: vmovq %xmm0, (%rcx)613; AVX512DQ-BW-NEXT: retq614;615; AVX512DQ-BW-FCP-LABEL: load_i8_stride3_vf8:616; AVX512DQ-BW-FCP: # %bb.0:617; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %xmm0618; AVX512DQ-BW-FCP-NEXT: vmovdqa 16(%rdi), %xmm1619; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,xmm1[2,5,u,u,u,u,u,u,u,u]620; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[0,3,6,9,12,15],zero,zero,xmm0[u,u,u,u,u,u,u,u]621; AVX512DQ-BW-FCP-NEXT: vpor %xmm2, %xmm3, %xmm2622; AVX512DQ-BW-FCP-NEXT: vmovq %xmm2, (%rsi)623; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,xmm1[0,3,6,u,u,u,u,u,u,u,u]624; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm0[1,4,7,10,13],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]625; AVX512DQ-BW-FCP-NEXT: vpor %xmm2, %xmm3, %xmm2626; AVX512DQ-BW-FCP-NEXT: vmovq %xmm2, (%rdx)627; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,xmm1[1,4,7,u,u,u,u,u,u,u,u]628; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[2,5,8,11,14],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]629; AVX512DQ-BW-FCP-NEXT: vpor %xmm1, %xmm0, %xmm0630; AVX512DQ-BW-FCP-NEXT: vmovq %xmm0, (%rcx)631; AVX512DQ-BW-FCP-NEXT: retq632 %wide.vec = load <24 x i8>, ptr %in.vec, align 64633 %strided.vec0 = shufflevector <24 x i8> %wide.vec, <24 x i8> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>634 %strided.vec1 = shufflevector <24 x i8> %wide.vec, <24 x i8> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>635 %strided.vec2 = shufflevector <24 x i8> %wide.vec, <24 x i8> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>636 store <8 x i8> %strided.vec0, ptr %out.vec0, align 64637 store <8 x i8> %strided.vec1, ptr %out.vec1, align 64638 store <8 x i8> %strided.vec2, ptr %out.vec2, align 64639 ret void640}641 642define void @load_i8_stride3_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2) nounwind {643; SSE-LABEL: load_i8_stride3_vf16:644; SSE: # %bb.0:645; SSE-NEXT: movdqa (%rdi), %xmm5646; SSE-NEXT: movdqa 16(%rdi), %xmm4647; SSE-NEXT: movdqa 32(%rdi), %xmm0648; SSE-NEXT: movdqa {{.*#+}} xmm3 = [255,255,0,255,255,0,255,255,0,255,255,0,255,255,0,255]649; SSE-NEXT: movdqa %xmm3, %xmm1650; SSE-NEXT: pandn %xmm4, %xmm1651; SSE-NEXT: movdqa %xmm5, %xmm2652; SSE-NEXT: pand %xmm3, %xmm2653; SSE-NEXT: por %xmm1, %xmm2654; SSE-NEXT: pxor %xmm6, %xmm6655; SSE-NEXT: movdqa %xmm2, %xmm7656; SSE-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]657; SSE-NEXT: movdqa {{.*#+}} xmm1 = [65535,0,65535,65535,0,65535,65535,0]658; SSE-NEXT: movdqa %xmm1, %xmm8659; SSE-NEXT: pandn %xmm7, %xmm8660; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3],xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]661; SSE-NEXT: pand %xmm1, %xmm2662; SSE-NEXT: por %xmm8, %xmm2663; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3]664; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,5]665; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,1]666; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,3,2,1,4,5,6,7]667; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm2[0,1,2,3,6,5,4,7]668; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm7[0,3,2,3,4,5,6,7]669; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]670; SSE-NEXT: packuswb %xmm2, %xmm10671; SSE-NEXT: movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]672; SSE-NEXT: pand %xmm2, %xmm10673; SSE-NEXT: movdqa %xmm0, %xmm7674; SSE-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]675; SSE-NEXT: movdqa {{.*#+}} xmm9 = [65535,65535,0,65535,65535,0,65535,65535]676; SSE-NEXT: movdqa %xmm9, %xmm8677; SSE-NEXT: pandn %xmm7, %xmm8678; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3],xmm0[4],xmm6[4],xmm0[5],xmm6[5],xmm0[6],xmm6[6],xmm0[7],xmm6[7]679; SSE-NEXT: movdqa %xmm0, %xmm11680; SSE-NEXT: pand %xmm9, %xmm11681; SSE-NEXT: por %xmm8, %xmm11682; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm11[3,1,2,0]683; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[2,1,2,3,4,5,6,7]684; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,3,2,0]685; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,7,6,5]686; SSE-NEXT: packuswb %xmm8, %xmm11687; SSE-NEXT: movdqa %xmm2, %xmm8688; SSE-NEXT: pandn %xmm11, %xmm8689; SSE-NEXT: por %xmm10, %xmm8690; SSE-NEXT: movdqa {{.*#+}} xmm10 = [255,0,255,255,0,255,255,0,255,255,0,255,255,0,255,255]691; SSE-NEXT: movdqa %xmm4, %xmm11692; SSE-NEXT: pand %xmm10, %xmm11693; SSE-NEXT: pandn %xmm5, %xmm10694; SSE-NEXT: por %xmm11, %xmm10695; SSE-NEXT: movdqa %xmm10, %xmm11696; SSE-NEXT: punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm6[8],xmm11[9],xmm6[9],xmm11[10],xmm6[10],xmm11[11],xmm6[11],xmm11[12],xmm6[12],xmm11[13],xmm6[13],xmm11[14],xmm6[14],xmm11[15],xmm6[15]697; SSE-NEXT: movdqa %xmm9, %xmm12698; SSE-NEXT: pandn %xmm11, %xmm12699; SSE-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3],xmm10[4],xmm6[4],xmm10[5],xmm6[5],xmm10[6],xmm6[6],xmm10[7],xmm6[7]700; SSE-NEXT: pand %xmm9, %xmm10701; SSE-NEXT: por %xmm12, %xmm10702; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[2,1,0,3,4,5,6,7]703; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,5,4,7]704; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,3,2,1]705; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[1,2,3,0,4,5,6,7]706; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,6,7,4]707; SSE-NEXT: pshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,7,6,7]708; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm11[0,2,2,3]709; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm11[1,2,3,3,4,5,6,7]710; SSE-NEXT: packuswb %xmm11, %xmm10711; SSE-NEXT: pand %xmm2, %xmm10712; SSE-NEXT: movdqa %xmm7, %xmm11713; SSE-NEXT: pand %xmm9, %xmm11714; SSE-NEXT: pandn %xmm0, %xmm9715; SSE-NEXT: por %xmm11, %xmm9716; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[0,3,2,3,4,5,6,7]717; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,6,5,6,7]718; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,1,2,0]719; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[2,2,2,2,4,5,6,7]720; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,5,6,7,4]721; SSE-NEXT: packuswb %xmm9, %xmm9722; SSE-NEXT: pandn %xmm9, %xmm2723; SSE-NEXT: por %xmm10, %xmm2724; SSE-NEXT: pand %xmm3, %xmm4725; SSE-NEXT: pandn %xmm5, %xmm3726; SSE-NEXT: por %xmm4, %xmm3727; SSE-NEXT: movdqa %xmm3, %xmm4728; SSE-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm6[8],xmm4[9],xmm6[9],xmm4[10],xmm6[10],xmm4[11],xmm6[11],xmm4[12],xmm6[12],xmm4[13],xmm6[13],xmm4[14],xmm6[14],xmm4[15],xmm6[15]729; SSE-NEXT: movdqa {{.*#+}} xmm5 = [0,65535,65535,0,65535,65535,0,65535]730; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3],xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]731; SSE-NEXT: pand %xmm5, %xmm3732; SSE-NEXT: pandn %xmm4, %xmm5733; SSE-NEXT: por %xmm3, %xmm5734; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm5[3,1,2,0]735; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,5]736; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[3,1,2,0]737; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[2,1,0,3,4,5,6,7]738; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,4,7]739; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[2,1,2,3]740; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[2,1,2,3,4,5,6,7]741; SSE-NEXT: packuswb %xmm4, %xmm3742; SSE-NEXT: movdqa {{.*#+}} xmm4 = [65535,65535,65535,65535,65535,0,0,0]743; SSE-NEXT: pand %xmm4, %xmm3744; SSE-NEXT: pand %xmm1, %xmm0745; SSE-NEXT: pandn %xmm7, %xmm1746; SSE-NEXT: por %xmm0, %xmm1747; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,1,3]748; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]749; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]750; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,3,4,5,6,7]751; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]752; SSE-NEXT: packuswb %xmm0, %xmm0753; SSE-NEXT: pandn %xmm0, %xmm4754; SSE-NEXT: por %xmm3, %xmm4755; SSE-NEXT: movdqa %xmm8, (%rsi)756; SSE-NEXT: movdqa %xmm2, (%rdx)757; SSE-NEXT: movdqa %xmm4, (%rcx)758; SSE-NEXT: retq759;760; AVX-LABEL: load_i8_stride3_vf16:761; AVX: # %bb.0:762; AVX-NEXT: vmovdqa (%rdi), %xmm0763; AVX-NEXT: vmovdqa 16(%rdi), %xmm1764; AVX-NEXT: vmovdqa 32(%rdi), %xmm2765; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]766; AVX-NEXT: vpshufb %xmm3, %xmm0, %xmm0767; AVX-NEXT: vpshufb %xmm3, %xmm1, %xmm1768; AVX-NEXT: vpshufb %xmm3, %xmm2, %xmm2769; AVX-NEXT: vpalignr {{.*#+}} xmm3 = xmm2[11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10]770; AVX-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]771; AVX-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[11,12,13,14,15],xmm2[0,1,2,3,4,5,6,7,8,9,10]772; AVX-NEXT: vpalignr {{.*#+}} xmm2 = xmm3[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]773; AVX-NEXT: vpmovsxdq {{.*#+}} xmm4 = [18446744073709551615,16777215]774; AVX-NEXT: vpblendvb %xmm4, %xmm0, %xmm1, %xmm1775; AVX-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm3[0,1,2,3,4,5,6,7,8,9,10]776; AVX-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]777; AVX-NEXT: vmovdqa %xmm0, (%rsi)778; AVX-NEXT: vmovdqa %xmm1, (%rdx)779; AVX-NEXT: vmovdqa %xmm2, (%rcx)780; AVX-NEXT: retq781;782; AVX2-LABEL: load_i8_stride3_vf16:783; AVX2: # %bb.0:784; AVX2-NEXT: vmovdqa (%rdi), %xmm0785; AVX2-NEXT: vmovdqa 16(%rdi), %xmm1786; AVX2-NEXT: vmovdqa 32(%rdi), %xmm2787; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]788; AVX2-NEXT: vpshufb %xmm3, %xmm0, %xmm0789; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm1790; AVX2-NEXT: vpshufb %xmm3, %xmm2, %xmm2791; AVX2-NEXT: vpalignr {{.*#+}} xmm3 = xmm2[11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10]792; AVX2-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]793; AVX2-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[11,12,13,14,15],xmm2[0,1,2,3,4,5,6,7,8,9,10]794; AVX2-NEXT: vpalignr {{.*#+}} xmm2 = xmm3[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]795; AVX2-NEXT: vpmovsxdq {{.*#+}} xmm4 = [18446744073709551615,16777215]796; AVX2-NEXT: vpblendvb %xmm4, %xmm0, %xmm1, %xmm1797; AVX2-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm3[0,1,2,3,4,5,6,7,8,9,10]798; AVX2-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]799; AVX2-NEXT: vmovdqa %xmm0, (%rsi)800; AVX2-NEXT: vmovdqa %xmm1, (%rdx)801; AVX2-NEXT: vmovdqa %xmm2, (%rcx)802; AVX2-NEXT: retq803;804; AVX2-FP-LABEL: load_i8_stride3_vf16:805; AVX2-FP: # %bb.0:806; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm0807; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm1808; AVX2-FP-NEXT: vmovdqa 32(%rdi), %xmm2809; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]810; AVX2-FP-NEXT: vpshufb %xmm3, %xmm0, %xmm0811; AVX2-FP-NEXT: vpshufb %xmm3, %xmm1, %xmm1812; AVX2-FP-NEXT: vpshufb %xmm3, %xmm2, %xmm2813; AVX2-FP-NEXT: vpalignr {{.*#+}} xmm3 = xmm2[11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10]814; AVX2-FP-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]815; AVX2-FP-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[11,12,13,14,15],xmm2[0,1,2,3,4,5,6,7,8,9,10]816; AVX2-FP-NEXT: vpalignr {{.*#+}} xmm2 = xmm3[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]817; AVX2-FP-NEXT: vpmovsxdq {{.*#+}} xmm4 = [18446744073709551615,16777215]818; AVX2-FP-NEXT: vpblendvb %xmm4, %xmm0, %xmm1, %xmm1819; AVX2-FP-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm3[0,1,2,3,4,5,6,7,8,9,10]820; AVX2-FP-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]821; AVX2-FP-NEXT: vmovdqa %xmm0, (%rsi)822; AVX2-FP-NEXT: vmovdqa %xmm1, (%rdx)823; AVX2-FP-NEXT: vmovdqa %xmm2, (%rcx)824; AVX2-FP-NEXT: retq825;826; AVX2-FCP-LABEL: load_i8_stride3_vf16:827; AVX2-FCP: # %bb.0:828; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm0829; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm1830; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %xmm2831; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]832; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm0, %xmm0833; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm1834; AVX2-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2835; AVX2-FCP-NEXT: vpalignr {{.*#+}} xmm3 = xmm2[11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10]836; AVX2-FCP-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]837; AVX2-FCP-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[11,12,13,14,15],xmm2[0,1,2,3,4,5,6,7,8,9,10]838; AVX2-FCP-NEXT: vpalignr {{.*#+}} xmm2 = xmm3[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]839; AVX2-FCP-NEXT: vpmovsxdq {{.*#+}} xmm4 = [18446744073709551615,16777215]840; AVX2-FCP-NEXT: vpblendvb %xmm4, %xmm0, %xmm1, %xmm1841; AVX2-FCP-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm3[0,1,2,3,4,5,6,7,8,9,10]842; AVX2-FCP-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]843; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rsi)844; AVX2-FCP-NEXT: vmovdqa %xmm1, (%rdx)845; AVX2-FCP-NEXT: vmovdqa %xmm2, (%rcx)846; AVX2-FCP-NEXT: retq847;848; AVX512-LABEL: load_i8_stride3_vf16:849; AVX512: # %bb.0:850; AVX512-NEXT: vmovdqa (%rdi), %xmm0851; AVX512-NEXT: vmovdqa 16(%rdi), %xmm1852; AVX512-NEXT: vmovdqa 32(%rdi), %xmm2853; AVX512-NEXT: vmovdqa {{.*#+}} xmm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]854; AVX512-NEXT: vpshufb %xmm3, %xmm0, %xmm0855; AVX512-NEXT: vpshufb %xmm3, %xmm1, %xmm1856; AVX512-NEXT: vpshufb %xmm3, %xmm2, %xmm2857; AVX512-NEXT: vpalignr {{.*#+}} xmm3 = xmm2[11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10]858; AVX512-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]859; AVX512-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[11,12,13,14,15],xmm2[0,1,2,3,4,5,6,7,8,9,10]860; AVX512-NEXT: vpalignr {{.*#+}} xmm2 = xmm3[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]861; AVX512-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 ^ (mem & (xmm1 ^ xmm0))862; AVX512-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm3[0,1,2,3,4,5,6,7,8,9,10]863; AVX512-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]864; AVX512-NEXT: vmovdqa %xmm0, (%rsi)865; AVX512-NEXT: vmovdqa %xmm1, (%rdx)866; AVX512-NEXT: vmovdqa %xmm2, (%rcx)867; AVX512-NEXT: retq868;869; AVX512-FCP-LABEL: load_i8_stride3_vf16:870; AVX512-FCP: # %bb.0:871; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm0872; AVX512-FCP-NEXT: vmovdqa 16(%rdi), %xmm1873; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %xmm2874; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]875; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm0, %xmm0876; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm1877; AVX512-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2878; AVX512-FCP-NEXT: vpalignr {{.*#+}} xmm3 = xmm2[11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10]879; AVX512-FCP-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]880; AVX512-FCP-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[11,12,13,14,15],xmm2[0,1,2,3,4,5,6,7,8,9,10]881; AVX512-FCP-NEXT: vpalignr {{.*#+}} xmm2 = xmm3[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]882; AVX512-FCP-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 ^ (mem & (xmm1 ^ xmm0))883; AVX512-FCP-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm3[0,1,2,3,4,5,6,7,8,9,10]884; AVX512-FCP-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]885; AVX512-FCP-NEXT: vmovdqa %xmm0, (%rsi)886; AVX512-FCP-NEXT: vmovdqa %xmm1, (%rdx)887; AVX512-FCP-NEXT: vmovdqa %xmm2, (%rcx)888; AVX512-FCP-NEXT: retq889;890; AVX512DQ-LABEL: load_i8_stride3_vf16:891; AVX512DQ: # %bb.0:892; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0893; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm1894; AVX512DQ-NEXT: vmovdqa 32(%rdi), %xmm2895; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]896; AVX512DQ-NEXT: vpshufb %xmm3, %xmm0, %xmm0897; AVX512DQ-NEXT: vpshufb %xmm3, %xmm1, %xmm1898; AVX512DQ-NEXT: vpshufb %xmm3, %xmm2, %xmm2899; AVX512DQ-NEXT: vpalignr {{.*#+}} xmm3 = xmm2[11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10]900; AVX512DQ-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]901; AVX512DQ-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[11,12,13,14,15],xmm2[0,1,2,3,4,5,6,7,8,9,10]902; AVX512DQ-NEXT: vpalignr {{.*#+}} xmm2 = xmm3[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]903; AVX512DQ-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 ^ (mem & (xmm1 ^ xmm0))904; AVX512DQ-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm3[0,1,2,3,4,5,6,7,8,9,10]905; AVX512DQ-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]906; AVX512DQ-NEXT: vmovdqa %xmm0, (%rsi)907; AVX512DQ-NEXT: vmovdqa %xmm1, (%rdx)908; AVX512DQ-NEXT: vmovdqa %xmm2, (%rcx)909; AVX512DQ-NEXT: retq910;911; AVX512DQ-FCP-LABEL: load_i8_stride3_vf16:912; AVX512DQ-FCP: # %bb.0:913; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm0914; AVX512DQ-FCP-NEXT: vmovdqa 16(%rdi), %xmm1915; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %xmm2916; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]917; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm0, %xmm0918; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm1919; AVX512DQ-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2920; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} xmm3 = xmm2[11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10]921; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]922; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[11,12,13,14,15],xmm2[0,1,2,3,4,5,6,7,8,9,10]923; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} xmm2 = xmm3[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]924; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 ^ (mem & (xmm1 ^ xmm0))925; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm3[0,1,2,3,4,5,6,7,8,9,10]926; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]927; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, (%rsi)928; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, (%rdx)929; AVX512DQ-FCP-NEXT: vmovdqa %xmm2, (%rcx)930; AVX512DQ-FCP-NEXT: retq931;932; AVX512BW-LABEL: load_i8_stride3_vf16:933; AVX512BW: # %bb.0:934; AVX512BW-NEXT: vmovdqa (%rdi), %xmm0935; AVX512BW-NEXT: vmovdqa 16(%rdi), %xmm1936; AVX512BW-NEXT: vmovdqa 32(%rdi), %xmm2937; AVX512BW-NEXT: vmovdqa {{.*#+}} xmm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]938; AVX512BW-NEXT: vpshufb %xmm3, %xmm0, %xmm0939; AVX512BW-NEXT: vpshufb %xmm3, %xmm1, %xmm1940; AVX512BW-NEXT: vpshufb %xmm3, %xmm2, %xmm2941; AVX512BW-NEXT: vpalignr {{.*#+}} xmm3 = xmm2[11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10]942; AVX512BW-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]943; AVX512BW-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[11,12,13,14,15],xmm2[0,1,2,3,4,5,6,7,8,9,10]944; AVX512BW-NEXT: movw $-2048, %ax # imm = 0xF800945; AVX512BW-NEXT: kmovd %eax, %k1946; AVX512BW-NEXT: vpblendmb %xmm1, %xmm0, %xmm2 {%k1}947; AVX512BW-NEXT: vpalignr {{.*#+}} xmm1 = xmm3[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]948; AVX512BW-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm3[0,1,2,3,4,5,6,7,8,9,10]949; AVX512BW-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]950; AVX512BW-NEXT: vmovdqa %xmm0, (%rsi)951; AVX512BW-NEXT: vmovdqa %xmm2, (%rdx)952; AVX512BW-NEXT: vmovdqa %xmm1, (%rcx)953; AVX512BW-NEXT: retq954;955; AVX512BW-FCP-LABEL: load_i8_stride3_vf16:956; AVX512BW-FCP: # %bb.0:957; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %xmm0958; AVX512BW-FCP-NEXT: vmovdqa 16(%rdi), %xmm1959; AVX512BW-FCP-NEXT: vmovdqa 32(%rdi), %xmm2960; AVX512BW-FCP-NEXT: vmovdqa {{.*#+}} xmm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]961; AVX512BW-FCP-NEXT: vpshufb %xmm3, %xmm0, %xmm0962; AVX512BW-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm1963; AVX512BW-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm2964; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} xmm3 = xmm2[11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10]965; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]966; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[11,12,13,14,15],xmm2[0,1,2,3,4,5,6,7,8,9,10]967; AVX512BW-FCP-NEXT: movw $-2048, %ax # imm = 0xF800968; AVX512BW-FCP-NEXT: kmovd %eax, %k1969; AVX512BW-FCP-NEXT: vpblendmb %xmm1, %xmm0, %xmm2 {%k1}970; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} xmm1 = xmm3[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]971; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm3[0,1,2,3,4,5,6,7,8,9,10]972; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]973; AVX512BW-FCP-NEXT: vmovdqa %xmm0, (%rsi)974; AVX512BW-FCP-NEXT: vmovdqa %xmm2, (%rdx)975; AVX512BW-FCP-NEXT: vmovdqa %xmm1, (%rcx)976; AVX512BW-FCP-NEXT: retq977;978; AVX512DQ-BW-LABEL: load_i8_stride3_vf16:979; AVX512DQ-BW: # %bb.0:980; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %xmm0981; AVX512DQ-BW-NEXT: vmovdqa 16(%rdi), %xmm1982; AVX512DQ-BW-NEXT: vmovdqa 32(%rdi), %xmm2983; AVX512DQ-BW-NEXT: vmovdqa {{.*#+}} xmm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]984; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm0, %xmm0985; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm1, %xmm1986; AVX512DQ-BW-NEXT: vpshufb %xmm3, %xmm2, %xmm2987; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} xmm3 = xmm2[11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10]988; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]989; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[11,12,13,14,15],xmm2[0,1,2,3,4,5,6,7,8,9,10]990; AVX512DQ-BW-NEXT: movw $-2048, %ax # imm = 0xF800991; AVX512DQ-BW-NEXT: kmovd %eax, %k1992; AVX512DQ-BW-NEXT: vpblendmb %xmm1, %xmm0, %xmm2 {%k1}993; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} xmm1 = xmm3[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]994; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm3[0,1,2,3,4,5,6,7,8,9,10]995; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]996; AVX512DQ-BW-NEXT: vmovdqa %xmm0, (%rsi)997; AVX512DQ-BW-NEXT: vmovdqa %xmm2, (%rdx)998; AVX512DQ-BW-NEXT: vmovdqa %xmm1, (%rcx)999; AVX512DQ-BW-NEXT: retq1000;1001; AVX512DQ-BW-FCP-LABEL: load_i8_stride3_vf16:1002; AVX512DQ-BW-FCP: # %bb.0:1003; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %xmm01004; AVX512DQ-BW-FCP-NEXT: vmovdqa 16(%rdi), %xmm11005; AVX512DQ-BW-FCP-NEXT: vmovdqa 32(%rdi), %xmm21006; AVX512DQ-BW-FCP-NEXT: vmovdqa {{.*#+}} xmm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]1007; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm3, %xmm0, %xmm01008; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm3, %xmm1, %xmm11009; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm3, %xmm2, %xmm21010; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} xmm3 = xmm2[11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10]1011; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]1012; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[11,12,13,14,15],xmm2[0,1,2,3,4,5,6,7,8,9,10]1013; AVX512DQ-BW-FCP-NEXT: movw $-2048, %ax # imm = 0xF8001014; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k11015; AVX512DQ-BW-FCP-NEXT: vpblendmb %xmm1, %xmm0, %xmm2 {%k1}1016; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} xmm1 = xmm3[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]1017; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm3[0,1,2,3,4,5,6,7,8,9,10]1018; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]1019; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm0, (%rsi)1020; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm2, (%rdx)1021; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm1, (%rcx)1022; AVX512DQ-BW-FCP-NEXT: retq1023 %wide.vec = load <48 x i8>, ptr %in.vec, align 641024 %strided.vec0 = shufflevector <48 x i8> %wide.vec, <48 x i8> poison, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>1025 %strided.vec1 = shufflevector <48 x i8> %wide.vec, <48 x i8> poison, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>1026 %strided.vec2 = shufflevector <48 x i8> %wide.vec, <48 x i8> poison, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>1027 store <16 x i8> %strided.vec0, ptr %out.vec0, align 641028 store <16 x i8> %strided.vec1, ptr %out.vec1, align 641029 store <16 x i8> %strided.vec2, ptr %out.vec2, align 641030 ret void1031}1032 1033define void @load_i8_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2) nounwind {1034; SSE-LABEL: load_i8_stride3_vf32:1035; SSE: # %bb.0:1036; SSE-NEXT: movdqa 64(%rdi), %xmm21037; SSE-NEXT: movdqa (%rdi), %xmm61038; SSE-NEXT: movdqa 16(%rdi), %xmm41039; SSE-NEXT: movdqa 32(%rdi), %xmm81040; SSE-NEXT: movdqa 48(%rdi), %xmm121041; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1042; SSE-NEXT: movdqa {{.*#+}} xmm13 = [255,255,0,255,255,0,255,255,0,255,255,0,255,255,0,255]1043; SSE-NEXT: movdqa %xmm13, %xmm71044; SSE-NEXT: pandn %xmm4, %xmm71045; SSE-NEXT: movdqa {{.*#+}} xmm5 = [255,0,255,255,0,255,255,0,255,255,0,255,255,0,255,255]1046; SSE-NEXT: movdqa %xmm5, %xmm141047; SSE-NEXT: pandn %xmm6, %xmm141048; SSE-NEXT: movdqa %xmm13, %xmm01049; SSE-NEXT: pandn %xmm6, %xmm01050; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1051; SSE-NEXT: movdqa %xmm6, %xmm01052; SSE-NEXT: pand %xmm13, %xmm01053; SSE-NEXT: por %xmm7, %xmm01054; SSE-NEXT: pxor %xmm9, %xmm91055; SSE-NEXT: movdqa %xmm0, %xmm11056; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm9[8],xmm1[9],xmm9[9],xmm1[10],xmm9[10],xmm1[11],xmm9[11],xmm1[12],xmm9[12],xmm1[13],xmm9[13],xmm1[14],xmm9[14],xmm1[15],xmm9[15]1057; SSE-NEXT: movdqa {{.*#+}} xmm6 = [65535,0,65535,65535,0,65535,65535,0]1058; SSE-NEXT: movdqa %xmm6, %xmm31059; SSE-NEXT: pandn %xmm1, %xmm31060; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3],xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7]1061; SSE-NEXT: pand %xmm6, %xmm01062; SSE-NEXT: por %xmm3, %xmm01063; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]1064; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5]1065; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]1066; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,1,4,5,6,7]1067; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]1068; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7]1069; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3]1070; SSE-NEXT: packuswb %xmm1, %xmm01071; SSE-NEXT: movdqa {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]1072; SSE-NEXT: pand %xmm7, %xmm01073; SSE-NEXT: movdqa %xmm8, %xmm31074; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm9[8],xmm3[9],xmm9[9],xmm3[10],xmm9[10],xmm3[11],xmm9[11],xmm3[12],xmm9[12],xmm3[13],xmm9[13],xmm3[14],xmm9[14],xmm3[15],xmm9[15]1075; SSE-NEXT: movdqa {{.*#+}} xmm15 = [65535,65535,0,65535,65535,0,65535,65535]1076; SSE-NEXT: movdqa %xmm15, %xmm11077; SSE-NEXT: pandn %xmm3, %xmm11078; SSE-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3],xmm8[4],xmm9[4],xmm8[5],xmm9[5],xmm8[6],xmm9[6],xmm8[7],xmm9[7]1079; SSE-NEXT: movdqa %xmm8, %xmm101080; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1081; SSE-NEXT: pand %xmm15, %xmm101082; SSE-NEXT: por %xmm1, %xmm101083; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm10[3,1,2,0]1084; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]1085; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,0]1086; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,5]1087; SSE-NEXT: packuswb %xmm1, %xmm11088; SSE-NEXT: movdqa %xmm7, %xmm101089; SSE-NEXT: pandn %xmm1, %xmm101090; SSE-NEXT: por %xmm0, %xmm101091; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1092; SSE-NEXT: movdqa %xmm13, %xmm01093; SSE-NEXT: pandn %xmm2, %xmm01094; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1095; SSE-NEXT: movdqa %xmm12, %xmm11096; SSE-NEXT: pand %xmm13, %xmm11097; SSE-NEXT: por %xmm0, %xmm11098; SSE-NEXT: movdqa %xmm1, %xmm01099; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm9[8],xmm0[9],xmm9[9],xmm0[10],xmm9[10],xmm0[11],xmm9[11],xmm0[12],xmm9[12],xmm0[13],xmm9[13],xmm0[14],xmm9[14],xmm0[15],xmm9[15]1100; SSE-NEXT: movdqa %xmm6, %xmm101101; SSE-NEXT: pandn %xmm0, %xmm101102; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm9[0],xmm1[1],xmm9[1],xmm1[2],xmm9[2],xmm1[3],xmm9[3],xmm1[4],xmm9[4],xmm1[5],xmm9[5],xmm1[6],xmm9[6],xmm1[7],xmm9[7]1103; SSE-NEXT: pand %xmm6, %xmm11104; SSE-NEXT: por %xmm10, %xmm11105; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]1106; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,5]1107; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,1]1108; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,1,4,5,6,7]1109; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,4,7]1110; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7]1111; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3]1112; SSE-NEXT: packuswb %xmm0, %xmm11113; SSE-NEXT: movdqa 80(%rdi), %xmm101114; SSE-NEXT: movdqa %xmm10, %xmm121115; SSE-NEXT: punpckhbw {{.*#+}} xmm12 = xmm12[8],xmm9[8],xmm12[9],xmm9[9],xmm12[10],xmm9[10],xmm12[11],xmm9[11],xmm12[12],xmm9[12],xmm12[13],xmm9[13],xmm12[14],xmm9[14],xmm12[15],xmm9[15]1116; SSE-NEXT: movdqa %xmm15, %xmm01117; SSE-NEXT: pandn %xmm12, %xmm01118; SSE-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3],xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]1119; SSE-NEXT: movdqa %xmm10, %xmm111120; SSE-NEXT: pand %xmm15, %xmm111121; SSE-NEXT: por %xmm0, %xmm111122; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm11[3,1,2,0]1123; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]1124; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,0]1125; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5]1126; SSE-NEXT: packuswb %xmm0, %xmm01127; SSE-NEXT: movdqa %xmm7, %xmm111128; SSE-NEXT: pandn %xmm0, %xmm111129; SSE-NEXT: pand %xmm7, %xmm11130; SSE-NEXT: por %xmm1, %xmm111131; SSE-NEXT: movdqa %xmm4, %xmm01132; SSE-NEXT: pand %xmm5, %xmm01133; SSE-NEXT: por %xmm14, %xmm01134; SSE-NEXT: movdqa %xmm0, %xmm11135; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm9[8],xmm1[9],xmm9[9],xmm1[10],xmm9[10],xmm1[11],xmm9[11],xmm1[12],xmm9[12],xmm1[13],xmm9[13],xmm1[14],xmm9[14],xmm1[15],xmm9[15]1136; SSE-NEXT: movdqa %xmm15, %xmm141137; SSE-NEXT: pandn %xmm1, %xmm141138; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3],xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7]1139; SSE-NEXT: pand %xmm15, %xmm01140; SSE-NEXT: por %xmm14, %xmm01141; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]1142; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]1143; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]1144; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,2,3,0,4,5,6,7]1145; SSE-NEXT: pshufhw {{.*#+}} xmm14 = xmm0[0,1,2,3,5,6,7,4]1146; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm1[0,1,2,3,4,7,6,7]1147; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]1148; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,2,3,3,4,5,6,7]1149; SSE-NEXT: packuswb %xmm0, %xmm141150; SSE-NEXT: movdqa %xmm15, %xmm01151; SSE-NEXT: pandn %xmm8, %xmm01152; SSE-NEXT: movdqa %xmm6, %xmm11153; SSE-NEXT: pandn %xmm3, %xmm11154; SSE-NEXT: pand %xmm15, %xmm31155; SSE-NEXT: por %xmm0, %xmm31156; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm3[0,3,2,3,4,5,6,7]1157; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7]1158; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0]1159; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,2,2,2,4,5,6,7]1160; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,6,7,4]1161; SSE-NEXT: packuswb %xmm0, %xmm01162; SSE-NEXT: movdqa %xmm7, %xmm31163; SSE-NEXT: pandn %xmm0, %xmm31164; SSE-NEXT: pand %xmm7, %xmm141165; SSE-NEXT: por %xmm14, %xmm31166; SSE-NEXT: pand %xmm5, %xmm21167; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload1168; SSE-NEXT: pandn %xmm8, %xmm51169; SSE-NEXT: por %xmm2, %xmm51170; SSE-NEXT: movdqa %xmm5, %xmm01171; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm9[8],xmm0[9],xmm9[9],xmm0[10],xmm9[10],xmm0[11],xmm9[11],xmm0[12],xmm9[12],xmm0[13],xmm9[13],xmm0[14],xmm9[14],xmm0[15],xmm9[15]1172; SSE-NEXT: movdqa %xmm15, %xmm141173; SSE-NEXT: pandn %xmm0, %xmm141174; SSE-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm9[0],xmm5[1],xmm9[1],xmm5[2],xmm9[2],xmm5[3],xmm9[3],xmm5[4],xmm9[4],xmm5[5],xmm9[5],xmm5[6],xmm9[6],xmm5[7],xmm9[7]1175; SSE-NEXT: pand %xmm15, %xmm51176; SSE-NEXT: por %xmm14, %xmm51177; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[2,1,0,3,4,5,6,7]1178; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,5,4,7]1179; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,3,2,1]1180; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,2,3,0,4,5,6,7]1181; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,6,7,4]1182; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7]1183; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]1184; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,2,3,3,4,5,6,7]1185; SSE-NEXT: packuswb %xmm0, %xmm51186; SSE-NEXT: movdqa %xmm12, %xmm01187; SSE-NEXT: pand %xmm15, %xmm01188; SSE-NEXT: pandn %xmm10, %xmm151189; SSE-NEXT: por %xmm0, %xmm151190; SSE-NEXT: pand %xmm7, %xmm51191; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm15[0,3,2,3,4,5,6,7]1192; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7]1193; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0]1194; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,2,2,2,4,5,6,7]1195; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,6,7,4]1196; SSE-NEXT: packuswb %xmm0, %xmm01197; SSE-NEXT: pandn %xmm0, %xmm71198; SSE-NEXT: por %xmm5, %xmm71199; SSE-NEXT: pand %xmm13, %xmm41200; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload1201; SSE-NEXT: movdqa %xmm4, %xmm01202; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm9[8],xmm0[9],xmm9[9],xmm0[10],xmm9[10],xmm0[11],xmm9[11],xmm0[12],xmm9[12],xmm0[13],xmm9[13],xmm0[14],xmm9[14],xmm0[15],xmm9[15]1203; SSE-NEXT: movdqa {{.*#+}} xmm5 = [0,65535,65535,0,65535,65535,0,65535]1204; SSE-NEXT: movdqa %xmm5, %xmm21205; SSE-NEXT: pandn %xmm0, %xmm21206; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm9[0],xmm4[1],xmm9[1],xmm4[2],xmm9[2],xmm4[3],xmm9[3],xmm4[4],xmm9[4],xmm4[5],xmm9[5],xmm4[6],xmm9[6],xmm4[7],xmm9[7]1207; SSE-NEXT: pand %xmm5, %xmm41208; SSE-NEXT: por %xmm2, %xmm41209; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm4[3,1,2,0]1210; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,5]1211; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[3,1,2,0]1212; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,0,3,4,5,6,7]1213; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm2[0,1,2,3,6,5,4,7]1214; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]1215; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]1216; SSE-NEXT: packuswb %xmm0, %xmm41217; SSE-NEXT: movdqa {{.*#+}} xmm2 = [65535,65535,65535,65535,65535,0,0,0]1218; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1219; SSE-NEXT: pand %xmm6, %xmm01220; SSE-NEXT: por %xmm1, %xmm01221; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]1222; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]1223; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]1224; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,3,4,5,6,7]1225; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]1226; SSE-NEXT: packuswb %xmm0, %xmm11227; SSE-NEXT: movdqa %xmm2, %xmm01228; SSE-NEXT: pandn %xmm1, %xmm01229; SSE-NEXT: pand %xmm2, %xmm41230; SSE-NEXT: por %xmm4, %xmm01231; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload1232; SSE-NEXT: pand %xmm13, %xmm11233; SSE-NEXT: pandn %xmm8, %xmm131234; SSE-NEXT: por %xmm1, %xmm131235; SSE-NEXT: movdqa %xmm13, %xmm11236; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm9[8],xmm1[9],xmm9[9],xmm1[10],xmm9[10],xmm1[11],xmm9[11],xmm1[12],xmm9[12],xmm1[13],xmm9[13],xmm1[14],xmm9[14],xmm1[15],xmm9[15]1237; SSE-NEXT: punpcklbw {{.*#+}} xmm13 = xmm13[0],xmm9[0],xmm13[1],xmm9[1],xmm13[2],xmm9[2],xmm13[3],xmm9[3],xmm13[4],xmm9[4],xmm13[5],xmm9[5],xmm13[6],xmm9[6],xmm13[7],xmm9[7]1238; SSE-NEXT: pand %xmm5, %xmm131239; SSE-NEXT: pandn %xmm1, %xmm51240; SSE-NEXT: por %xmm13, %xmm51241; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm5[3,1,2,0]1242; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,7,6,5]1243; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[3,1,2,0]1244; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[2,1,0,3,4,5,6,7]1245; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,6,5,4,7]1246; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]1247; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]1248; SSE-NEXT: packuswb %xmm1, %xmm41249; SSE-NEXT: pand %xmm6, %xmm101250; SSE-NEXT: pandn %xmm12, %xmm61251; SSE-NEXT: por %xmm10, %xmm61252; SSE-NEXT: pand %xmm2, %xmm41253; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm6[0,2,1,3]1254; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,2,1,4,5,6,7]1255; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]1256; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]1257; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,4,7]1258; SSE-NEXT: packuswb %xmm1, %xmm11259; SSE-NEXT: pandn %xmm1, %xmm21260; SSE-NEXT: por %xmm4, %xmm21261; SSE-NEXT: movdqa %xmm11, 16(%rsi)1262; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload1263; SSE-NEXT: movaps %xmm1, (%rsi)1264; SSE-NEXT: movdqa %xmm7, 16(%rdx)1265; SSE-NEXT: movdqa %xmm3, (%rdx)1266; SSE-NEXT: movdqa %xmm2, 16(%rcx)1267; SSE-NEXT: movdqa %xmm0, (%rcx)1268; SSE-NEXT: retq1269;1270; AVX-LABEL: load_i8_stride3_vf32:1271; AVX: # %bb.0:1272; AVX-NEXT: vmovdqa (%rdi), %xmm01273; AVX-NEXT: vmovdqa 16(%rdi), %xmm11274; AVX-NEXT: vmovdqa 32(%rdi), %xmm21275; AVX-NEXT: vmovdqa 48(%rdi), %xmm31276; AVX-NEXT: vmovdqa 64(%rdi), %xmm41277; AVX-NEXT: vmovdqa 80(%rdi), %xmm51278; AVX-NEXT: vmovdqa {{.*#+}} xmm6 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]1279; AVX-NEXT: vpshufb %xmm6, %xmm0, %xmm01280; AVX-NEXT: vpshufb %xmm6, %xmm3, %xmm31281; AVX-NEXT: vpshufb %xmm6, %xmm1, %xmm11282; AVX-NEXT: vpshufb %xmm6, %xmm4, %xmm41283; AVX-NEXT: vpshufb %xmm6, %xmm2, %xmm21284; AVX-NEXT: vpshufb %xmm6, %xmm5, %xmm51285; AVX-NEXT: vpalignr {{.*#+}} xmm6 = xmm5[11,12,13,14,15],xmm3[0,1,2,3,4,5,6,7,8,9,10]1286; AVX-NEXT: vpalignr {{.*#+}} xmm7 = xmm2[11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10]1287; AVX-NEXT: vpalignr {{.*#+}} xmm3 = xmm3[11,12,13,14,15],xmm4[0,1,2,3,4,5,6,7,8,9,10]1288; AVX-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]1289; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm81290; AVX-NEXT: vpalignr {{.*#+}} xmm4 = xmm4[11,12,13,14,15],xmm5[0,1,2,3,4,5,6,7,8,9,10]1291; AVX-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[11,12,13,14,15],xmm2[0,1,2,3,4,5,6,7,8,9,10]1292; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm1, %ymm21293; AVX-NEXT: vpalignr {{.*#+}} xmm1 = xmm7[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]1294; AVX-NEXT: vpalignr {{.*#+}} xmm4 = xmm6[11,12,13,14,15],xmm4[0,1,2,3,4,5,6,7,8,9,10]1295; AVX-NEXT: vbroadcastf128 {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]1296; AVX-NEXT: # ymm5 = mem[0,1,0,1]1297; AVX-NEXT: vandnps %ymm2, %ymm5, %ymm21298; AVX-NEXT: vandps %ymm5, %ymm8, %ymm51299; AVX-NEXT: vorps %ymm2, %ymm5, %ymm21300; AVX-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[11,12,13,14,15],xmm7[0,1,2,3,4,5,6,7,8,9,10]1301; AVX-NEXT: vpalignr {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]1302; AVX-NEXT: vpalignr {{.*#+}} xmm3 = xmm3[11,12,13,14,15],xmm6[0,1,2,3,4,5,6,7,8,9,10]1303; AVX-NEXT: vpalignr {{.*#+}} xmm3 = xmm3[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9]1304; AVX-NEXT: vmovdqa %xmm3, 16(%rsi)1305; AVX-NEXT: vmovdqa %xmm0, (%rsi)1306; AVX-NEXT: vmovaps %ymm2, (%rdx)1307; AVX-NEXT: vmovdqa %xmm4, 16(%rcx)1308; AVX-NEXT: vmovdqa %xmm1, (%rcx)1309; AVX-NEXT: vzeroupper1310; AVX-NEXT: retq1311;1312; AVX2-LABEL: load_i8_stride3_vf32:1313; AVX2: # %bb.0:1314; AVX2-NEXT: vmovdqa (%rdi), %xmm01315; AVX2-NEXT: vmovdqa 16(%rdi), %xmm11316; AVX2-NEXT: vmovdqa 32(%rdi), %xmm21317; AVX2-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm01318; AVX2-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm11319; AVX2-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm21320; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]1321; AVX2-NEXT: # ymm3 = mem[0,1,0,1]1322; AVX2-NEXT: vpshufb %ymm3, %ymm0, %ymm01323; AVX2-NEXT: vpshufb %ymm3, %ymm1, %ymm11324; AVX2-NEXT: vpshufb %ymm3, %ymm2, %ymm21325; AVX2-NEXT: vpalignr {{.*#+}} ymm3 = ymm2[11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7,8,9,10],ymm2[27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23,24,25,26]1326; AVX2-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]1327; AVX2-NEXT: vpalignr {{.*#+}} ymm1 = ymm1[11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7,8,9,10],ymm1[27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23,24,25,26]1328; AVX2-NEXT: vpalignr {{.*#+}} ymm2 = ymm3[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm3[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]1329; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]1330; AVX2-NEXT: # ymm4 = mem[0,1,0,1]1331; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm1, %ymm11332; AVX2-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]1333; AVX2-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]1334; AVX2-NEXT: vmovdqa %ymm0, (%rsi)1335; AVX2-NEXT: vmovdqa %ymm1, (%rdx)1336; AVX2-NEXT: vmovdqa %ymm2, (%rcx)1337; AVX2-NEXT: vzeroupper1338; AVX2-NEXT: retq1339;1340; AVX2-FP-LABEL: load_i8_stride3_vf32:1341; AVX2-FP: # %bb.0:1342; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm01343; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm11344; AVX2-FP-NEXT: vmovdqa 32(%rdi), %xmm21345; AVX2-FP-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm01346; AVX2-FP-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm11347; AVX2-FP-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm21348; AVX2-FP-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]1349; AVX2-FP-NEXT: # ymm3 = mem[0,1,0,1]1350; AVX2-FP-NEXT: vpshufb %ymm3, %ymm0, %ymm01351; AVX2-FP-NEXT: vpshufb %ymm3, %ymm1, %ymm11352; AVX2-FP-NEXT: vpshufb %ymm3, %ymm2, %ymm21353; AVX2-FP-NEXT: vpalignr {{.*#+}} ymm3 = ymm2[11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7,8,9,10],ymm2[27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23,24,25,26]1354; AVX2-FP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]1355; AVX2-FP-NEXT: vpalignr {{.*#+}} ymm1 = ymm1[11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7,8,9,10],ymm1[27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23,24,25,26]1356; AVX2-FP-NEXT: vpalignr {{.*#+}} ymm2 = ymm3[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm3[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]1357; AVX2-FP-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]1358; AVX2-FP-NEXT: # ymm4 = mem[0,1,0,1]1359; AVX2-FP-NEXT: vpblendvb %ymm4, %ymm0, %ymm1, %ymm11360; AVX2-FP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]1361; AVX2-FP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]1362; AVX2-FP-NEXT: vmovdqa %ymm0, (%rsi)1363; AVX2-FP-NEXT: vmovdqa %ymm1, (%rdx)1364; AVX2-FP-NEXT: vmovdqa %ymm2, (%rcx)1365; AVX2-FP-NEXT: vzeroupper1366; AVX2-FP-NEXT: retq1367;1368; AVX2-FCP-LABEL: load_i8_stride3_vf32:1369; AVX2-FCP: # %bb.0:1370; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm01371; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm11372; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %xmm21373; AVX2-FCP-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm01374; AVX2-FCP-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm11375; AVX2-FCP-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm21376; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]1377; AVX2-FCP-NEXT: # ymm3 = mem[0,1,0,1]1378; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm0, %ymm01379; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm1, %ymm11380; AVX2-FCP-NEXT: vpshufb %ymm3, %ymm2, %ymm21381; AVX2-FCP-NEXT: vpalignr {{.*#+}} ymm3 = ymm2[11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7,8,9,10],ymm2[27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23,24,25,26]1382; AVX2-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]1383; AVX2-FCP-NEXT: vpalignr {{.*#+}} ymm1 = ymm1[11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7,8,9,10],ymm1[27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23,24,25,26]1384; AVX2-FCP-NEXT: vpalignr {{.*#+}} ymm2 = ymm3[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm3[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]1385; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]1386; AVX2-FCP-NEXT: # ymm4 = mem[0,1,0,1]1387; AVX2-FCP-NEXT: vpblendvb %ymm4, %ymm0, %ymm1, %ymm11388; AVX2-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]1389; AVX2-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]1390; AVX2-FCP-NEXT: vmovdqa %ymm0, (%rsi)1391; AVX2-FCP-NEXT: vmovdqa %ymm1, (%rdx)1392; AVX2-FCP-NEXT: vmovdqa %ymm2, (%rcx)1393; AVX2-FCP-NEXT: vzeroupper1394; AVX2-FCP-NEXT: retq1395;1396; AVX512-LABEL: load_i8_stride3_vf32:1397; AVX512: # %bb.0:1398; AVX512-NEXT: vmovdqa (%rdi), %xmm01399; AVX512-NEXT: vmovdqa 16(%rdi), %xmm11400; AVX512-NEXT: vmovdqa 32(%rdi), %xmm21401; AVX512-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm01402; AVX512-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm11403; AVX512-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm21404; AVX512-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]1405; AVX512-NEXT: # ymm3 = mem[0,1,0,1]1406; AVX512-NEXT: vpshufb %ymm3, %ymm0, %ymm01407; AVX512-NEXT: vpshufb %ymm3, %ymm1, %ymm11408; AVX512-NEXT: vpshufb %ymm3, %ymm2, %ymm21409; AVX512-NEXT: vpalignr {{.*#+}} ymm3 = ymm2[11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7,8,9,10],ymm2[27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23,24,25,26]1410; AVX512-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]1411; AVX512-NEXT: vpalignr {{.*#+}} ymm1 = ymm1[11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7,8,9,10],ymm1[27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23,24,25,26]1412; AVX512-NEXT: vpalignr {{.*#+}} ymm2 = ymm3[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm3[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]1413; AVX512-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]1414; AVX512-NEXT: # ymm4 = mem[0,1,0,1]1415; AVX512-NEXT: vpternlogq {{.*#+}} ymm4 = ymm1 ^ (ymm4 & (ymm0 ^ ymm1))1416; AVX512-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]1417; AVX512-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]1418; AVX512-NEXT: vmovdqa %ymm0, (%rsi)1419; AVX512-NEXT: vmovdqa %ymm4, (%rdx)1420; AVX512-NEXT: vmovdqa %ymm2, (%rcx)1421; AVX512-NEXT: vzeroupper1422; AVX512-NEXT: retq1423;1424; AVX512-FCP-LABEL: load_i8_stride3_vf32:1425; AVX512-FCP: # %bb.0:1426; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm01427; AVX512-FCP-NEXT: vmovdqa 16(%rdi), %xmm11428; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %xmm21429; AVX512-FCP-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm01430; AVX512-FCP-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm11431; AVX512-FCP-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm21432; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]1433; AVX512-FCP-NEXT: # ymm3 = mem[0,1,0,1]1434; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm0, %ymm01435; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm1, %ymm11436; AVX512-FCP-NEXT: vpshufb %ymm3, %ymm2, %ymm21437; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm3 = ymm2[11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7,8,9,10],ymm2[27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23,24,25,26]1438; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]1439; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm1 = ymm1[11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7,8,9,10],ymm1[27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23,24,25,26]1440; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm2 = ymm3[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm3[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]1441; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]1442; AVX512-FCP-NEXT: # ymm4 = mem[0,1,0,1]1443; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm4 = ymm1 ^ (ymm4 & (ymm0 ^ ymm1))1444; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]1445; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]1446; AVX512-FCP-NEXT: vmovdqa %ymm0, (%rsi)1447; AVX512-FCP-NEXT: vmovdqa %ymm4, (%rdx)1448; AVX512-FCP-NEXT: vmovdqa %ymm2, (%rcx)1449; AVX512-FCP-NEXT: vzeroupper1450; AVX512-FCP-NEXT: retq1451;1452; AVX512DQ-LABEL: load_i8_stride3_vf32:1453; AVX512DQ: # %bb.0:1454; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm01455; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm11456; AVX512DQ-NEXT: vmovdqa 32(%rdi), %xmm21457; AVX512DQ-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm01458; AVX512DQ-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm11459; AVX512DQ-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm21460; AVX512DQ-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]1461; AVX512DQ-NEXT: # ymm3 = mem[0,1,0,1]1462; AVX512DQ-NEXT: vpshufb %ymm3, %ymm0, %ymm01463; AVX512DQ-NEXT: vpshufb %ymm3, %ymm1, %ymm11464; AVX512DQ-NEXT: vpshufb %ymm3, %ymm2, %ymm21465; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm3 = ymm2[11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7,8,9,10],ymm2[27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23,24,25,26]1466; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]1467; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm1 = ymm1[11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7,8,9,10],ymm1[27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23,24,25,26]1468; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm2 = ymm3[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm3[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]1469; AVX512DQ-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]1470; AVX512DQ-NEXT: # ymm4 = mem[0,1,0,1]1471; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm4 = ymm1 ^ (ymm4 & (ymm0 ^ ymm1))1472; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]1473; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]1474; AVX512DQ-NEXT: vmovdqa %ymm0, (%rsi)1475; AVX512DQ-NEXT: vmovdqa %ymm4, (%rdx)1476; AVX512DQ-NEXT: vmovdqa %ymm2, (%rcx)1477; AVX512DQ-NEXT: vzeroupper1478; AVX512DQ-NEXT: retq1479;1480; AVX512DQ-FCP-LABEL: load_i8_stride3_vf32:1481; AVX512DQ-FCP: # %bb.0:1482; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm01483; AVX512DQ-FCP-NEXT: vmovdqa 16(%rdi), %xmm11484; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %xmm21485; AVX512DQ-FCP-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm01486; AVX512DQ-FCP-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm11487; AVX512DQ-FCP-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm21488; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]1489; AVX512DQ-FCP-NEXT: # ymm3 = mem[0,1,0,1]1490; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm0, %ymm01491; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm1, %ymm11492; AVX512DQ-FCP-NEXT: vpshufb %ymm3, %ymm2, %ymm21493; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm3 = ymm2[11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7,8,9,10],ymm2[27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23,24,25,26]1494; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]1495; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm1 = ymm1[11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7,8,9,10],ymm1[27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23,24,25,26]1496; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm2 = ymm3[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm3[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]1497; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]1498; AVX512DQ-FCP-NEXT: # ymm4 = mem[0,1,0,1]1499; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm4 = ymm1 ^ (ymm4 & (ymm0 ^ ymm1))1500; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]1501; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]1502; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, (%rsi)1503; AVX512DQ-FCP-NEXT: vmovdqa %ymm4, (%rdx)1504; AVX512DQ-FCP-NEXT: vmovdqa %ymm2, (%rcx)1505; AVX512DQ-FCP-NEXT: vzeroupper1506; AVX512DQ-FCP-NEXT: retq1507;1508; AVX512BW-LABEL: load_i8_stride3_vf32:1509; AVX512BW: # %bb.0:1510; AVX512BW-NEXT: vmovdqa (%rdi), %xmm01511; AVX512BW-NEXT: vmovdqa 16(%rdi), %xmm11512; AVX512BW-NEXT: vmovdqa 32(%rdi), %xmm21513; AVX512BW-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm01514; AVX512BW-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm11515; AVX512BW-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm21516; AVX512BW-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]1517; AVX512BW-NEXT: # ymm3 = mem[0,1,0,1]1518; AVX512BW-NEXT: vpshufb %ymm3, %ymm0, %ymm01519; AVX512BW-NEXT: vpshufb %ymm3, %ymm1, %ymm11520; AVX512BW-NEXT: vpshufb %ymm3, %ymm2, %ymm21521; AVX512BW-NEXT: vpalignr {{.*#+}} ymm3 = ymm2[11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7,8,9,10],ymm2[27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23,24,25,26]1522; AVX512BW-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]1523; AVX512BW-NEXT: vpalignr {{.*#+}} ymm1 = ymm1[11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7,8,9,10],ymm1[27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23,24,25,26]1524; AVX512BW-NEXT: movl $-134154240, %eax # imm = 0xF800F8001525; AVX512BW-NEXT: kmovd %eax, %k11526; AVX512BW-NEXT: vpblendmb %ymm1, %ymm0, %ymm2 {%k1}1527; AVX512BW-NEXT: vpalignr {{.*#+}} ymm1 = ymm3[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm3[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]1528; AVX512BW-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]1529; AVX512BW-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]1530; AVX512BW-NEXT: vmovdqa %ymm0, (%rsi)1531; AVX512BW-NEXT: vmovdqa %ymm2, (%rdx)1532; AVX512BW-NEXT: vmovdqa %ymm1, (%rcx)1533; AVX512BW-NEXT: vzeroupper1534; AVX512BW-NEXT: retq1535;1536; AVX512BW-FCP-LABEL: load_i8_stride3_vf32:1537; AVX512BW-FCP: # %bb.0:1538; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %xmm01539; AVX512BW-FCP-NEXT: vmovdqa 16(%rdi), %xmm11540; AVX512BW-FCP-NEXT: vmovdqa 32(%rdi), %xmm21541; AVX512BW-FCP-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm01542; AVX512BW-FCP-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm11543; AVX512BW-FCP-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm21544; AVX512BW-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]1545; AVX512BW-FCP-NEXT: # ymm3 = mem[0,1,0,1]1546; AVX512BW-FCP-NEXT: vpshufb %ymm3, %ymm0, %ymm01547; AVX512BW-FCP-NEXT: vpshufb %ymm3, %ymm1, %ymm11548; AVX512BW-FCP-NEXT: vpshufb %ymm3, %ymm2, %ymm21549; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm3 = ymm2[11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7,8,9,10],ymm2[27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23,24,25,26]1550; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]1551; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm1 = ymm1[11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7,8,9,10],ymm1[27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23,24,25,26]1552; AVX512BW-FCP-NEXT: movl $-134154240, %eax # imm = 0xF800F8001553; AVX512BW-FCP-NEXT: kmovd %eax, %k11554; AVX512BW-FCP-NEXT: vpblendmb %ymm1, %ymm0, %ymm2 {%k1}1555; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm1 = ymm3[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm3[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]1556; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]1557; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]1558; AVX512BW-FCP-NEXT: vmovdqa %ymm0, (%rsi)1559; AVX512BW-FCP-NEXT: vmovdqa %ymm2, (%rdx)1560; AVX512BW-FCP-NEXT: vmovdqa %ymm1, (%rcx)1561; AVX512BW-FCP-NEXT: vzeroupper1562; AVX512BW-FCP-NEXT: retq1563;1564; AVX512DQ-BW-LABEL: load_i8_stride3_vf32:1565; AVX512DQ-BW: # %bb.0:1566; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %xmm01567; AVX512DQ-BW-NEXT: vmovdqa 16(%rdi), %xmm11568; AVX512DQ-BW-NEXT: vmovdqa 32(%rdi), %xmm21569; AVX512DQ-BW-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm01570; AVX512DQ-BW-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm11571; AVX512DQ-BW-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm21572; AVX512DQ-BW-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]1573; AVX512DQ-BW-NEXT: # ymm3 = mem[0,1,0,1]1574; AVX512DQ-BW-NEXT: vpshufb %ymm3, %ymm0, %ymm01575; AVX512DQ-BW-NEXT: vpshufb %ymm3, %ymm1, %ymm11576; AVX512DQ-BW-NEXT: vpshufb %ymm3, %ymm2, %ymm21577; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} ymm3 = ymm2[11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7,8,9,10],ymm2[27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23,24,25,26]1578; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]1579; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} ymm1 = ymm1[11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7,8,9,10],ymm1[27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23,24,25,26]1580; AVX512DQ-BW-NEXT: movl $-134154240, %eax # imm = 0xF800F8001581; AVX512DQ-BW-NEXT: kmovd %eax, %k11582; AVX512DQ-BW-NEXT: vpblendmb %ymm1, %ymm0, %ymm2 {%k1}1583; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} ymm1 = ymm3[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm3[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]1584; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]1585; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]1586; AVX512DQ-BW-NEXT: vmovdqa %ymm0, (%rsi)1587; AVX512DQ-BW-NEXT: vmovdqa %ymm2, (%rdx)1588; AVX512DQ-BW-NEXT: vmovdqa %ymm1, (%rcx)1589; AVX512DQ-BW-NEXT: vzeroupper1590; AVX512DQ-BW-NEXT: retq1591;1592; AVX512DQ-BW-FCP-LABEL: load_i8_stride3_vf32:1593; AVX512DQ-BW-FCP: # %bb.0:1594; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %xmm01595; AVX512DQ-BW-FCP-NEXT: vmovdqa 16(%rdi), %xmm11596; AVX512DQ-BW-FCP-NEXT: vmovdqa 32(%rdi), %xmm21597; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm01598; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm11599; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm21600; AVX512DQ-BW-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]1601; AVX512DQ-BW-FCP-NEXT: # ymm3 = mem[0,1,0,1]1602; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm3, %ymm0, %ymm01603; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm3, %ymm1, %ymm11604; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm3, %ymm2, %ymm21605; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm3 = ymm2[11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7,8,9,10],ymm2[27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23,24,25,26]1606; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]1607; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm1 = ymm1[11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7,8,9,10],ymm1[27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23,24,25,26]1608; AVX512DQ-BW-FCP-NEXT: movl $-134154240, %eax # imm = 0xF800F8001609; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k11610; AVX512DQ-BW-FCP-NEXT: vpblendmb %ymm1, %ymm0, %ymm2 {%k1}1611; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm1 = ymm3[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm3[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]1612; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]1613; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]1614; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm0, (%rsi)1615; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm2, (%rdx)1616; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm1, (%rcx)1617; AVX512DQ-BW-FCP-NEXT: vzeroupper1618; AVX512DQ-BW-FCP-NEXT: retq1619 %wide.vec = load <96 x i8>, ptr %in.vec, align 641620 %strided.vec0 = shufflevector <96 x i8> %wide.vec, <96 x i8> poison, <32 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45, i32 48, i32 51, i32 54, i32 57, i32 60, i32 63, i32 66, i32 69, i32 72, i32 75, i32 78, i32 81, i32 84, i32 87, i32 90, i32 93>1621 %strided.vec1 = shufflevector <96 x i8> %wide.vec, <96 x i8> poison, <32 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46, i32 49, i32 52, i32 55, i32 58, i32 61, i32 64, i32 67, i32 70, i32 73, i32 76, i32 79, i32 82, i32 85, i32 88, i32 91, i32 94>1622 %strided.vec2 = shufflevector <96 x i8> %wide.vec, <96 x i8> poison, <32 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47, i32 50, i32 53, i32 56, i32 59, i32 62, i32 65, i32 68, i32 71, i32 74, i32 77, i32 80, i32 83, i32 86, i32 89, i32 92, i32 95>1623 store <32 x i8> %strided.vec0, ptr %out.vec0, align 641624 store <32 x i8> %strided.vec1, ptr %out.vec1, align 641625 store <32 x i8> %strided.vec2, ptr %out.vec2, align 641626 ret void1627}1628 1629define void @load_i8_stride3_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2) nounwind {1630; SSE-LABEL: load_i8_stride3_vf64:1631; SSE: # %bb.0:1632; SSE-NEXT: subq $168, %rsp1633; SSE-NEXT: movdqa 80(%rdi), %xmm91634; SSE-NEXT: movdqa (%rdi), %xmm01635; SSE-NEXT: movdqa 16(%rdi), %xmm101636; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1637; SSE-NEXT: movdqa 48(%rdi), %xmm51638; SSE-NEXT: movdqa 64(%rdi), %xmm11639; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1640; SSE-NEXT: movdqa {{.*#+}} xmm2 = [255,255,0,255,255,0,255,255,0,255,255,0,255,255,0,255]1641; SSE-NEXT: movdqa %xmm2, %xmm61642; SSE-NEXT: pandn %xmm1, %xmm61643; SSE-NEXT: movdqa {{.*#+}} xmm4 = [255,0,255,255,0,255,255,0,255,255,0,255,255,0,255,255]1644; SSE-NEXT: movdqa %xmm4, %xmm11645; SSE-NEXT: pandn %xmm5, %xmm11646; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1647; SSE-NEXT: movdqa %xmm2, %xmm11648; SSE-NEXT: pandn %xmm5, %xmm11649; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1650; SSE-NEXT: pand %xmm2, %xmm51651; SSE-NEXT: movdqa %xmm2, %xmm111652; SSE-NEXT: por %xmm6, %xmm51653; SSE-NEXT: pxor %xmm8, %xmm81654; SSE-NEXT: movdqa %xmm5, %xmm11655; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm8[8],xmm1[9],xmm8[9],xmm1[10],xmm8[10],xmm1[11],xmm8[11],xmm1[12],xmm8[12],xmm1[13],xmm8[13],xmm1[14],xmm8[14],xmm1[15],xmm8[15]1656; SSE-NEXT: movdqa {{.*#+}} xmm6 = [65535,0,65535,65535,0,65535,65535,0]1657; SSE-NEXT: movdqa %xmm6, %xmm21658; SSE-NEXT: pandn %xmm1, %xmm21659; SSE-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm8[0],xmm5[1],xmm8[1],xmm5[2],xmm8[2],xmm5[3],xmm8[3],xmm5[4],xmm8[4],xmm5[5],xmm8[5],xmm5[6],xmm8[6],xmm5[7],xmm8[7]1660; SSE-NEXT: pand %xmm6, %xmm51661; SSE-NEXT: por %xmm2, %xmm51662; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm5[0,2,1,3]1663; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,5]1664; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,1]1665; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,3,2,1,4,5,6,7]1666; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7]1667; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7]1668; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3]1669; SSE-NEXT: packuswb %xmm1, %xmm21670; SSE-NEXT: movdqa {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]1671; SSE-NEXT: pand %xmm7, %xmm21672; SSE-NEXT: movdqa %xmm9, %xmm131673; SSE-NEXT: punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm8[8],xmm13[9],xmm8[9],xmm13[10],xmm8[10],xmm13[11],xmm8[11],xmm13[12],xmm8[12],xmm13[13],xmm8[13],xmm13[14],xmm8[14],xmm13[15],xmm8[15]1674; SSE-NEXT: movdqa {{.*#+}} xmm12 = [65535,65535,0,65535,65535,0,65535,65535]1675; SSE-NEXT: movdqa %xmm12, %xmm11676; SSE-NEXT: pandn %xmm13, %xmm11677; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3],xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]1678; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1679; SSE-NEXT: pand %xmm12, %xmm91680; SSE-NEXT: por %xmm1, %xmm91681; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm9[3,1,2,0]1682; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]1683; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,0]1684; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,5]1685; SSE-NEXT: packuswb %xmm1, %xmm11686; SSE-NEXT: movdqa %xmm7, %xmm31687; SSE-NEXT: pandn %xmm1, %xmm31688; SSE-NEXT: por %xmm2, %xmm31689; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1690; SSE-NEXT: movdqa %xmm11, %xmm11691; SSE-NEXT: pandn %xmm10, %xmm11692; SSE-NEXT: movdqa %xmm4, %xmm21693; SSE-NEXT: pandn %xmm0, %xmm21694; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1695; SSE-NEXT: movdqa %xmm11, %xmm21696; SSE-NEXT: pandn %xmm0, %xmm21697; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1698; SSE-NEXT: pand %xmm11, %xmm01699; SSE-NEXT: por %xmm1, %xmm01700; SSE-NEXT: movdqa %xmm0, %xmm11701; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm8[8],xmm1[9],xmm8[9],xmm1[10],xmm8[10],xmm1[11],xmm8[11],xmm1[12],xmm8[12],xmm1[13],xmm8[13],xmm1[14],xmm8[14],xmm1[15],xmm8[15]1702; SSE-NEXT: movdqa %xmm6, %xmm21703; SSE-NEXT: pandn %xmm1, %xmm21704; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1],xmm0[2],xmm8[2],xmm0[3],xmm8[3],xmm0[4],xmm8[4],xmm0[5],xmm8[5],xmm0[6],xmm8[6],xmm0[7],xmm8[7]1705; SSE-NEXT: pand %xmm6, %xmm01706; SSE-NEXT: por %xmm2, %xmm01707; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]1708; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5]1709; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]1710; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,1,4,5,6,7]1711; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]1712; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7]1713; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3]1714; SSE-NEXT: packuswb %xmm1, %xmm01715; SSE-NEXT: movdqa 32(%rdi), %xmm151716; SSE-NEXT: movdqa %xmm15, %xmm101717; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm8[8],xmm10[9],xmm8[9],xmm10[10],xmm8[10],xmm10[11],xmm8[11],xmm10[12],xmm8[12],xmm10[13],xmm8[13],xmm10[14],xmm8[14],xmm10[15],xmm8[15]1718; SSE-NEXT: movdqa %xmm12, %xmm11719; SSE-NEXT: pandn %xmm10, %xmm11720; SSE-NEXT: punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm8[0],xmm15[1],xmm8[1],xmm15[2],xmm8[2],xmm15[3],xmm8[3],xmm15[4],xmm8[4],xmm15[5],xmm8[5],xmm15[6],xmm8[6],xmm15[7],xmm8[7]1721; SSE-NEXT: movdqa %xmm15, %xmm21722; SSE-NEXT: pand %xmm12, %xmm21723; SSE-NEXT: por %xmm1, %xmm21724; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,1,2,0]1725; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]1726; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,0]1727; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,5]1728; SSE-NEXT: packuswb %xmm1, %xmm11729; SSE-NEXT: movdqa %xmm7, %xmm21730; SSE-NEXT: pandn %xmm1, %xmm21731; SSE-NEXT: pand %xmm7, %xmm01732; SSE-NEXT: por %xmm0, %xmm21733; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1734; SSE-NEXT: movdqa 160(%rdi), %xmm141735; SSE-NEXT: movdqa %xmm11, %xmm01736; SSE-NEXT: pandn %xmm14, %xmm01737; SSE-NEXT: movdqa 144(%rdi), %xmm11738; SSE-NEXT: movdqa %xmm4, %xmm21739; SSE-NEXT: pandn %xmm1, %xmm21740; SSE-NEXT: movdqa %xmm2, (%rsp) # 16-byte Spill1741; SSE-NEXT: movdqa %xmm11, %xmm51742; SSE-NEXT: movdqa %xmm11, %xmm21743; SSE-NEXT: pandn %xmm1, %xmm21744; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1745; SSE-NEXT: pand %xmm11, %xmm11746; SSE-NEXT: por %xmm0, %xmm11747; SSE-NEXT: movdqa %xmm1, %xmm01748; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15]1749; SSE-NEXT: movdqa %xmm6, %xmm21750; SSE-NEXT: pandn %xmm0, %xmm21751; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3],xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7]1752; SSE-NEXT: pand %xmm6, %xmm11753; SSE-NEXT: por %xmm2, %xmm11754; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]1755; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,5]1756; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,1]1757; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,1,4,5,6,7]1758; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm1[0,1,2,3,6,5,4,7]1759; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7]1760; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3]1761; SSE-NEXT: packuswb %xmm0, %xmm91762; SSE-NEXT: movdqa 176(%rdi), %xmm31763; SSE-NEXT: movdqa %xmm3, %xmm11764; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm8[8],xmm1[9],xmm8[9],xmm1[10],xmm8[10],xmm1[11],xmm8[11],xmm1[12],xmm8[12],xmm1[13],xmm8[13],xmm1[14],xmm8[14],xmm1[15],xmm8[15]1765; SSE-NEXT: movdqa %xmm12, %xmm01766; SSE-NEXT: pandn %xmm1, %xmm01767; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm8[0],xmm3[1],xmm8[1],xmm3[2],xmm8[2],xmm3[3],xmm8[3],xmm3[4],xmm8[4],xmm3[5],xmm8[5],xmm3[6],xmm8[6],xmm3[7],xmm8[7]1768; SSE-NEXT: movdqa %xmm3, %xmm111769; SSE-NEXT: pand %xmm12, %xmm111770; SSE-NEXT: por %xmm0, %xmm111771; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm11[3,1,2,0]1772; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]1773; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,0]1774; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5]1775; SSE-NEXT: packuswb %xmm0, %xmm01776; SSE-NEXT: movdqa %xmm7, %xmm21777; SSE-NEXT: pandn %xmm0, %xmm21778; SSE-NEXT: pand %xmm7, %xmm91779; SSE-NEXT: por %xmm9, %xmm21780; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1781; SSE-NEXT: movdqa 112(%rdi), %xmm91782; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1783; SSE-NEXT: movdqa %xmm5, %xmm01784; SSE-NEXT: pandn %xmm9, %xmm01785; SSE-NEXT: movdqa 96(%rdi), %xmm91786; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1787; SSE-NEXT: pand %xmm5, %xmm91788; SSE-NEXT: por %xmm0, %xmm91789; SSE-NEXT: movdqa %xmm9, %xmm01790; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15]1791; SSE-NEXT: movdqa %xmm6, %xmm111792; SSE-NEXT: pandn %xmm0, %xmm111793; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3],xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]1794; SSE-NEXT: pand %xmm6, %xmm91795; SSE-NEXT: por %xmm11, %xmm91796; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,2,1,3]1797; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,7,6,5]1798; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,3,2,1]1799; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[0,3,2,1,4,5,6,7]1800; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,6,5,4,7]1801; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7]1802; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3]1803; SSE-NEXT: packuswb %xmm0, %xmm91804; SSE-NEXT: movdqa 128(%rdi), %xmm21805; SSE-NEXT: movdqa %xmm2, %xmm111806; SSE-NEXT: punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm8[8],xmm11[9],xmm8[9],xmm11[10],xmm8[10],xmm11[11],xmm8[11],xmm11[12],xmm8[12],xmm11[13],xmm8[13],xmm11[14],xmm8[14],xmm11[15],xmm8[15]1807; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1808; SSE-NEXT: movdqa %xmm12, %xmm01809; SSE-NEXT: pandn %xmm11, %xmm01810; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm8[0],xmm2[1],xmm8[1],xmm2[2],xmm8[2],xmm2[3],xmm8[3],xmm2[4],xmm8[4],xmm2[5],xmm8[5],xmm2[6],xmm8[6],xmm2[7],xmm8[7]1811; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1812; SSE-NEXT: movdqa %xmm2, %xmm111813; SSE-NEXT: pand %xmm12, %xmm111814; SSE-NEXT: por %xmm0, %xmm111815; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm11[3,1,2,0]1816; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]1817; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,0]1818; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5]1819; SSE-NEXT: packuswb %xmm0, %xmm01820; SSE-NEXT: movdqa %xmm7, %xmm21821; SSE-NEXT: pandn %xmm0, %xmm21822; SSE-NEXT: pand %xmm7, %xmm91823; SSE-NEXT: por %xmm9, %xmm21824; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1825; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1826; SSE-NEXT: pand %xmm4, %xmm01827; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload1828; SSE-NEXT: movdqa %xmm0, %xmm91829; SSE-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm8[8],xmm9[9],xmm8[9],xmm9[10],xmm8[10],xmm9[11],xmm8[11],xmm9[12],xmm8[12],xmm9[13],xmm8[13],xmm9[14],xmm8[14],xmm9[15],xmm8[15]1830; SSE-NEXT: movdqa %xmm12, %xmm111831; SSE-NEXT: pandn %xmm9, %xmm111832; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1],xmm0[2],xmm8[2],xmm0[3],xmm8[3],xmm0[4],xmm8[4],xmm0[5],xmm8[5],xmm0[6],xmm8[6],xmm0[7],xmm8[7]1833; SSE-NEXT: pand %xmm12, %xmm01834; SSE-NEXT: por %xmm11, %xmm01835; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]1836; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]1837; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]1838; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,2,3,0,4,5,6,7]1839; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,6,7,4]1840; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,7,6,7]1841; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,2,2,3]1842; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[1,2,3,3,4,5,6,7]1843; SSE-NEXT: packuswb %xmm9, %xmm01844; SSE-NEXT: movdqa %xmm12, %xmm91845; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload1846; SSE-NEXT: pandn %xmm5, %xmm91847; SSE-NEXT: movdqa %xmm6, %xmm21848; SSE-NEXT: pandn %xmm13, %xmm21849; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1850; SSE-NEXT: pand %xmm12, %xmm131851; SSE-NEXT: por %xmm9, %xmm131852; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm13[0,3,2,3,4,5,6,7]1853; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,6,5,6,7]1854; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,1,2,0]1855; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[2,2,2,2,4,5,6,7]1856; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,5,6,7,4]1857; SSE-NEXT: packuswb %xmm9, %xmm91858; SSE-NEXT: movdqa %xmm7, %xmm21859; SSE-NEXT: pandn %xmm9, %xmm21860; SSE-NEXT: pand %xmm7, %xmm01861; SSE-NEXT: por %xmm0, %xmm21862; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1863; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1864; SSE-NEXT: pand %xmm4, %xmm01865; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload1866; SSE-NEXT: movdqa %xmm0, %xmm91867; SSE-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm8[8],xmm9[9],xmm8[9],xmm9[10],xmm8[10],xmm9[11],xmm8[11],xmm9[12],xmm8[12],xmm9[13],xmm8[13],xmm9[14],xmm8[14],xmm9[15],xmm8[15]1868; SSE-NEXT: movdqa %xmm12, %xmm111869; SSE-NEXT: pandn %xmm9, %xmm111870; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1],xmm0[2],xmm8[2],xmm0[3],xmm8[3],xmm0[4],xmm8[4],xmm0[5],xmm8[5],xmm0[6],xmm8[6],xmm0[7],xmm8[7]1871; SSE-NEXT: pand %xmm12, %xmm01872; SSE-NEXT: por %xmm11, %xmm01873; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]1874; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]1875; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]1876; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,2,3,0,4,5,6,7]1877; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,6,7,4]1878; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,7,6,7]1879; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,2,2,3]1880; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[1,2,3,3,4,5,6,7]1881; SSE-NEXT: packuswb %xmm9, %xmm01882; SSE-NEXT: movdqa %xmm12, %xmm111883; SSE-NEXT: pandn %xmm15, %xmm111884; SSE-NEXT: movdqa %xmm6, %xmm91885; SSE-NEXT: pandn %xmm10, %xmm91886; SSE-NEXT: pand %xmm12, %xmm101887; SSE-NEXT: por %xmm11, %xmm101888; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[0,3,2,3,4,5,6,7]1889; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,5,6,7]1890; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,1,2,0]1891; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[2,2,2,2,4,5,6,7]1892; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,6,7,4]1893; SSE-NEXT: packuswb %xmm10, %xmm111894; SSE-NEXT: movdqa %xmm7, %xmm101895; SSE-NEXT: pandn %xmm11, %xmm101896; SSE-NEXT: pand %xmm7, %xmm01897; SSE-NEXT: por %xmm0, %xmm101898; SSE-NEXT: movdqa %xmm14, %xmm01899; SSE-NEXT: pand %xmm4, %xmm01900; SSE-NEXT: por (%rsp), %xmm0 # 16-byte Folded Reload1901; SSE-NEXT: movdqa %xmm0, %xmm111902; SSE-NEXT: punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm8[8],xmm11[9],xmm8[9],xmm11[10],xmm8[10],xmm11[11],xmm8[11],xmm11[12],xmm8[12],xmm11[13],xmm8[13],xmm11[14],xmm8[14],xmm11[15],xmm8[15]1903; SSE-NEXT: movdqa %xmm12, %xmm131904; SSE-NEXT: pandn %xmm11, %xmm131905; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1],xmm0[2],xmm8[2],xmm0[3],xmm8[3],xmm0[4],xmm8[4],xmm0[5],xmm8[5],xmm0[6],xmm8[6],xmm0[7],xmm8[7]1906; SSE-NEXT: pand %xmm12, %xmm01907; SSE-NEXT: por %xmm13, %xmm01908; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]1909; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]1910; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]1911; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,2,3,0,4,5,6,7]1912; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,6,7,4]1913; SSE-NEXT: pshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,7,6,7]1914; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm11[0,2,2,3]1915; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm11[1,2,3,3,4,5,6,7]1916; SSE-NEXT: packuswb %xmm11, %xmm01917; SSE-NEXT: movdqa %xmm12, %xmm131918; SSE-NEXT: pandn %xmm3, %xmm131919; SSE-NEXT: movdqa %xmm6, %xmm111920; SSE-NEXT: pandn %xmm1, %xmm111921; SSE-NEXT: pand %xmm12, %xmm11922; SSE-NEXT: por %xmm13, %xmm11923; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7]1924; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7]1925; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]1926; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,2,2,2,4,5,6,7]1927; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,7,4]1928; SSE-NEXT: packuswb %xmm1, %xmm11929; SSE-NEXT: movdqa %xmm7, %xmm131930; SSE-NEXT: pandn %xmm1, %xmm131931; SSE-NEXT: pand %xmm7, %xmm01932; SSE-NEXT: por %xmm0, %xmm131933; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1934; SSE-NEXT: pand %xmm4, %xmm01935; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload1936; SSE-NEXT: por %xmm0, %xmm41937; SSE-NEXT: movdqa %xmm4, %xmm01938; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15]1939; SSE-NEXT: movdqa %xmm12, %xmm11940; SSE-NEXT: pandn %xmm0, %xmm11941; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm8[0],xmm4[1],xmm8[1],xmm4[2],xmm8[2],xmm4[3],xmm8[3],xmm4[4],xmm8[4],xmm4[5],xmm8[5],xmm4[6],xmm8[6],xmm4[7],xmm8[7]1942; SSE-NEXT: pand %xmm12, %xmm41943; SSE-NEXT: por %xmm1, %xmm41944; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm4[2,1,0,3,4,5,6,7]1945; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,4,7]1946; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,1]1947; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,0,4,5,6,7]1948; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,7,4]1949; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7]1950; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]1951; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,2,3,3,4,5,6,7]1952; SSE-NEXT: packuswb %xmm0, %xmm11953; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1954; SSE-NEXT: pand %xmm12, %xmm01955; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload1956; SSE-NEXT: por %xmm0, %xmm121957; SSE-NEXT: pand %xmm7, %xmm11958; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm12[0,3,2,3,4,5,6,7]1959; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7]1960; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0]1961; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,2,2,2,4,5,6,7]1962; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,6,7,4]1963; SSE-NEXT: packuswb %xmm0, %xmm01964; SSE-NEXT: pandn %xmm0, %xmm71965; SSE-NEXT: por %xmm1, %xmm71966; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1967; SSE-NEXT: movdqa {{.*#+}} xmm2 = [255,255,0,255,255,0,255,255,0,255,255,0,255,255,0,255]1968; SSE-NEXT: pand %xmm2, %xmm01969; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload1970; SSE-NEXT: movdqa %xmm0, %xmm41971; SSE-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm8[8],xmm4[9],xmm8[9],xmm4[10],xmm8[10],xmm4[11],xmm8[11],xmm4[12],xmm8[12],xmm4[13],xmm8[13],xmm4[14],xmm8[14],xmm4[15],xmm8[15]1972; SSE-NEXT: movdqa {{.*#+}} xmm12 = [0,65535,65535,0,65535,65535,0,65535]1973; SSE-NEXT: movdqa %xmm12, %xmm11974; SSE-NEXT: pandn %xmm4, %xmm11975; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1],xmm0[2],xmm8[2],xmm0[3],xmm8[3],xmm0[4],xmm8[4],xmm0[5],xmm8[5],xmm0[6],xmm8[6],xmm0[7],xmm8[7]1976; SSE-NEXT: pand %xmm12, %xmm01977; SSE-NEXT: por %xmm1, %xmm01978; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,1,2,0]1979; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,5]1980; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,1,2,0]1981; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,0,3,4,5,6,7]1982; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm1[0,1,2,3,6,5,4,7]1983; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm4[2,1,2,3]1984; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]1985; SSE-NEXT: packuswb %xmm1, %xmm01986; SSE-NEXT: movdqa %xmm5, %xmm11987; SSE-NEXT: pand %xmm6, %xmm11988; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload1989; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]1990; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,2,1,4,5,6,7]1991; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]1992; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]1993; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,4,7]1994; SSE-NEXT: packuswb %xmm1, %xmm11995; SSE-NEXT: movdqa {{.*#+}} xmm4 = [65535,65535,65535,65535,65535,0,0,0]1996; SSE-NEXT: movdqa %xmm4, %xmm51997; SSE-NEXT: pandn %xmm1, %xmm51998; SSE-NEXT: pand %xmm4, %xmm01999; SSE-NEXT: por %xmm0, %xmm52000; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2001; SSE-NEXT: pand %xmm2, %xmm02002; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload2003; SSE-NEXT: movdqa %xmm0, %xmm22004; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm8[8],xmm2[9],xmm8[9],xmm2[10],xmm8[10],xmm2[11],xmm8[11],xmm2[12],xmm8[12],xmm2[13],xmm8[13],xmm2[14],xmm8[14],xmm2[15],xmm8[15]2005; SSE-NEXT: movdqa %xmm12, %xmm12006; SSE-NEXT: pandn %xmm2, %xmm12007; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1],xmm0[2],xmm8[2],xmm0[3],xmm8[3],xmm0[4],xmm8[4],xmm0[5],xmm8[5],xmm0[6],xmm8[6],xmm0[7],xmm8[7]2008; SSE-NEXT: pand %xmm12, %xmm02009; SSE-NEXT: por %xmm1, %xmm02010; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,1,2,0]2011; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,5]2012; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,1,2,0]2013; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,0,3,4,5,6,7]2014; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,4,7]2015; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,1,2,3]2016; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]2017; SSE-NEXT: packuswb %xmm0, %xmm12018; SSE-NEXT: pand %xmm6, %xmm152019; SSE-NEXT: por %xmm9, %xmm152020; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm15[0,2,1,3]2021; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]2022; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]2023; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,3,4,5,6,7]2024; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]2025; SSE-NEXT: packuswb %xmm0, %xmm02026; SSE-NEXT: movdqa %xmm4, %xmm92027; SSE-NEXT: pandn %xmm0, %xmm92028; SSE-NEXT: pand %xmm4, %xmm12029; SSE-NEXT: por %xmm1, %xmm92030; SSE-NEXT: movdqa {{.*#+}} xmm2 = [255,255,0,255,255,0,255,255,0,255,255,0,255,255,0,255]2031; SSE-NEXT: pand %xmm2, %xmm142032; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload2033; SSE-NEXT: movdqa %xmm14, %xmm02034; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15]2035; SSE-NEXT: movdqa %xmm12, %xmm12036; SSE-NEXT: pandn %xmm0, %xmm12037; SSE-NEXT: punpcklbw {{.*#+}} xmm14 = xmm14[0],xmm8[0],xmm14[1],xmm8[1],xmm14[2],xmm8[2],xmm14[3],xmm8[3],xmm14[4],xmm8[4],xmm14[5],xmm8[5],xmm14[6],xmm8[6],xmm14[7],xmm8[7]2038; SSE-NEXT: pand %xmm12, %xmm142039; SSE-NEXT: por %xmm1, %xmm142040; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm14[3,1,2,0]2041; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,5]2042; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,1,2,0]2043; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,0,3,4,5,6,7]2044; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,4,7]2045; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]2046; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7]2047; SSE-NEXT: packuswb %xmm0, %xmm12048; SSE-NEXT: pand %xmm6, %xmm32049; SSE-NEXT: por %xmm11, %xmm32050; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,2,1,3]2051; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]2052; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]2053; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,3,4,5,6,7]2054; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7]2055; SSE-NEXT: packuswb %xmm0, %xmm32056; SSE-NEXT: movdqa %xmm4, %xmm02057; SSE-NEXT: pandn %xmm3, %xmm02058; SSE-NEXT: pand %xmm4, %xmm12059; SSE-NEXT: por %xmm1, %xmm02060; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload2061; SSE-NEXT: pand %xmm2, %xmm12062; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload2063; SSE-NEXT: por %xmm1, %xmm22064; SSE-NEXT: movdqa %xmm2, %xmm12065; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm8[8],xmm1[9],xmm8[9],xmm1[10],xmm8[10],xmm1[11],xmm8[11],xmm1[12],xmm8[12],xmm1[13],xmm8[13],xmm1[14],xmm8[14],xmm1[15],xmm8[15]2066; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm8[0],xmm2[1],xmm8[1],xmm2[2],xmm8[2],xmm2[3],xmm8[3],xmm2[4],xmm8[4],xmm2[5],xmm8[5],xmm2[6],xmm8[6],xmm2[7],xmm8[7]2067; SSE-NEXT: pand %xmm12, %xmm22068; SSE-NEXT: pandn %xmm1, %xmm122069; SSE-NEXT: por %xmm2, %xmm122070; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm12[3,1,2,0]2071; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,5]2072; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[3,1,2,0]2073; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[2,1,0,3,4,5,6,7]2074; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,4,7]2075; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]2076; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7]2077; SSE-NEXT: packuswb %xmm1, %xmm32078; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload2079; SSE-NEXT: pand %xmm6, %xmm12080; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload2081; SSE-NEXT: por %xmm1, %xmm62082; SSE-NEXT: pand %xmm4, %xmm32083; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm6[0,2,1,3]2084; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,2,1,4,5,6,7]2085; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]2086; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]2087; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,4,7]2088; SSE-NEXT: packuswb %xmm1, %xmm12089; SSE-NEXT: pandn %xmm1, %xmm42090; SSE-NEXT: por %xmm3, %xmm42091; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload2092; SSE-NEXT: movaps %xmm1, 32(%rsi)2093; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload2094; SSE-NEXT: movaps %xmm1, 48(%rsi)2095; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload2096; SSE-NEXT: movaps %xmm1, (%rsi)2097; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload2098; SSE-NEXT: movaps %xmm1, 16(%rsi)2099; SSE-NEXT: movdqa %xmm7, 32(%rdx)2100; SSE-NEXT: movdqa %xmm13, 48(%rdx)2101; SSE-NEXT: movdqa %xmm10, (%rdx)2102; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload2103; SSE-NEXT: movaps %xmm1, 16(%rdx)2104; SSE-NEXT: movdqa %xmm4, 32(%rcx)2105; SSE-NEXT: movdqa %xmm0, 48(%rcx)2106; SSE-NEXT: movdqa %xmm9, (%rcx)2107; SSE-NEXT: movdqa %xmm5, 16(%rcx)2108; SSE-NEXT: addq $168, %rsp2109; SSE-NEXT: retq2110;2111; AVX-LABEL: load_i8_stride3_vf64:2112; AVX: # %bb.0:2113; AVX-NEXT: vmovdqa (%rdi), %xmm62114; AVX-NEXT: vmovdqa 16(%rdi), %xmm02115; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2116; AVX-NEXT: vmovdqa 32(%rdi), %xmm42117; AVX-NEXT: vmovdqa 48(%rdi), %xmm72118; AVX-NEXT: vmovdqa 64(%rdi), %xmm22119; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2120; AVX-NEXT: vmovdqa 96(%rdi), %xmm112121; AVX-NEXT: vmovdqa 112(%rdi), %xmm32122; AVX-NEXT: vmovdqa 144(%rdi), %xmm102123; AVX-NEXT: vmovdqa 160(%rdi), %xmm12124; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2125; AVX-NEXT: vmovdqa {{.*#+}} xmm12 = [128,128,128,128,128,0,3,6,9,12,15,2,5,8,11,14]2126; AVX-NEXT: vpshufb %xmm12, %xmm6, %xmm52127; AVX-NEXT: vpshufb %xmm12, %xmm10, %xmm82128; AVX-NEXT: vpshufb %xmm12, %xmm11, %xmm92129; AVX-NEXT: vpshufb %xmm12, %xmm7, %xmm132130; AVX-NEXT: vmovdqa {{.*#+}} xmm14 = [1,4,7,10,13,128,128,128,128,128,128,u,u,u,u,u]2131; AVX-NEXT: vpshufb %xmm14, %xmm6, %xmm62132; AVX-NEXT: vmovdqa {{.*#+}} xmm15 = [128,128,128,128,128,0,3,6,9,12,15,u,u,u,u,u]2133; AVX-NEXT: vpshufb %xmm15, %xmm0, %xmm122134; AVX-NEXT: vpor %xmm6, %xmm12, %xmm62135; AVX-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2136; AVX-NEXT: vpshufb %xmm14, %xmm10, %xmm102137; AVX-NEXT: vpshufb %xmm15, %xmm1, %xmm122138; AVX-NEXT: vpor %xmm10, %xmm12, %xmm62139; AVX-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2140; AVX-NEXT: vpshufb %xmm14, %xmm11, %xmm112141; AVX-NEXT: vpshufb %xmm15, %xmm3, %xmm122142; AVX-NEXT: vpor %xmm11, %xmm12, %xmm62143; AVX-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2144; AVX-NEXT: vpshufb %xmm14, %xmm7, %xmm72145; AVX-NEXT: vpshufb %xmm15, %xmm2, %xmm122146; AVX-NEXT: vpor %xmm7, %xmm12, %xmm62147; AVX-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2148; AVX-NEXT: vpshufb %xmm14, %xmm0, %xmm122149; AVX-NEXT: vpshufb %xmm15, %xmm4, %xmm62150; AVX-NEXT: vpor %xmm6, %xmm12, %xmm62151; AVX-NEXT: vmovdqa 176(%rdi), %xmm122152; AVX-NEXT: vpshufb %xmm14, %xmm1, %xmm02153; AVX-NEXT: vpshufb %xmm15, %xmm12, %xmm102154; AVX-NEXT: vpor %xmm0, %xmm10, %xmm12155; AVX-NEXT: vmovdqa 128(%rdi), %xmm102156; AVX-NEXT: vpshufb %xmm14, %xmm3, %xmm02157; AVX-NEXT: vpshufb %xmm15, %xmm10, %xmm112158; AVX-NEXT: vpor %xmm0, %xmm11, %xmm02159; AVX-NEXT: vpshufb %xmm14, %xmm2, %xmm112160; AVX-NEXT: vmovdqa 80(%rdi), %xmm142161; AVX-NEXT: vpshufb %xmm15, %xmm14, %xmm152162; AVX-NEXT: vpor %xmm11, %xmm15, %xmm112163; AVX-NEXT: vmovdqa {{.*#+}} xmm15 = [1,4,7,10,13,128,128,128,128,128,128,128,128,128,128,128]2164; AVX-NEXT: vpshufb %xmm15, %xmm14, %xmm72165; AVX-NEXT: vpor %xmm7, %xmm13, %xmm72166; AVX-NEXT: vpalignr {{.*#+}} xmm13 = xmm13[11,12,13,14,15],xmm11[0,1,2,3,4,5,6,7,8,9,10]2167; AVX-NEXT: vpshufb %xmm15, %xmm10, %xmm112168; AVX-NEXT: vpor %xmm11, %xmm9, %xmm112169; AVX-NEXT: vpalignr {{.*#+}} xmm9 = xmm9[11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10]2170; AVX-NEXT: vpshufb %xmm15, %xmm12, %xmm02171; AVX-NEXT: vpor %xmm0, %xmm8, %xmm22172; AVX-NEXT: vpalignr {{.*#+}} xmm8 = xmm8[11,12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10]2173; AVX-NEXT: vpshufb %xmm15, %xmm4, %xmm12174; AVX-NEXT: vpor %xmm1, %xmm5, %xmm12175; AVX-NEXT: vpalignr {{.*#+}} xmm5 = xmm5[11,12,13,14,15],xmm6[0,1,2,3,4,5,6,7,8,9,10]2176; AVX-NEXT: vmovdqa {{.*#+}} xmm6 = [0,1,2,3,4,5,6,7,8,9,10,128,128,128,128,128]2177; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2178; AVX-NEXT: vpshufb %xmm6, %xmm0, %xmm152179; AVX-NEXT: vmovdqa {{.*#+}} xmm0 = [128,128,128,128,128,128,128,128,128,128,128,2,5,8,11,14]2180; AVX-NEXT: vpshufb %xmm0, %xmm14, %xmm142181; AVX-NEXT: vpor %xmm14, %xmm15, %xmm142182; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload2183; AVX-NEXT: vpshufb %xmm6, %xmm15, %xmm152184; AVX-NEXT: vpshufb %xmm0, %xmm10, %xmm102185; AVX-NEXT: vpor %xmm10, %xmm15, %xmm102186; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload2187; AVX-NEXT: vpshufb %xmm6, %xmm15, %xmm152188; AVX-NEXT: vpshufb %xmm0, %xmm12, %xmm122189; AVX-NEXT: vpor %xmm12, %xmm15, %xmm122190; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload2191; AVX-NEXT: vpshufb %xmm6, %xmm15, %xmm62192; AVX-NEXT: vpshufb %xmm0, %xmm4, %xmm02193; AVX-NEXT: vpor %xmm0, %xmm6, %xmm02194; AVX-NEXT: vmovdqa {{.*#+}} xmm4 = [5,6,7,8,9,10,128,128,128,128,128,0,1,2,3,4]2195; AVX-NEXT: vpshufb %xmm4, %xmm7, %xmm62196; AVX-NEXT: vmovdqa {{.*#+}} xmm7 = [128,128,128,128,128,128,2,5,8,11,14,128,128,128,128,128]2197; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload2198; AVX-NEXT: vpshufb %xmm7, %xmm15, %xmm152199; AVX-NEXT: vpor %xmm6, %xmm15, %xmm62200; AVX-NEXT: vpshufb %xmm4, %xmm11, %xmm112201; AVX-NEXT: vpshufb %xmm7, %xmm3, %xmm152202; AVX-NEXT: vpor %xmm15, %xmm11, %xmm112203; AVX-NEXT: vpshufb %xmm4, %xmm2, %xmm22204; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload2205; AVX-NEXT: vpshufb %xmm7, %xmm3, %xmm32206; AVX-NEXT: vpor %xmm3, %xmm2, %xmm22207; AVX-NEXT: vpshufb %xmm4, %xmm1, %xmm12208; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload2209; AVX-NEXT: vpshufb %xmm7, %xmm3, %xmm32210; AVX-NEXT: vpor %xmm3, %xmm1, %xmm12211; AVX-NEXT: vmovdqa %xmm1, (%rsi)2212; AVX-NEXT: vmovdqa %xmm2, 48(%rsi)2213; AVX-NEXT: vmovdqa %xmm11, 32(%rsi)2214; AVX-NEXT: vmovdqa %xmm6, 16(%rsi)2215; AVX-NEXT: vmovdqa %xmm0, (%rdx)2216; AVX-NEXT: vmovdqa %xmm12, 48(%rdx)2217; AVX-NEXT: vmovdqa %xmm10, 32(%rdx)2218; AVX-NEXT: vmovdqa %xmm14, 16(%rdx)2219; AVX-NEXT: vmovdqa %xmm5, (%rcx)2220; AVX-NEXT: vmovdqa %xmm8, 48(%rcx)2221; AVX-NEXT: vmovdqa %xmm9, 32(%rcx)2222; AVX-NEXT: vmovdqa %xmm13, 16(%rcx)2223; AVX-NEXT: retq2224;2225; AVX2-LABEL: load_i8_stride3_vf64:2226; AVX2: # %bb.0:2227; AVX2-NEXT: vmovdqa (%rdi), %xmm02228; AVX2-NEXT: vmovdqa 16(%rdi), %xmm12229; AVX2-NEXT: vmovdqa 32(%rdi), %xmm22230; AVX2-NEXT: vmovdqa 96(%rdi), %xmm32231; AVX2-NEXT: vmovdqa 112(%rdi), %xmm42232; AVX2-NEXT: vmovdqa 128(%rdi), %xmm52233; AVX2-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm62234; AVX2-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm02235; AVX2-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm12236; AVX2-NEXT: vinserti128 $1, 144(%rdi), %ymm3, %ymm22237; AVX2-NEXT: vinserti128 $1, 160(%rdi), %ymm4, %ymm32238; AVX2-NEXT: vinserti128 $1, 176(%rdi), %ymm5, %ymm42239; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [255,0,255,255,0,255,255,0,255,255,0,255,255,0,255,255,255,0,255,255,0,255,255,0,255,255,0,255,255,0,255,255]2240; AVX2-NEXT: # ymm5 = mem[0,1,0,1]2241; AVX2-NEXT: vpblendvb %ymm5, %ymm6, %ymm1, %ymm72242; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm8 = [1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14]2243; AVX2-NEXT: # ymm8 = mem[0,1,0,1]2244; AVX2-NEXT: vpshufb %ymm8, %ymm7, %ymm72245; AVX2-NEXT: vpblendvb %ymm5, %ymm2, %ymm4, %ymm52246; AVX2-NEXT: vpshufb %ymm8, %ymm5, %ymm52247; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm8 = [255,0,0,255,0,0,255,0,0,255,0,0,255,0,0,255,255,0,0,255,0,0,255,0,0,255,0,0,255,0,0,255]2248; AVX2-NEXT: # ymm8 = mem[0,1,0,1]2249; AVX2-NEXT: vpblendvb %ymm8, %ymm0, %ymm6, %ymm62250; AVX2-NEXT: vpblendvb %ymm8, %ymm3, %ymm2, %ymm22251; AVX2-NEXT: vpblendvb %ymm8, %ymm1, %ymm0, %ymm92252; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm10 = [1,4,7,10,13,0,3,6,9,12,15,0,0,0,0,0,1,4,7,10,13,0,3,6,9,12,15,0,0,0,0,0]2253; AVX2-NEXT: # ymm10 = mem[0,1,0,1]2254; AVX2-NEXT: vpshufb %ymm10, %ymm9, %ymm92255; AVX2-NEXT: vpblendvb %ymm8, %ymm4, %ymm3, %ymm82256; AVX2-NEXT: vpshufb %ymm10, %ymm8, %ymm82257; AVX2-NEXT: vpalignr {{.*#+}} ymm8 = ymm5[11,12,13,14,15],ymm8[0,1,2,3,4,5,6,7,8,9,10],ymm5[27,28,29,30,31],ymm8[16,17,18,19,20,21,22,23,24,25,26]2258; AVX2-NEXT: vpalignr {{.*#+}} ymm9 = ymm7[11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7,8,9,10],ymm7[27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23,24,25,26]2259; AVX2-NEXT: vmovdqa {{.*#+}} ymm10 = [1,4,7,10,13,0,3,6,9,12,15,128,128,128,128,128,17,20,23,26,29,16,19,22,25,28,31,128,128,128,128,128]2260; AVX2-NEXT: vpshufb %ymm10, %ymm2, %ymm22261; AVX2-NEXT: vmovdqa {{.*#+}} ymm11 = [128,128,128,128,128,128,128,128,128,128,128,2,5,8,11,14,128,128,128,128,128,128,128,128,128,128,128,18,21,24,27,30]2262; AVX2-NEXT: vpshufb %ymm11, %ymm4, %ymm42263; AVX2-NEXT: vpor %ymm4, %ymm2, %ymm22264; AVX2-NEXT: vpshufb %ymm10, %ymm6, %ymm42265; AVX2-NEXT: vpshufb %ymm11, %ymm1, %ymm12266; AVX2-NEXT: vpor %ymm1, %ymm4, %ymm12267; AVX2-NEXT: vpalignr {{.*#+}} ymm4 = ymm5[5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,21,22,23,24,25,26,27,28,29,30,31,16,17,18,19,20]2268; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm5 = [8,11,14,0,0,0,2,5,8,11,14,0,0,0,2,5,8,11,14,0,0,0,2,5,8,11,14,0,0,0,2,5]2269; AVX2-NEXT: vpshufb %ymm5, %ymm3, %ymm32270; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255]2271; AVX2-NEXT: # ymm6 = mem[0,1,0,1]2272; AVX2-NEXT: vpblendvb %ymm6, %ymm4, %ymm3, %ymm32273; AVX2-NEXT: vpalignr {{.*#+}} ymm4 = ymm7[5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,21,22,23,24,25,26,27,28,29,30,31,16,17,18,19,20]2274; AVX2-NEXT: vpshufb %ymm5, %ymm0, %ymm02275; AVX2-NEXT: vpblendvb %ymm6, %ymm4, %ymm0, %ymm02276; AVX2-NEXT: vmovdqa %ymm0, (%rsi)2277; AVX2-NEXT: vmovdqa %ymm3, 32(%rsi)2278; AVX2-NEXT: vmovdqa %ymm1, (%rdx)2279; AVX2-NEXT: vmovdqa %ymm2, 32(%rdx)2280; AVX2-NEXT: vmovdqa %ymm9, (%rcx)2281; AVX2-NEXT: vmovdqa %ymm8, 32(%rcx)2282; AVX2-NEXT: vzeroupper2283; AVX2-NEXT: retq2284;2285; AVX2-FP-LABEL: load_i8_stride3_vf64:2286; AVX2-FP: # %bb.0:2287; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm02288; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm12289; AVX2-FP-NEXT: vmovdqa 32(%rdi), %xmm22290; AVX2-FP-NEXT: vmovdqa 96(%rdi), %xmm32291; AVX2-FP-NEXT: vmovdqa 112(%rdi), %xmm42292; AVX2-FP-NEXT: vmovdqa 128(%rdi), %xmm52293; AVX2-FP-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm62294; AVX2-FP-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm02295; AVX2-FP-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm12296; AVX2-FP-NEXT: vinserti128 $1, 144(%rdi), %ymm3, %ymm22297; AVX2-FP-NEXT: vinserti128 $1, 160(%rdi), %ymm4, %ymm32298; AVX2-FP-NEXT: vinserti128 $1, 176(%rdi), %ymm5, %ymm42299; AVX2-FP-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [255,0,255,255,0,255,255,0,255,255,0,255,255,0,255,255,255,0,255,255,0,255,255,0,255,255,0,255,255,0,255,255]2300; AVX2-FP-NEXT: # ymm5 = mem[0,1,0,1]2301; AVX2-FP-NEXT: vpblendvb %ymm5, %ymm6, %ymm1, %ymm72302; AVX2-FP-NEXT: vbroadcasti128 {{.*#+}} ymm8 = [1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14]2303; AVX2-FP-NEXT: # ymm8 = mem[0,1,0,1]2304; AVX2-FP-NEXT: vpshufb %ymm8, %ymm7, %ymm72305; AVX2-FP-NEXT: vpblendvb %ymm5, %ymm2, %ymm4, %ymm52306; AVX2-FP-NEXT: vpshufb %ymm8, %ymm5, %ymm52307; AVX2-FP-NEXT: vbroadcasti128 {{.*#+}} ymm8 = [255,0,0,255,0,0,255,0,0,255,0,0,255,0,0,255,255,0,0,255,0,0,255,0,0,255,0,0,255,0,0,255]2308; AVX2-FP-NEXT: # ymm8 = mem[0,1,0,1]2309; AVX2-FP-NEXT: vpblendvb %ymm8, %ymm0, %ymm6, %ymm62310; AVX2-FP-NEXT: vpblendvb %ymm8, %ymm3, %ymm2, %ymm22311; AVX2-FP-NEXT: vpblendvb %ymm8, %ymm1, %ymm0, %ymm92312; AVX2-FP-NEXT: vbroadcasti128 {{.*#+}} ymm10 = [1,4,7,10,13,0,3,6,9,12,15,0,0,0,0,0,1,4,7,10,13,0,3,6,9,12,15,0,0,0,0,0]2313; AVX2-FP-NEXT: # ymm10 = mem[0,1,0,1]2314; AVX2-FP-NEXT: vpshufb %ymm10, %ymm9, %ymm92315; AVX2-FP-NEXT: vpblendvb %ymm8, %ymm4, %ymm3, %ymm82316; AVX2-FP-NEXT: vpshufb %ymm10, %ymm8, %ymm82317; AVX2-FP-NEXT: vpalignr {{.*#+}} ymm8 = ymm5[11,12,13,14,15],ymm8[0,1,2,3,4,5,6,7,8,9,10],ymm5[27,28,29,30,31],ymm8[16,17,18,19,20,21,22,23,24,25,26]2318; AVX2-FP-NEXT: vpalignr {{.*#+}} ymm9 = ymm7[11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7,8,9,10],ymm7[27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23,24,25,26]2319; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm10 = [1,4,7,10,13,0,3,6,9,12,15,128,128,128,128,128,17,20,23,26,29,16,19,22,25,28,31,128,128,128,128,128]2320; AVX2-FP-NEXT: vpshufb %ymm10, %ymm2, %ymm22321; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm11 = [128,128,128,128,128,128,128,128,128,128,128,2,5,8,11,14,128,128,128,128,128,128,128,128,128,128,128,18,21,24,27,30]2322; AVX2-FP-NEXT: vpshufb %ymm11, %ymm4, %ymm42323; AVX2-FP-NEXT: vpor %ymm4, %ymm2, %ymm22324; AVX2-FP-NEXT: vpshufb %ymm10, %ymm6, %ymm42325; AVX2-FP-NEXT: vpshufb %ymm11, %ymm1, %ymm12326; AVX2-FP-NEXT: vpor %ymm1, %ymm4, %ymm12327; AVX2-FP-NEXT: vpalignr {{.*#+}} ymm4 = ymm5[5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,21,22,23,24,25,26,27,28,29,30,31,16,17,18,19,20]2328; AVX2-FP-NEXT: vpbroadcastq {{.*#+}} ymm5 = [8,11,14,0,0,0,2,5,8,11,14,0,0,0,2,5,8,11,14,0,0,0,2,5,8,11,14,0,0,0,2,5]2329; AVX2-FP-NEXT: vpshufb %ymm5, %ymm3, %ymm32330; AVX2-FP-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255]2331; AVX2-FP-NEXT: # ymm6 = mem[0,1,0,1]2332; AVX2-FP-NEXT: vpblendvb %ymm6, %ymm4, %ymm3, %ymm32333; AVX2-FP-NEXT: vpalignr {{.*#+}} ymm4 = ymm7[5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,21,22,23,24,25,26,27,28,29,30,31,16,17,18,19,20]2334; AVX2-FP-NEXT: vpshufb %ymm5, %ymm0, %ymm02335; AVX2-FP-NEXT: vpblendvb %ymm6, %ymm4, %ymm0, %ymm02336; AVX2-FP-NEXT: vmovdqa %ymm0, (%rsi)2337; AVX2-FP-NEXT: vmovdqa %ymm3, 32(%rsi)2338; AVX2-FP-NEXT: vmovdqa %ymm1, (%rdx)2339; AVX2-FP-NEXT: vmovdqa %ymm2, 32(%rdx)2340; AVX2-FP-NEXT: vmovdqa %ymm9, (%rcx)2341; AVX2-FP-NEXT: vmovdqa %ymm8, 32(%rcx)2342; AVX2-FP-NEXT: vzeroupper2343; AVX2-FP-NEXT: retq2344;2345; AVX2-FCP-LABEL: load_i8_stride3_vf64:2346; AVX2-FCP: # %bb.0:2347; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm02348; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm12349; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %xmm22350; AVX2-FCP-NEXT: vmovdqa 96(%rdi), %xmm32351; AVX2-FCP-NEXT: vmovdqa 112(%rdi), %xmm42352; AVX2-FCP-NEXT: vmovdqa 128(%rdi), %xmm52353; AVX2-FCP-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm62354; AVX2-FCP-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm02355; AVX2-FCP-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm12356; AVX2-FCP-NEXT: vinserti128 $1, 144(%rdi), %ymm3, %ymm22357; AVX2-FCP-NEXT: vinserti128 $1, 160(%rdi), %ymm4, %ymm32358; AVX2-FCP-NEXT: vinserti128 $1, 176(%rdi), %ymm5, %ymm42359; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [255,0,255,255,0,255,255,0,255,255,0,255,255,0,255,255,255,0,255,255,0,255,255,0,255,255,0,255,255,0,255,255]2360; AVX2-FCP-NEXT: # ymm5 = mem[0,1,0,1]2361; AVX2-FCP-NEXT: vpblendvb %ymm5, %ymm6, %ymm1, %ymm72362; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm8 = [1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14]2363; AVX2-FCP-NEXT: # ymm8 = mem[0,1,0,1]2364; AVX2-FCP-NEXT: vpshufb %ymm8, %ymm7, %ymm72365; AVX2-FCP-NEXT: vpblendvb %ymm5, %ymm2, %ymm4, %ymm52366; AVX2-FCP-NEXT: vpshufb %ymm8, %ymm5, %ymm52367; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm8 = [255,0,0,255,0,0,255,0,0,255,0,0,255,0,0,255,255,0,0,255,0,0,255,0,0,255,0,0,255,0,0,255]2368; AVX2-FCP-NEXT: # ymm8 = mem[0,1,0,1]2369; AVX2-FCP-NEXT: vpblendvb %ymm8, %ymm0, %ymm6, %ymm62370; AVX2-FCP-NEXT: vpblendvb %ymm8, %ymm3, %ymm2, %ymm22371; AVX2-FCP-NEXT: vpblendvb %ymm8, %ymm1, %ymm0, %ymm92372; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm10 = [1,4,7,10,13,0,3,6,9,12,15,0,0,0,0,0,1,4,7,10,13,0,3,6,9,12,15,0,0,0,0,0]2373; AVX2-FCP-NEXT: # ymm10 = mem[0,1,0,1]2374; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm9, %ymm92375; AVX2-FCP-NEXT: vpblendvb %ymm8, %ymm4, %ymm3, %ymm82376; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm8, %ymm82377; AVX2-FCP-NEXT: vpalignr {{.*#+}} ymm8 = ymm5[11,12,13,14,15],ymm8[0,1,2,3,4,5,6,7,8,9,10],ymm5[27,28,29,30,31],ymm8[16,17,18,19,20,21,22,23,24,25,26]2378; AVX2-FCP-NEXT: vpalignr {{.*#+}} ymm9 = ymm7[11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7,8,9,10],ymm7[27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23,24,25,26]2379; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm10 = [1,4,7,10,13,0,3,6,9,12,15,128,128,128,128,128,17,20,23,26,29,16,19,22,25,28,31,128,128,128,128,128]2380; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm2, %ymm22381; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm11 = [128,128,128,128,128,128,128,128,128,128,128,2,5,8,11,14,128,128,128,128,128,128,128,128,128,128,128,18,21,24,27,30]2382; AVX2-FCP-NEXT: vpshufb %ymm11, %ymm4, %ymm42383; AVX2-FCP-NEXT: vpor %ymm4, %ymm2, %ymm22384; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm6, %ymm42385; AVX2-FCP-NEXT: vpshufb %ymm11, %ymm1, %ymm12386; AVX2-FCP-NEXT: vpor %ymm1, %ymm4, %ymm12387; AVX2-FCP-NEXT: vpalignr {{.*#+}} ymm4 = ymm5[5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,21,22,23,24,25,26,27,28,29,30,31,16,17,18,19,20]2388; AVX2-FCP-NEXT: vpbroadcastq {{.*#+}} ymm5 = [8,11,14,0,0,0,2,5,8,11,14,0,0,0,2,5,8,11,14,0,0,0,2,5,8,11,14,0,0,0,2,5]2389; AVX2-FCP-NEXT: vpshufb %ymm5, %ymm3, %ymm32390; AVX2-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255]2391; AVX2-FCP-NEXT: # ymm6 = mem[0,1,0,1]2392; AVX2-FCP-NEXT: vpblendvb %ymm6, %ymm4, %ymm3, %ymm32393; AVX2-FCP-NEXT: vpalignr {{.*#+}} ymm4 = ymm7[5,6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,21,22,23,24,25,26,27,28,29,30,31,16,17,18,19,20]2394; AVX2-FCP-NEXT: vpshufb %ymm5, %ymm0, %ymm02395; AVX2-FCP-NEXT: vpblendvb %ymm6, %ymm4, %ymm0, %ymm02396; AVX2-FCP-NEXT: vmovdqa %ymm0, (%rsi)2397; AVX2-FCP-NEXT: vmovdqa %ymm3, 32(%rsi)2398; AVX2-FCP-NEXT: vmovdqa %ymm1, (%rdx)2399; AVX2-FCP-NEXT: vmovdqa %ymm2, 32(%rdx)2400; AVX2-FCP-NEXT: vmovdqa %ymm9, (%rcx)2401; AVX2-FCP-NEXT: vmovdqa %ymm8, 32(%rcx)2402; AVX2-FCP-NEXT: vzeroupper2403; AVX2-FCP-NEXT: retq2404;2405; AVX512-LABEL: load_i8_stride3_vf64:2406; AVX512: # %bb.0:2407; AVX512-NEXT: vmovdqa (%rdi), %xmm02408; AVX512-NEXT: vmovdqa 16(%rdi), %xmm12409; AVX512-NEXT: vmovdqa 32(%rdi), %xmm22410; AVX512-NEXT: vmovdqa 96(%rdi), %xmm32411; AVX512-NEXT: vmovdqa 112(%rdi), %xmm42412; AVX512-NEXT: vmovdqa 128(%rdi), %xmm52413; AVX512-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm02414; AVX512-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]2415; AVX512-NEXT: # ymm6 = mem[0,1,0,1]2416; AVX512-NEXT: vpshufb %ymm6, %ymm0, %ymm02417; AVX512-NEXT: vinserti128 $1, 144(%rdi), %ymm3, %ymm32418; AVX512-NEXT: vpshufb %ymm6, %ymm3, %ymm32419; AVX512-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm12420; AVX512-NEXT: vpshufb %ymm6, %ymm1, %ymm12421; AVX512-NEXT: vinserti128 $1, 160(%rdi), %ymm4, %ymm42422; AVX512-NEXT: vpshufb %ymm6, %ymm4, %ymm42423; AVX512-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm22424; AVX512-NEXT: vpshufb %ymm6, %ymm2, %ymm22425; AVX512-NEXT: vinserti128 $1, 176(%rdi), %ymm5, %ymm52426; AVX512-NEXT: vpshufb %ymm6, %ymm5, %ymm52427; AVX512-NEXT: vpalignr {{.*#+}} ymm6 = ymm5[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm5[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]2428; AVX512-NEXT: vpalignr {{.*#+}} ymm7 = ymm2[11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7,8,9,10],ymm2[27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23,24,25,26]2429; AVX512-NEXT: vpalignr {{.*#+}} ymm3 = ymm3[11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7,8,9,10],ymm3[27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23,24,25,26]2430; AVX512-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]2431; AVX512-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm82432; AVX512-NEXT: vpalignr {{.*#+}} ymm4 = ymm4[11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7,8,9,10],ymm4[27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23,24,25,26]2433; AVX512-NEXT: vpalignr {{.*#+}} ymm1 = ymm1[11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7,8,9,10],ymm1[27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23,24,25,26]2434; AVX512-NEXT: vinserti64x4 $1, %ymm4, %zmm1, %zmm22435; AVX512-NEXT: vpalignr {{.*#+}} ymm1 = ymm7[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm7[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]2436; AVX512-NEXT: vpalignr {{.*#+}} ymm4 = ymm6[11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7,8,9,10],ymm6[27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23,24,25,26]2437; AVX512-NEXT: vbroadcasti32x4 {{.*#+}} zmm5 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]2438; AVX512-NEXT: # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]2439; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm2 ^ (zmm5 & (zmm8 ^ zmm2))2440; AVX512-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm7[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm7[16,17,18,19,20,21,22,23,24,25,26]2441; AVX512-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]2442; AVX512-NEXT: vpalignr {{.*#+}} ymm2 = ymm3[11,12,13,14,15],ymm6[0,1,2,3,4,5,6,7,8,9,10],ymm3[27,28,29,30,31],ymm6[16,17,18,19,20,21,22,23,24,25,26]2443; AVX512-NEXT: vpalignr {{.*#+}} ymm2 = ymm2[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]2444; AVX512-NEXT: vmovdqa %ymm2, 32(%rsi)2445; AVX512-NEXT: vmovdqa %ymm0, (%rsi)2446; AVX512-NEXT: vmovdqa64 %zmm5, (%rdx)2447; AVX512-NEXT: vmovdqa %ymm4, 32(%rcx)2448; AVX512-NEXT: vmovdqa %ymm1, (%rcx)2449; AVX512-NEXT: vzeroupper2450; AVX512-NEXT: retq2451;2452; AVX512-FCP-LABEL: load_i8_stride3_vf64:2453; AVX512-FCP: # %bb.0:2454; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm02455; AVX512-FCP-NEXT: vmovdqa 16(%rdi), %xmm12456; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %xmm22457; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %xmm32458; AVX512-FCP-NEXT: vmovdqa 112(%rdi), %xmm42459; AVX512-FCP-NEXT: vmovdqa 128(%rdi), %xmm52460; AVX512-FCP-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm02461; AVX512-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]2462; AVX512-FCP-NEXT: # ymm6 = mem[0,1,0,1]2463; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm02464; AVX512-FCP-NEXT: vinserti128 $1, 144(%rdi), %ymm3, %ymm32465; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm32466; AVX512-FCP-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm12467; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm1, %ymm12468; AVX512-FCP-NEXT: vinserti128 $1, 160(%rdi), %ymm4, %ymm42469; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm4, %ymm42470; AVX512-FCP-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm22471; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm2, %ymm22472; AVX512-FCP-NEXT: vinserti128 $1, 176(%rdi), %ymm5, %ymm52473; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm5, %ymm52474; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm6 = ymm5[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm5[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]2475; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm7 = ymm2[11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7,8,9,10],ymm2[27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23,24,25,26]2476; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm3 = ymm3[11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7,8,9,10],ymm3[27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23,24,25,26]2477; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]2478; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm82479; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm4 = ymm4[11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7,8,9,10],ymm4[27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23,24,25,26]2480; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm1 = ymm1[11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7,8,9,10],ymm1[27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23,24,25,26]2481; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm4, %zmm1, %zmm22482; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm1 = ymm7[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm7[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]2483; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm4 = ymm6[11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7,8,9,10],ymm6[27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23,24,25,26]2484; AVX512-FCP-NEXT: vbroadcasti32x4 {{.*#+}} zmm5 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]2485; AVX512-FCP-NEXT: # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]2486; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm2 ^ (zmm5 & (zmm8 ^ zmm2))2487; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm7[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm7[16,17,18,19,20,21,22,23,24,25,26]2488; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]2489; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm2 = ymm3[11,12,13,14,15],ymm6[0,1,2,3,4,5,6,7,8,9,10],ymm3[27,28,29,30,31],ymm6[16,17,18,19,20,21,22,23,24,25,26]2490; AVX512-FCP-NEXT: vpalignr {{.*#+}} ymm2 = ymm2[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]2491; AVX512-FCP-NEXT: vmovdqa %ymm2, 32(%rsi)2492; AVX512-FCP-NEXT: vmovdqa %ymm0, (%rsi)2493; AVX512-FCP-NEXT: vmovdqa64 %zmm5, (%rdx)2494; AVX512-FCP-NEXT: vmovdqa %ymm4, 32(%rcx)2495; AVX512-FCP-NEXT: vmovdqa %ymm1, (%rcx)2496; AVX512-FCP-NEXT: vzeroupper2497; AVX512-FCP-NEXT: retq2498;2499; AVX512DQ-LABEL: load_i8_stride3_vf64:2500; AVX512DQ: # %bb.0:2501; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm02502; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm12503; AVX512DQ-NEXT: vmovdqa 32(%rdi), %xmm22504; AVX512DQ-NEXT: vmovdqa 96(%rdi), %xmm32505; AVX512DQ-NEXT: vmovdqa 112(%rdi), %xmm42506; AVX512DQ-NEXT: vmovdqa 128(%rdi), %xmm52507; AVX512DQ-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm02508; AVX512DQ-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]2509; AVX512DQ-NEXT: # ymm6 = mem[0,1,0,1]2510; AVX512DQ-NEXT: vpshufb %ymm6, %ymm0, %ymm02511; AVX512DQ-NEXT: vinserti128 $1, 144(%rdi), %ymm3, %ymm32512; AVX512DQ-NEXT: vpshufb %ymm6, %ymm3, %ymm32513; AVX512DQ-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm12514; AVX512DQ-NEXT: vpshufb %ymm6, %ymm1, %ymm12515; AVX512DQ-NEXT: vinserti128 $1, 160(%rdi), %ymm4, %ymm42516; AVX512DQ-NEXT: vpshufb %ymm6, %ymm4, %ymm42517; AVX512DQ-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm22518; AVX512DQ-NEXT: vpshufb %ymm6, %ymm2, %ymm22519; AVX512DQ-NEXT: vinserti128 $1, 176(%rdi), %ymm5, %ymm52520; AVX512DQ-NEXT: vpshufb %ymm6, %ymm5, %ymm52521; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm6 = ymm5[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm5[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]2522; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm7 = ymm2[11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7,8,9,10],ymm2[27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23,24,25,26]2523; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm3 = ymm3[11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7,8,9,10],ymm3[27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23,24,25,26]2524; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]2525; AVX512DQ-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm82526; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm4 = ymm4[11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7,8,9,10],ymm4[27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23,24,25,26]2527; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm1 = ymm1[11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7,8,9,10],ymm1[27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23,24,25,26]2528; AVX512DQ-NEXT: vinserti64x4 $1, %ymm4, %zmm1, %zmm22529; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm1 = ymm7[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm7[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]2530; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm4 = ymm6[11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7,8,9,10],ymm6[27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23,24,25,26]2531; AVX512DQ-NEXT: vbroadcasti32x4 {{.*#+}} zmm5 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]2532; AVX512DQ-NEXT: # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]2533; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm5 = zmm2 ^ (zmm5 & (zmm8 ^ zmm2))2534; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm7[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm7[16,17,18,19,20,21,22,23,24,25,26]2535; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]2536; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm2 = ymm3[11,12,13,14,15],ymm6[0,1,2,3,4,5,6,7,8,9,10],ymm3[27,28,29,30,31],ymm6[16,17,18,19,20,21,22,23,24,25,26]2537; AVX512DQ-NEXT: vpalignr {{.*#+}} ymm2 = ymm2[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]2538; AVX512DQ-NEXT: vmovdqa %ymm2, 32(%rsi)2539; AVX512DQ-NEXT: vmovdqa %ymm0, (%rsi)2540; AVX512DQ-NEXT: vmovdqa64 %zmm5, (%rdx)2541; AVX512DQ-NEXT: vmovdqa %ymm4, 32(%rcx)2542; AVX512DQ-NEXT: vmovdqa %ymm1, (%rcx)2543; AVX512DQ-NEXT: vzeroupper2544; AVX512DQ-NEXT: retq2545;2546; AVX512DQ-FCP-LABEL: load_i8_stride3_vf64:2547; AVX512DQ-FCP: # %bb.0:2548; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm02549; AVX512DQ-FCP-NEXT: vmovdqa 16(%rdi), %xmm12550; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %xmm22551; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %xmm32552; AVX512DQ-FCP-NEXT: vmovdqa 112(%rdi), %xmm42553; AVX512DQ-FCP-NEXT: vmovdqa 128(%rdi), %xmm52554; AVX512DQ-FCP-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm02555; AVX512DQ-FCP-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]2556; AVX512DQ-FCP-NEXT: # ymm6 = mem[0,1,0,1]2557; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm02558; AVX512DQ-FCP-NEXT: vinserti128 $1, 144(%rdi), %ymm3, %ymm32559; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm3, %ymm32560; AVX512DQ-FCP-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm12561; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm1, %ymm12562; AVX512DQ-FCP-NEXT: vinserti128 $1, 160(%rdi), %ymm4, %ymm42563; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm4, %ymm42564; AVX512DQ-FCP-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm22565; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm2, %ymm22566; AVX512DQ-FCP-NEXT: vinserti128 $1, 176(%rdi), %ymm5, %ymm52567; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm5, %ymm52568; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm6 = ymm5[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm5[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]2569; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm7 = ymm2[11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7,8,9,10],ymm2[27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23,24,25,26]2570; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm3 = ymm3[11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7,8,9,10],ymm3[27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23,24,25,26]2571; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]2572; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm82573; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm4 = ymm4[11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7,8,9,10],ymm4[27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23,24,25,26]2574; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm1 = ymm1[11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7,8,9,10],ymm1[27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23,24,25,26]2575; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm4, %zmm1, %zmm22576; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm1 = ymm7[11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7,8,9,10],ymm7[27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23,24,25,26]2577; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm4 = ymm6[11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7,8,9,10],ymm6[27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23,24,25,26]2578; AVX512DQ-FCP-NEXT: vbroadcasti32x4 {{.*#+}} zmm5 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]2579; AVX512DQ-FCP-NEXT: # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]2580; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm2 ^ (zmm5 & (zmm8 ^ zmm2))2581; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[11,12,13,14,15],ymm7[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm7[16,17,18,19,20,21,22,23,24,25,26]2582; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm0 = ymm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]2583; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm2 = ymm3[11,12,13,14,15],ymm6[0,1,2,3,4,5,6,7,8,9,10],ymm3[27,28,29,30,31],ymm6[16,17,18,19,20,21,22,23,24,25,26]2584; AVX512DQ-FCP-NEXT: vpalignr {{.*#+}} ymm2 = ymm2[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]2585; AVX512DQ-FCP-NEXT: vmovdqa %ymm2, 32(%rsi)2586; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, (%rsi)2587; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, (%rdx)2588; AVX512DQ-FCP-NEXT: vmovdqa %ymm4, 32(%rcx)2589; AVX512DQ-FCP-NEXT: vmovdqa %ymm1, (%rcx)2590; AVX512DQ-FCP-NEXT: vzeroupper2591; AVX512DQ-FCP-NEXT: retq2592;2593; AVX512BW-LABEL: load_i8_stride3_vf64:2594; AVX512BW: # %bb.0:2595; AVX512BW-NEXT: vmovdqa (%rdi), %xmm02596; AVX512BW-NEXT: vmovdqa 16(%rdi), %xmm12597; AVX512BW-NEXT: vmovdqa 32(%rdi), %xmm22598; AVX512BW-NEXT: vmovdqa 96(%rdi), %xmm32599; AVX512BW-NEXT: vmovdqa 112(%rdi), %xmm42600; AVX512BW-NEXT: vmovdqa 128(%rdi), %xmm52601; AVX512BW-NEXT: vinserti128 $1, 144(%rdi), %ymm3, %ymm32602; AVX512BW-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm02603; AVX512BW-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm02604; AVX512BW-NEXT: vinserti128 $1, 160(%rdi), %ymm4, %ymm32605; AVX512BW-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm12606; AVX512BW-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm12607; AVX512BW-NEXT: vinserti128 $1, 176(%rdi), %ymm5, %ymm32608; AVX512BW-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm22609; AVX512BW-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm22610; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]2611; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]2612; AVX512BW-NEXT: vpshufb %zmm3, %zmm0, %zmm02613; AVX512BW-NEXT: vpshufb %zmm3, %zmm1, %zmm12614; AVX512BW-NEXT: vpshufb %zmm3, %zmm2, %zmm22615; AVX512BW-NEXT: vpalignr {{.*#+}} zmm3 = zmm2[11,12,13,14,15],zmm0[0,1,2,3,4,5,6,7,8,9,10],zmm2[27,28,29,30,31],zmm0[16,17,18,19,20,21,22,23,24,25,26],zmm2[43,44,45,46,47],zmm0[32,33,34,35,36,37,38,39,40,41,42],zmm2[59,60,61,62,63],zmm0[48,49,50,51,52,53,54,55,56,57,58]2616; AVX512BW-NEXT: vpalignr {{.*#+}} zmm0 = zmm0[11,12,13,14,15],zmm1[0,1,2,3,4,5,6,7,8,9,10],zmm0[27,28,29,30,31],zmm1[16,17,18,19,20,21,22,23,24,25,26],zmm0[43,44,45,46,47],zmm1[32,33,34,35,36,37,38,39,40,41,42],zmm0[59,60,61,62,63],zmm1[48,49,50,51,52,53,54,55,56,57,58]2617; AVX512BW-NEXT: vpalignr {{.*#+}} zmm1 = zmm1[11,12,13,14,15],zmm2[0,1,2,3,4,5,6,7,8,9,10],zmm1[27,28,29,30,31],zmm2[16,17,18,19,20,21,22,23,24,25,26],zmm1[43,44,45,46,47],zmm2[32,33,34,35,36,37,38,39,40,41,42],zmm1[59,60,61,62,63],zmm2[48,49,50,51,52,53,54,55,56,57,58]2618; AVX512BW-NEXT: movabsq $-576188069258921984, %rax # imm = 0xF800F800F800F8002619; AVX512BW-NEXT: kmovq %rax, %k12620; AVX512BW-NEXT: vpblendmb %zmm1, %zmm0, %zmm2 {%k1}2621; AVX512BW-NEXT: vpalignr {{.*#+}} zmm1 = zmm3[11,12,13,14,15],zmm1[0,1,2,3,4,5,6,7,8,9,10],zmm3[27,28,29,30,31],zmm1[16,17,18,19,20,21,22,23,24,25,26],zmm3[43,44,45,46,47],zmm1[32,33,34,35,36,37,38,39,40,41,42],zmm3[59,60,61,62,63],zmm1[48,49,50,51,52,53,54,55,56,57,58]2622; AVX512BW-NEXT: vpalignr {{.*#+}} zmm0 = zmm0[11,12,13,14,15],zmm3[0,1,2,3,4,5,6,7,8,9,10],zmm0[27,28,29,30,31],zmm3[16,17,18,19,20,21,22,23,24,25,26],zmm0[43,44,45,46,47],zmm3[32,33,34,35,36,37,38,39,40,41,42],zmm0[59,60,61,62,63],zmm3[48,49,50,51,52,53,54,55,56,57,58]2623; AVX512BW-NEXT: vpalignr {{.*#+}} zmm0 = zmm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25,42,43,44,45,46,47,32,33,34,35,36,37,38,39,40,41,58,59,60,61,62,63,48,49,50,51,52,53,54,55,56,57]2624; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rsi)2625; AVX512BW-NEXT: vmovdqa64 %zmm2, (%rdx)2626; AVX512BW-NEXT: vmovdqa64 %zmm1, (%rcx)2627; AVX512BW-NEXT: vzeroupper2628; AVX512BW-NEXT: retq2629;2630; AVX512BW-FCP-LABEL: load_i8_stride3_vf64:2631; AVX512BW-FCP: # %bb.0:2632; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %xmm02633; AVX512BW-FCP-NEXT: vmovdqa 16(%rdi), %xmm12634; AVX512BW-FCP-NEXT: vmovdqa 32(%rdi), %xmm22635; AVX512BW-FCP-NEXT: vmovdqa 96(%rdi), %xmm32636; AVX512BW-FCP-NEXT: vmovdqa 112(%rdi), %xmm42637; AVX512BW-FCP-NEXT: vmovdqa 128(%rdi), %xmm52638; AVX512BW-FCP-NEXT: vinserti128 $1, 144(%rdi), %ymm3, %ymm32639; AVX512BW-FCP-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm02640; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm02641; AVX512BW-FCP-NEXT: vinserti128 $1, 160(%rdi), %ymm4, %ymm32642; AVX512BW-FCP-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm12643; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm12644; AVX512BW-FCP-NEXT: vinserti128 $1, 176(%rdi), %ymm5, %ymm32645; AVX512BW-FCP-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm22646; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm22647; AVX512BW-FCP-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]2648; AVX512BW-FCP-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]2649; AVX512BW-FCP-NEXT: vpshufb %zmm3, %zmm0, %zmm02650; AVX512BW-FCP-NEXT: vpshufb %zmm3, %zmm1, %zmm12651; AVX512BW-FCP-NEXT: vpshufb %zmm3, %zmm2, %zmm22652; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} zmm3 = zmm2[11,12,13,14,15],zmm0[0,1,2,3,4,5,6,7,8,9,10],zmm2[27,28,29,30,31],zmm0[16,17,18,19,20,21,22,23,24,25,26],zmm2[43,44,45,46,47],zmm0[32,33,34,35,36,37,38,39,40,41,42],zmm2[59,60,61,62,63],zmm0[48,49,50,51,52,53,54,55,56,57,58]2653; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} zmm0 = zmm0[11,12,13,14,15],zmm1[0,1,2,3,4,5,6,7,8,9,10],zmm0[27,28,29,30,31],zmm1[16,17,18,19,20,21,22,23,24,25,26],zmm0[43,44,45,46,47],zmm1[32,33,34,35,36,37,38,39,40,41,42],zmm0[59,60,61,62,63],zmm1[48,49,50,51,52,53,54,55,56,57,58]2654; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} zmm1 = zmm1[11,12,13,14,15],zmm2[0,1,2,3,4,5,6,7,8,9,10],zmm1[27,28,29,30,31],zmm2[16,17,18,19,20,21,22,23,24,25,26],zmm1[43,44,45,46,47],zmm2[32,33,34,35,36,37,38,39,40,41,42],zmm1[59,60,61,62,63],zmm2[48,49,50,51,52,53,54,55,56,57,58]2655; AVX512BW-FCP-NEXT: movabsq $-576188069258921984, %rax # imm = 0xF800F800F800F8002656; AVX512BW-FCP-NEXT: kmovq %rax, %k12657; AVX512BW-FCP-NEXT: vpblendmb %zmm1, %zmm0, %zmm2 {%k1}2658; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} zmm1 = zmm3[11,12,13,14,15],zmm1[0,1,2,3,4,5,6,7,8,9,10],zmm3[27,28,29,30,31],zmm1[16,17,18,19,20,21,22,23,24,25,26],zmm3[43,44,45,46,47],zmm1[32,33,34,35,36,37,38,39,40,41,42],zmm3[59,60,61,62,63],zmm1[48,49,50,51,52,53,54,55,56,57,58]2659; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} zmm0 = zmm0[11,12,13,14,15],zmm3[0,1,2,3,4,5,6,7,8,9,10],zmm0[27,28,29,30,31],zmm3[16,17,18,19,20,21,22,23,24,25,26],zmm0[43,44,45,46,47],zmm3[32,33,34,35,36,37,38,39,40,41,42],zmm0[59,60,61,62,63],zmm3[48,49,50,51,52,53,54,55,56,57,58]2660; AVX512BW-FCP-NEXT: vpalignr {{.*#+}} zmm0 = zmm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25,42,43,44,45,46,47,32,33,34,35,36,37,38,39,40,41,58,59,60,61,62,63,48,49,50,51,52,53,54,55,56,57]2661; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, (%rsi)2662; AVX512BW-FCP-NEXT: vmovdqa64 %zmm2, (%rdx)2663; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, (%rcx)2664; AVX512BW-FCP-NEXT: vzeroupper2665; AVX512BW-FCP-NEXT: retq2666;2667; AVX512DQ-BW-LABEL: load_i8_stride3_vf64:2668; AVX512DQ-BW: # %bb.0:2669; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %xmm02670; AVX512DQ-BW-NEXT: vmovdqa 16(%rdi), %xmm12671; AVX512DQ-BW-NEXT: vmovdqa 32(%rdi), %xmm22672; AVX512DQ-BW-NEXT: vmovdqa 96(%rdi), %xmm32673; AVX512DQ-BW-NEXT: vmovdqa 112(%rdi), %xmm42674; AVX512DQ-BW-NEXT: vmovdqa 128(%rdi), %xmm52675; AVX512DQ-BW-NEXT: vinserti128 $1, 144(%rdi), %ymm3, %ymm32676; AVX512DQ-BW-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm02677; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm02678; AVX512DQ-BW-NEXT: vinserti128 $1, 160(%rdi), %ymm4, %ymm32679; AVX512DQ-BW-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm12680; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm12681; AVX512DQ-BW-NEXT: vinserti128 $1, 176(%rdi), %ymm5, %ymm32682; AVX512DQ-BW-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm22683; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm22684; AVX512DQ-BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]2685; AVX512DQ-BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]2686; AVX512DQ-BW-NEXT: vpshufb %zmm3, %zmm0, %zmm02687; AVX512DQ-BW-NEXT: vpshufb %zmm3, %zmm1, %zmm12688; AVX512DQ-BW-NEXT: vpshufb %zmm3, %zmm2, %zmm22689; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} zmm3 = zmm2[11,12,13,14,15],zmm0[0,1,2,3,4,5,6,7,8,9,10],zmm2[27,28,29,30,31],zmm0[16,17,18,19,20,21,22,23,24,25,26],zmm2[43,44,45,46,47],zmm0[32,33,34,35,36,37,38,39,40,41,42],zmm2[59,60,61,62,63],zmm0[48,49,50,51,52,53,54,55,56,57,58]2690; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} zmm0 = zmm0[11,12,13,14,15],zmm1[0,1,2,3,4,5,6,7,8,9,10],zmm0[27,28,29,30,31],zmm1[16,17,18,19,20,21,22,23,24,25,26],zmm0[43,44,45,46,47],zmm1[32,33,34,35,36,37,38,39,40,41,42],zmm0[59,60,61,62,63],zmm1[48,49,50,51,52,53,54,55,56,57,58]2691; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} zmm1 = zmm1[11,12,13,14,15],zmm2[0,1,2,3,4,5,6,7,8,9,10],zmm1[27,28,29,30,31],zmm2[16,17,18,19,20,21,22,23,24,25,26],zmm1[43,44,45,46,47],zmm2[32,33,34,35,36,37,38,39,40,41,42],zmm1[59,60,61,62,63],zmm2[48,49,50,51,52,53,54,55,56,57,58]2692; AVX512DQ-BW-NEXT: movabsq $-576188069258921984, %rax # imm = 0xF800F800F800F8002693; AVX512DQ-BW-NEXT: kmovq %rax, %k12694; AVX512DQ-BW-NEXT: vpblendmb %zmm1, %zmm0, %zmm2 {%k1}2695; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} zmm1 = zmm3[11,12,13,14,15],zmm1[0,1,2,3,4,5,6,7,8,9,10],zmm3[27,28,29,30,31],zmm1[16,17,18,19,20,21,22,23,24,25,26],zmm3[43,44,45,46,47],zmm1[32,33,34,35,36,37,38,39,40,41,42],zmm3[59,60,61,62,63],zmm1[48,49,50,51,52,53,54,55,56,57,58]2696; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} zmm0 = zmm0[11,12,13,14,15],zmm3[0,1,2,3,4,5,6,7,8,9,10],zmm0[27,28,29,30,31],zmm3[16,17,18,19,20,21,22,23,24,25,26],zmm0[43,44,45,46,47],zmm3[32,33,34,35,36,37,38,39,40,41,42],zmm0[59,60,61,62,63],zmm3[48,49,50,51,52,53,54,55,56,57,58]2697; AVX512DQ-BW-NEXT: vpalignr {{.*#+}} zmm0 = zmm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25,42,43,44,45,46,47,32,33,34,35,36,37,38,39,40,41,58,59,60,61,62,63,48,49,50,51,52,53,54,55,56,57]2698; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, (%rsi)2699; AVX512DQ-BW-NEXT: vmovdqa64 %zmm2, (%rdx)2700; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, (%rcx)2701; AVX512DQ-BW-NEXT: vzeroupper2702; AVX512DQ-BW-NEXT: retq2703;2704; AVX512DQ-BW-FCP-LABEL: load_i8_stride3_vf64:2705; AVX512DQ-BW-FCP: # %bb.0:2706; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %xmm02707; AVX512DQ-BW-FCP-NEXT: vmovdqa 16(%rdi), %xmm12708; AVX512DQ-BW-FCP-NEXT: vmovdqa 32(%rdi), %xmm22709; AVX512DQ-BW-FCP-NEXT: vmovdqa 96(%rdi), %xmm32710; AVX512DQ-BW-FCP-NEXT: vmovdqa 112(%rdi), %xmm42711; AVX512DQ-BW-FCP-NEXT: vmovdqa 128(%rdi), %xmm52712; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 144(%rdi), %ymm3, %ymm32713; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 48(%rdi), %ymm0, %ymm02714; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm02715; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 160(%rdi), %ymm4, %ymm32716; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 64(%rdi), %ymm1, %ymm12717; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm12718; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 176(%rdi), %ymm5, %ymm32719; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, 80(%rdi), %ymm2, %ymm22720; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm22721; AVX512DQ-BW-FCP-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13,0,3,6,9,12,15,2,5,8,11,14,1,4,7,10,13]2722; AVX512DQ-BW-FCP-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]2723; AVX512DQ-BW-FCP-NEXT: vpshufb %zmm3, %zmm0, %zmm02724; AVX512DQ-BW-FCP-NEXT: vpshufb %zmm3, %zmm1, %zmm12725; AVX512DQ-BW-FCP-NEXT: vpshufb %zmm3, %zmm2, %zmm22726; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} zmm3 = zmm2[11,12,13,14,15],zmm0[0,1,2,3,4,5,6,7,8,9,10],zmm2[27,28,29,30,31],zmm0[16,17,18,19,20,21,22,23,24,25,26],zmm2[43,44,45,46,47],zmm0[32,33,34,35,36,37,38,39,40,41,42],zmm2[59,60,61,62,63],zmm0[48,49,50,51,52,53,54,55,56,57,58]2727; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} zmm0 = zmm0[11,12,13,14,15],zmm1[0,1,2,3,4,5,6,7,8,9,10],zmm0[27,28,29,30,31],zmm1[16,17,18,19,20,21,22,23,24,25,26],zmm0[43,44,45,46,47],zmm1[32,33,34,35,36,37,38,39,40,41,42],zmm0[59,60,61,62,63],zmm1[48,49,50,51,52,53,54,55,56,57,58]2728; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} zmm1 = zmm1[11,12,13,14,15],zmm2[0,1,2,3,4,5,6,7,8,9,10],zmm1[27,28,29,30,31],zmm2[16,17,18,19,20,21,22,23,24,25,26],zmm1[43,44,45,46,47],zmm2[32,33,34,35,36,37,38,39,40,41,42],zmm1[59,60,61,62,63],zmm2[48,49,50,51,52,53,54,55,56,57,58]2729; AVX512DQ-BW-FCP-NEXT: movabsq $-576188069258921984, %rax # imm = 0xF800F800F800F8002730; AVX512DQ-BW-FCP-NEXT: kmovq %rax, %k12731; AVX512DQ-BW-FCP-NEXT: vpblendmb %zmm1, %zmm0, %zmm2 {%k1}2732; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} zmm1 = zmm3[11,12,13,14,15],zmm1[0,1,2,3,4,5,6,7,8,9,10],zmm3[27,28,29,30,31],zmm1[16,17,18,19,20,21,22,23,24,25,26],zmm3[43,44,45,46,47],zmm1[32,33,34,35,36,37,38,39,40,41,42],zmm3[59,60,61,62,63],zmm1[48,49,50,51,52,53,54,55,56,57,58]2733; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} zmm0 = zmm0[11,12,13,14,15],zmm3[0,1,2,3,4,5,6,7,8,9,10],zmm0[27,28,29,30,31],zmm3[16,17,18,19,20,21,22,23,24,25,26],zmm0[43,44,45,46,47],zmm3[32,33,34,35,36,37,38,39,40,41,42],zmm0[59,60,61,62,63],zmm3[48,49,50,51,52,53,54,55,56,57,58]2734; AVX512DQ-BW-FCP-NEXT: vpalignr {{.*#+}} zmm0 = zmm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25,42,43,44,45,46,47,32,33,34,35,36,37,38,39,40,41,58,59,60,61,62,63,48,49,50,51,52,53,54,55,56,57]2735; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, (%rsi)2736; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm2, (%rdx)2737; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, (%rcx)2738; AVX512DQ-BW-FCP-NEXT: vzeroupper2739; AVX512DQ-BW-FCP-NEXT: retq2740 %wide.vec = load <192 x i8>, ptr %in.vec, align 642741 %strided.vec0 = shufflevector <192 x i8> %wide.vec, <192 x i8> poison, <64 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45, i32 48, i32 51, i32 54, i32 57, i32 60, i32 63, i32 66, i32 69, i32 72, i32 75, i32 78, i32 81, i32 84, i32 87, i32 90, i32 93, i32 96, i32 99, i32 102, i32 105, i32 108, i32 111, i32 114, i32 117, i32 120, i32 123, i32 126, i32 129, i32 132, i32 135, i32 138, i32 141, i32 144, i32 147, i32 150, i32 153, i32 156, i32 159, i32 162, i32 165, i32 168, i32 171, i32 174, i32 177, i32 180, i32 183, i32 186, i32 189>2742 %strided.vec1 = shufflevector <192 x i8> %wide.vec, <192 x i8> poison, <64 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46, i32 49, i32 52, i32 55, i32 58, i32 61, i32 64, i32 67, i32 70, i32 73, i32 76, i32 79, i32 82, i32 85, i32 88, i32 91, i32 94, i32 97, i32 100, i32 103, i32 106, i32 109, i32 112, i32 115, i32 118, i32 121, i32 124, i32 127, i32 130, i32 133, i32 136, i32 139, i32 142, i32 145, i32 148, i32 151, i32 154, i32 157, i32 160, i32 163, i32 166, i32 169, i32 172, i32 175, i32 178, i32 181, i32 184, i32 187, i32 190>2743 %strided.vec2 = shufflevector <192 x i8> %wide.vec, <192 x i8> poison, <64 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47, i32 50, i32 53, i32 56, i32 59, i32 62, i32 65, i32 68, i32 71, i32 74, i32 77, i32 80, i32 83, i32 86, i32 89, i32 92, i32 95, i32 98, i32 101, i32 104, i32 107, i32 110, i32 113, i32 116, i32 119, i32 122, i32 125, i32 128, i32 131, i32 134, i32 137, i32 140, i32 143, i32 146, i32 149, i32 152, i32 155, i32 158, i32 161, i32 164, i32 167, i32 170, i32 173, i32 176, i32 179, i32 182, i32 185, i32 188, i32 191>2744 store <64 x i8> %strided.vec0, ptr %out.vec0, align 642745 store <64 x i8> %strided.vec1, ptr %out.vec1, align 642746 store <64 x i8> %strided.vec2, ptr %out.vec2, align 642747 ret void2748}2749