brintos / llvm-project-archived public Read only

1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2,FALLBACK03; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse4.2 | FileCheck %s --check-prefixes=SSE,SSE42,FALLBACK14; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx  | FileCheck %s --check-prefixes=AVX,AVX1-ONLY,FALLBACK25; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2,AVX2-SLOW,FALLBACK36; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST-PERLANE,FALLBACK47; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST,FALLBACK58; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512F,AVX512F-SLOW,FALLBACK69; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512F,AVX512F-FAST,FALLBACK710; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW,FALLBACK811; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-FAST,FALLBACK912 13define void @vec16_v2i8_to_v1i16_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {14; SSE2-LABEL: vec16_v2i8_to_v1i16_factor2:15; SSE2:       # %bb.0:16; SSE2-NEXT:    movdqa (%rdi), %xmm017; SSE2-NEXT:    paddb (%rsi), %xmm018; SSE2-NEXT:    pxor %xmm1, %xmm119; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]20; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]21; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]22; SSE2-NEXT:    paddb (%rdx), %xmm023; SSE2-NEXT:    movdqa %xmm0, (%rcx)24; SSE2-NEXT:    retq25;26; SSE42-LABEL: vec16_v2i8_to_v1i16_factor2:27; SSE42:       # %bb.0:28; SSE42-NEXT:    movdqa (%rdi), %xmm029; SSE42-NEXT:    paddb (%rsi), %xmm030; SSE42-NEXT:    pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero31; SSE42-NEXT:    paddb (%rdx), %xmm032; SSE42-NEXT:    movdqa %xmm0, (%rcx)33; SSE42-NEXT:    retq34;35; AVX-LABEL: vec16_v2i8_to_v1i16_factor2:36; AVX:       # %bb.0:37; AVX-NEXT:    vmovdqa (%rdi), %xmm038; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm039; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero40; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm041; AVX-NEXT:    vmovdqa %xmm0, (%rcx)42; AVX-NEXT:    retq43;44; AVX2-LABEL: vec16_v2i8_to_v1i16_factor2:45; AVX2:       # %bb.0:46; AVX2-NEXT:    vmovdqa (%rdi), %xmm047; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm048; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero49; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm050; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)51; AVX2-NEXT:    vzeroupper52; AVX2-NEXT:    retq53;54; AVX512F-LABEL: vec16_v2i8_to_v1i16_factor2:55; AVX512F:       # %bb.0:56; AVX512F-NEXT:    vmovdqa (%rdi), %xmm057; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm058; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero59; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm060; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)61; AVX512F-NEXT:    vzeroupper62; AVX512F-NEXT:    retq63;64; AVX512BW-LABEL: vec16_v2i8_to_v1i16_factor2:65; AVX512BW:       # %bb.0:66; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm067; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm068; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero69; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm070; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)71; AVX512BW-NEXT:    vzeroupper72; AVX512BW-NEXT:    retq73  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 6474  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 6475  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias76  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <2 x i32> <i32 0, i32 1>77  %zextd.vec = shufflevector <2 x i8> %in.vec.trunc, <2 x i8> zeroinitializer, <2 x i32> <i32 0, i32 3>78  %out.bytevec.padded = shufflevector <2 x i8> %zextd.vec, <2 x i8> poison, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>79  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 6480  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias81  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 6482  ret void83}84 85define void @vec32_v4i8_to_v2i16_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {86; SSE2-LABEL: vec32_v4i8_to_v2i16_factor2:87; SSE2:       # %bb.0:88; SSE2-NEXT:    movdqa (%rdi), %xmm089; SSE2-NEXT:    paddb (%rsi), %xmm090; SSE2-NEXT:    pxor %xmm1, %xmm191; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]92; SSE2-NEXT:    paddb (%rdx), %xmm093; SSE2-NEXT:    movdqa %xmm0, (%rcx)94; SSE2-NEXT:    retq95;96; SSE42-LABEL: vec32_v4i8_to_v2i16_factor2:97; SSE42:       # %bb.0:98; SSE42-NEXT:    movdqa (%rdi), %xmm099; SSE42-NEXT:    paddb (%rsi), %xmm0100; SSE42-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero101; SSE42-NEXT:    paddb (%rdx), %xmm0102; SSE42-NEXT:    movdqa %xmm0, (%rcx)103; SSE42-NEXT:    retq104;105; AVX-LABEL: vec32_v4i8_to_v2i16_factor2:106; AVX:       # %bb.0:107; AVX-NEXT:    vmovdqa (%rdi), %xmm0108; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm0109; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero110; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm0111; AVX-NEXT:    vmovdqa %xmm0, (%rcx)112; AVX-NEXT:    retq113;114; AVX2-LABEL: vec32_v4i8_to_v2i16_factor2:115; AVX2:       # %bb.0:116; AVX2-NEXT:    vmovdqa (%rdi), %xmm0117; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm0118; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero119; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm0120; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)121; AVX2-NEXT:    vzeroupper122; AVX2-NEXT:    retq123;124; AVX512F-LABEL: vec32_v4i8_to_v2i16_factor2:125; AVX512F:       # %bb.0:126; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0127; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0128; AVX512F-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero129; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0130; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)131; AVX512F-NEXT:    vzeroupper132; AVX512F-NEXT:    retq133;134; AVX512BW-LABEL: vec32_v4i8_to_v2i16_factor2:135; AVX512BW:       # %bb.0:136; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0137; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm0138; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero139; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0140; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)141; AVX512BW-NEXT:    vzeroupper142; AVX512BW-NEXT:    retq143  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 64144  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 64145  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias146  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>147  %zextd.vec = shufflevector <4 x i8> %in.vec.trunc, <4 x i8> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 1, i32 7>148  %out.bytevec.padded = shufflevector <4 x i8> %zextd.vec, <4 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>149  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 64150  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias151  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 64152  ret void153}154 155define void @vec32_v4i8_to_v1i32_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {156; SSE2-LABEL: vec32_v4i8_to_v1i32_factor4:157; SSE2:       # %bb.0:158; SSE2-NEXT:    movdqa (%rdi), %xmm0159; SSE2-NEXT:    paddb (%rsi), %xmm0160; SSE2-NEXT:    pxor %xmm1, %xmm1161; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]162; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]163; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]164; SSE2-NEXT:    paddb (%rdx), %xmm0165; SSE2-NEXT:    movdqa %xmm0, (%rcx)166; SSE2-NEXT:    retq167;168; SSE42-LABEL: vec32_v4i8_to_v1i32_factor4:169; SSE42:       # %bb.0:170; SSE42-NEXT:    movdqa (%rdi), %xmm0171; SSE42-NEXT:    paddb (%rsi), %xmm0172; SSE42-NEXT:    pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero173; SSE42-NEXT:    paddb (%rdx), %xmm0174; SSE42-NEXT:    movdqa %xmm0, (%rcx)175; SSE42-NEXT:    retq176;177; AVX-LABEL: vec32_v4i8_to_v1i32_factor4:178; AVX:       # %bb.0:179; AVX-NEXT:    vmovdqa (%rdi), %xmm0180; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm0181; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero182; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm0183; AVX-NEXT:    vmovdqa %xmm0, (%rcx)184; AVX-NEXT:    retq185;186; AVX2-LABEL: vec32_v4i8_to_v1i32_factor4:187; AVX2:       # %bb.0:188; AVX2-NEXT:    vmovdqa (%rdi), %xmm0189; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm0190; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero191; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm0192; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)193; AVX2-NEXT:    vzeroupper194; AVX2-NEXT:    retq195;196; AVX512F-LABEL: vec32_v4i8_to_v1i32_factor4:197; AVX512F:       # %bb.0:198; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0199; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0200; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero201; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0202; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)203; AVX512F-NEXT:    vzeroupper204; AVX512F-NEXT:    retq205;206; AVX512BW-LABEL: vec32_v4i8_to_v1i32_factor4:207; AVX512BW:       # %bb.0:208; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0209; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm0210; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero211; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0212; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)213; AVX512BW-NEXT:    vzeroupper214; AVX512BW-NEXT:    retq215  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 64216  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 64217  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias218  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>219  %zextd.vec = shufflevector <4 x i8> %in.vec.trunc, <4 x i8> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 6, i32 7>220  %out.bytevec.padded = shufflevector <4 x i8> %zextd.vec, <4 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>221  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 64222  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias223  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 64224  ret void225}226 227define void @vec32_v2i16_to_v1i32_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {228; SSE2-LABEL: vec32_v2i16_to_v1i32_factor2:229; SSE2:       # %bb.0:230; SSE2-NEXT:    movdqa (%rdi), %xmm0231; SSE2-NEXT:    paddb (%rsi), %xmm0232; SSE2-NEXT:    pxor %xmm1, %xmm1233; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]234; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]235; SSE2-NEXT:    paddb (%rdx), %xmm0236; SSE2-NEXT:    movdqa %xmm0, (%rcx)237; SSE2-NEXT:    retq238;239; SSE42-LABEL: vec32_v2i16_to_v1i32_factor2:240; SSE42:       # %bb.0:241; SSE42-NEXT:    movdqa (%rdi), %xmm0242; SSE42-NEXT:    paddb (%rsi), %xmm0243; SSE42-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero244; SSE42-NEXT:    paddb (%rdx), %xmm0245; SSE42-NEXT:    movdqa %xmm0, (%rcx)246; SSE42-NEXT:    retq247;248; AVX-LABEL: vec32_v2i16_to_v1i32_factor2:249; AVX:       # %bb.0:250; AVX-NEXT:    vmovdqa (%rdi), %xmm0251; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm0252; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero253; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm0254; AVX-NEXT:    vmovdqa %xmm0, (%rcx)255; AVX-NEXT:    retq256;257; AVX2-LABEL: vec32_v2i16_to_v1i32_factor2:258; AVX2:       # %bb.0:259; AVX2-NEXT:    vmovdqa (%rdi), %xmm0260; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm0261; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero262; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm0263; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)264; AVX2-NEXT:    vzeroupper265; AVX2-NEXT:    retq266;267; AVX512F-LABEL: vec32_v2i16_to_v1i32_factor2:268; AVX512F:       # %bb.0:269; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0270; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0271; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero272; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0273; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)274; AVX512F-NEXT:    vzeroupper275; AVX512F-NEXT:    retq276;277; AVX512BW-LABEL: vec32_v2i16_to_v1i32_factor2:278; AVX512BW:       # %bb.0:279; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0280; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm0281; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero282; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0283; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)284; AVX512BW-NEXT:    vzeroupper285; AVX512BW-NEXT:    retq286  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 64287  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 64288  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias289  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>290  %in.vec.cast = bitcast <4 x i8> %in.vec.trunc to <2 x i16>291  %zextd.vec = shufflevector <2 x i16> %in.vec.cast, <2 x i16> zeroinitializer, <2 x i32> <i32 0, i32 3>292  %out.bytevec = bitcast <2 x i16> %zextd.vec to <4 x i8>293  %out.bytevec.padded = shufflevector <4 x i8> %out.bytevec, <4 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>294  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 64295  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias296  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 64297  ret void298}299 300define void @vec64_v8i8_to_v4i16_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {301; SSE2-LABEL: vec64_v8i8_to_v4i16_factor2:302; SSE2:       # %bb.0:303; SSE2-NEXT:    movdqa (%rdi), %xmm0304; SSE2-NEXT:    paddb (%rsi), %xmm0305; SSE2-NEXT:    pxor %xmm1, %xmm1306; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]307; SSE2-NEXT:    paddb (%rdx), %xmm0308; SSE2-NEXT:    movdqa %xmm0, (%rcx)309; SSE2-NEXT:    retq310;311; SSE42-LABEL: vec64_v8i8_to_v4i16_factor2:312; SSE42:       # %bb.0:313; SSE42-NEXT:    movdqa (%rdi), %xmm0314; SSE42-NEXT:    paddb (%rsi), %xmm0315; SSE42-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero316; SSE42-NEXT:    paddb (%rdx), %xmm0317; SSE42-NEXT:    movdqa %xmm0, (%rcx)318; SSE42-NEXT:    retq319;320; AVX-LABEL: vec64_v8i8_to_v4i16_factor2:321; AVX:       # %bb.0:322; AVX-NEXT:    vmovdqa (%rdi), %xmm0323; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm0324; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero325; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm0326; AVX-NEXT:    vmovdqa %xmm0, (%rcx)327; AVX-NEXT:    retq328;329; AVX2-LABEL: vec64_v8i8_to_v4i16_factor2:330; AVX2:       # %bb.0:331; AVX2-NEXT:    vmovdqa (%rdi), %xmm0332; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm0333; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero334; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm0335; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)336; AVX2-NEXT:    vzeroupper337; AVX2-NEXT:    retq338;339; AVX512F-LABEL: vec64_v8i8_to_v4i16_factor2:340; AVX512F:       # %bb.0:341; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0342; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0343; AVX512F-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero344; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0345; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)346; AVX512F-NEXT:    vzeroupper347; AVX512F-NEXT:    retq348;349; AVX512BW-LABEL: vec64_v8i8_to_v4i16_factor2:350; AVX512BW:       # %bb.0:351; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0352; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm0353; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero354; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0355; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)356; AVX512BW-NEXT:    vzeroupper357; AVX512BW-NEXT:    retq358  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 64359  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 64360  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias361  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>362  %zextd.vec = shufflevector <8 x i8> %in.vec.trunc, <8 x i8> zeroinitializer, <8 x i32> <i32 0, i32 9, i32 1, i32 11, i32 2, i32 13, i32 3, i32 15>363  %out.bytevec.padded = shufflevector <8 x i8> %zextd.vec, <8 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>364  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 64365  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias366  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 64367  ret void368}369 370define void @vec64_v8i8_to_v2i32_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {371; SSE2-LABEL: vec64_v8i8_to_v2i32_factor4:372; SSE2:       # %bb.0:373; SSE2-NEXT:    movdqa (%rdi), %xmm0374; SSE2-NEXT:    paddb (%rsi), %xmm0375; SSE2-NEXT:    pxor %xmm1, %xmm1376; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]377; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]378; SSE2-NEXT:    paddb (%rdx), %xmm0379; SSE2-NEXT:    movdqa %xmm0, (%rcx)380; SSE2-NEXT:    retq381;382; SSE42-LABEL: vec64_v8i8_to_v2i32_factor4:383; SSE42:       # %bb.0:384; SSE42-NEXT:    movdqa (%rdi), %xmm0385; SSE42-NEXT:    paddb (%rsi), %xmm0386; SSE42-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero387; SSE42-NEXT:    paddb (%rdx), %xmm0388; SSE42-NEXT:    movdqa %xmm0, (%rcx)389; SSE42-NEXT:    retq390;391; AVX-LABEL: vec64_v8i8_to_v2i32_factor4:392; AVX:       # %bb.0:393; AVX-NEXT:    vmovdqa (%rdi), %xmm0394; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm0395; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero396; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm0397; AVX-NEXT:    vmovdqa %xmm0, (%rcx)398; AVX-NEXT:    retq399;400; AVX2-LABEL: vec64_v8i8_to_v2i32_factor4:401; AVX2:       # %bb.0:402; AVX2-NEXT:    vmovdqa (%rdi), %xmm0403; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm0404; AVX2-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero405; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm0406; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)407; AVX2-NEXT:    vzeroupper408; AVX2-NEXT:    retq409;410; AVX512F-LABEL: vec64_v8i8_to_v2i32_factor4:411; AVX512F:       # %bb.0:412; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0413; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0414; AVX512F-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero415; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0416; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)417; AVX512F-NEXT:    vzeroupper418; AVX512F-NEXT:    retq419;420; AVX512BW-LABEL: vec64_v8i8_to_v2i32_factor4:421; AVX512BW:       # %bb.0:422; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0423; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm0424; AVX512BW-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero425; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0426; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)427; AVX512BW-NEXT:    vzeroupper428; AVX512BW-NEXT:    retq429  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 64430  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 64431  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias432  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>433  %zextd.vec = shufflevector <8 x i8> %in.vec.trunc, <8 x i8> zeroinitializer, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 1, i32 13, i32 14, i32 15>434  %out.bytevec.padded = shufflevector <8 x i8> %zextd.vec, <8 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>435  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 64436  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias437  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 64438  ret void439}440 441define void @vec64_v8i8_to_v1i64_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {442; SSE2-LABEL: vec64_v8i8_to_v1i64_factor8:443; SSE2:       # %bb.0:444; SSE2-NEXT:    movdqa (%rdi), %xmm0445; SSE2-NEXT:    paddb (%rsi), %xmm0446; SSE2-NEXT:    pxor %xmm1, %xmm1447; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]448; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]449; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]450; SSE2-NEXT:    paddb (%rdx), %xmm0451; SSE2-NEXT:    movdqa %xmm0, (%rcx)452; SSE2-NEXT:    retq453;454; SSE42-LABEL: vec64_v8i8_to_v1i64_factor8:455; SSE42:       # %bb.0:456; SSE42-NEXT:    movdqa (%rdi), %xmm0457; SSE42-NEXT:    paddb (%rsi), %xmm0458; SSE42-NEXT:    pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero459; SSE42-NEXT:    paddb (%rdx), %xmm0460; SSE42-NEXT:    movdqa %xmm0, (%rcx)461; SSE42-NEXT:    retq462;463; AVX-LABEL: vec64_v8i8_to_v1i64_factor8:464; AVX:       # %bb.0:465; AVX-NEXT:    vmovdqa (%rdi), %xmm0466; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm0467; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero468; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm0469; AVX-NEXT:    vmovdqa %xmm0, (%rcx)470; AVX-NEXT:    retq471;472; AVX2-LABEL: vec64_v8i8_to_v1i64_factor8:473; AVX2:       # %bb.0:474; AVX2-NEXT:    vmovdqa (%rdi), %xmm0475; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm0476; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero477; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm0478; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)479; AVX2-NEXT:    vzeroupper480; AVX2-NEXT:    retq481;482; AVX512F-LABEL: vec64_v8i8_to_v1i64_factor8:483; AVX512F:       # %bb.0:484; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0485; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0486; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero487; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0488; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)489; AVX512F-NEXT:    vzeroupper490; AVX512F-NEXT:    retq491;492; AVX512BW-LABEL: vec64_v8i8_to_v1i64_factor8:493; AVX512BW:       # %bb.0:494; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0495; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm0496; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero497; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0498; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)499; AVX512BW-NEXT:    vzeroupper500; AVX512BW-NEXT:    retq501  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 64502  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 64503  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias504  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>505  %zextd.vec = shufflevector <8 x i8> %in.vec.trunc, <8 x i8> zeroinitializer, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>506  %out.bytevec.padded = shufflevector <8 x i8> %zextd.vec, <8 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>507  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 64508  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias509  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 64510  ret void511}512 513define void @vec64_v4i16_to_v2i32_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {514; SSE2-LABEL: vec64_v4i16_to_v2i32_factor2:515; SSE2:       # %bb.0:516; SSE2-NEXT:    movdqa (%rdi), %xmm0517; SSE2-NEXT:    paddb (%rsi), %xmm0518; SSE2-NEXT:    pxor %xmm1, %xmm1519; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]520; SSE2-NEXT:    paddb (%rdx), %xmm0521; SSE2-NEXT:    movdqa %xmm0, (%rcx)522; SSE2-NEXT:    retq523;524; SSE42-LABEL: vec64_v4i16_to_v2i32_factor2:525; SSE42:       # %bb.0:526; SSE42-NEXT:    movdqa (%rdi), %xmm0527; SSE42-NEXT:    paddb (%rsi), %xmm0528; SSE42-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero529; SSE42-NEXT:    paddb (%rdx), %xmm0530; SSE42-NEXT:    movdqa %xmm0, (%rcx)531; SSE42-NEXT:    retq532;533; AVX-LABEL: vec64_v4i16_to_v2i32_factor2:534; AVX:       # %bb.0:535; AVX-NEXT:    vmovdqa (%rdi), %xmm0536; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm0537; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero538; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm0539; AVX-NEXT:    vmovdqa %xmm0, (%rcx)540; AVX-NEXT:    retq541;542; AVX2-LABEL: vec64_v4i16_to_v2i32_factor2:543; AVX2:       # %bb.0:544; AVX2-NEXT:    vmovdqa (%rdi), %xmm0545; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm0546; AVX2-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero547; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm0548; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)549; AVX2-NEXT:    vzeroupper550; AVX2-NEXT:    retq551;552; AVX512F-LABEL: vec64_v4i16_to_v2i32_factor2:553; AVX512F:       # %bb.0:554; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0555; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0556; AVX512F-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero557; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0558; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)559; AVX512F-NEXT:    vzeroupper560; AVX512F-NEXT:    retq561;562; AVX512BW-LABEL: vec64_v4i16_to_v2i32_factor2:563; AVX512BW:       # %bb.0:564; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0565; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm0566; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero567; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0568; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)569; AVX512BW-NEXT:    vzeroupper570; AVX512BW-NEXT:    retq571  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 64572  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 64573  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias574  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>575  %in.vec.cast = bitcast <8 x i8> %in.vec.trunc to <4 x i16>576  %zextd.vec = shufflevector <4 x i16> %in.vec.cast, <4 x i16> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 1, i32 7>577  %out.bytevec = bitcast <4 x i16> %zextd.vec to <8 x i8>578  %out.bytevec.padded = shufflevector <8 x i8> %out.bytevec, <8 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>579  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 64580  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias581  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 64582  ret void583}584 585define void @vec64_v4i16_to_v1i64_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {586; SSE2-LABEL: vec64_v4i16_to_v1i64_factor4:587; SSE2:       # %bb.0:588; SSE2-NEXT:    movdqa (%rdi), %xmm0589; SSE2-NEXT:    paddb (%rsi), %xmm0590; SSE2-NEXT:    pxor %xmm1, %xmm1591; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]592; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]593; SSE2-NEXT:    paddb (%rdx), %xmm0594; SSE2-NEXT:    movdqa %xmm0, (%rcx)595; SSE2-NEXT:    retq596;597; SSE42-LABEL: vec64_v4i16_to_v1i64_factor4:598; SSE42:       # %bb.0:599; SSE42-NEXT:    movdqa (%rdi), %xmm0600; SSE42-NEXT:    paddb (%rsi), %xmm0601; SSE42-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero602; SSE42-NEXT:    paddb (%rdx), %xmm0603; SSE42-NEXT:    movdqa %xmm0, (%rcx)604; SSE42-NEXT:    retq605;606; AVX-LABEL: vec64_v4i16_to_v1i64_factor4:607; AVX:       # %bb.0:608; AVX-NEXT:    vmovdqa (%rdi), %xmm0609; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm0610; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero611; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm0612; AVX-NEXT:    vmovdqa %xmm0, (%rcx)613; AVX-NEXT:    retq614;615; AVX2-LABEL: vec64_v4i16_to_v1i64_factor4:616; AVX2:       # %bb.0:617; AVX2-NEXT:    vmovdqa (%rdi), %xmm0618; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm0619; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero620; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm0621; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)622; AVX2-NEXT:    vzeroupper623; AVX2-NEXT:    retq624;625; AVX512F-LABEL: vec64_v4i16_to_v1i64_factor4:626; AVX512F:       # %bb.0:627; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0628; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0629; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero630; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0631; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)632; AVX512F-NEXT:    vzeroupper633; AVX512F-NEXT:    retq634;635; AVX512BW-LABEL: vec64_v4i16_to_v1i64_factor4:636; AVX512BW:       # %bb.0:637; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0638; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm0639; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero640; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0641; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)642; AVX512BW-NEXT:    vzeroupper643; AVX512BW-NEXT:    retq644  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 64645  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 64646  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias647  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>648  %in.vec.cast = bitcast <8 x i8> %in.vec.trunc to <4 x i16>649  %zextd.vec = shufflevector <4 x i16> %in.vec.cast, <4 x i16> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 6, i32 7>650  %out.bytevec = bitcast <4 x i16> %zextd.vec to <8 x i8>651  %out.bytevec.padded = shufflevector <8 x i8> %out.bytevec, <8 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>652  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 64653  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias654  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 64655  ret void656}657 658define void @vec64_v2i32_to_v1i64_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {659; SSE2-LABEL: vec64_v2i32_to_v1i64_factor2:660; SSE2:       # %bb.0:661; SSE2-NEXT:    movdqa (%rdi), %xmm0662; SSE2-NEXT:    paddb (%rsi), %xmm0663; SSE2-NEXT:    pxor %xmm1, %xmm1664; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]665; SSE2-NEXT:    paddb (%rdx), %xmm0666; SSE2-NEXT:    movdqa %xmm0, (%rcx)667; SSE2-NEXT:    retq668;669; SSE42-LABEL: vec64_v2i32_to_v1i64_factor2:670; SSE42:       # %bb.0:671; SSE42-NEXT:    movdqa (%rdi), %xmm0672; SSE42-NEXT:    paddb (%rsi), %xmm0673; SSE42-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero674; SSE42-NEXT:    paddb (%rdx), %xmm0675; SSE42-NEXT:    movdqa %xmm0, (%rcx)676; SSE42-NEXT:    retq677;678; AVX-LABEL: vec64_v2i32_to_v1i64_factor2:679; AVX:       # %bb.0:680; AVX-NEXT:    vmovdqa (%rdi), %xmm0681; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm0682; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero683; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm0684; AVX-NEXT:    vmovdqa %xmm0, (%rcx)685; AVX-NEXT:    retq686;687; AVX2-LABEL: vec64_v2i32_to_v1i64_factor2:688; AVX2:       # %bb.0:689; AVX2-NEXT:    vmovdqa (%rdi), %xmm0690; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm0691; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero692; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm0693; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)694; AVX2-NEXT:    vzeroupper695; AVX2-NEXT:    retq696;697; AVX512F-LABEL: vec64_v2i32_to_v1i64_factor2:698; AVX512F:       # %bb.0:699; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0700; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0701; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero702; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0703; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)704; AVX512F-NEXT:    vzeroupper705; AVX512F-NEXT:    retq706;707; AVX512BW-LABEL: vec64_v2i32_to_v1i64_factor2:708; AVX512BW:       # %bb.0:709; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0710; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm0711; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero712; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0713; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)714; AVX512BW-NEXT:    vzeroupper715; AVX512BW-NEXT:    retq716  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 64717  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 64718  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias719  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>720  %in.vec.cast = bitcast <8 x i8> %in.vec.trunc to <2 x i32>721  %zextd.vec = shufflevector <2 x i32> %in.vec.cast, <2 x i32> zeroinitializer, <2 x i32> <i32 0, i32 3>722  %out.bytevec = bitcast <2 x i32> %zextd.vec to <8 x i8>723  %out.bytevec.padded = shufflevector <8 x i8> %out.bytevec, <8 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>724  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 64725  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias726  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 64727  ret void728}729 730define void @vec128_v16i8_to_v8i16_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {731; SSE2-LABEL: vec128_v16i8_to_v8i16_factor2:732; SSE2:       # %bb.0:733; SSE2-NEXT:    movdqa (%rdi), %xmm0734; SSE2-NEXT:    paddb (%rsi), %xmm0735; SSE2-NEXT:    pxor %xmm1, %xmm1736; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]737; SSE2-NEXT:    paddb (%rdx), %xmm0738; SSE2-NEXT:    movdqa %xmm0, (%rcx)739; SSE2-NEXT:    retq740;741; SSE42-LABEL: vec128_v16i8_to_v8i16_factor2:742; SSE42:       # %bb.0:743; SSE42-NEXT:    movdqa (%rdi), %xmm0744; SSE42-NEXT:    paddb (%rsi), %xmm0745; SSE42-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero746; SSE42-NEXT:    paddb (%rdx), %xmm0747; SSE42-NEXT:    movdqa %xmm0, (%rcx)748; SSE42-NEXT:    retq749;750; AVX-LABEL: vec128_v16i8_to_v8i16_factor2:751; AVX:       # %bb.0:752; AVX-NEXT:    vmovdqa (%rdi), %xmm0753; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm0754; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero755; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm0756; AVX-NEXT:    vmovdqa %xmm0, (%rcx)757; AVX-NEXT:    retq758;759; AVX2-LABEL: vec128_v16i8_to_v8i16_factor2:760; AVX2:       # %bb.0:761; AVX2-NEXT:    vmovdqa (%rdi), %xmm0762; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm0763; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero764; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm0765; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)766; AVX2-NEXT:    vzeroupper767; AVX2-NEXT:    retq768;769; AVX512F-LABEL: vec128_v16i8_to_v8i16_factor2:770; AVX512F:       # %bb.0:771; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0772; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0773; AVX512F-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero774; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0775; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)776; AVX512F-NEXT:    vzeroupper777; AVX512F-NEXT:    retq778;779; AVX512BW-LABEL: vec128_v16i8_to_v8i16_factor2:780; AVX512BW:       # %bb.0:781; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0782; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm0783; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero784; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0785; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)786; AVX512BW-NEXT:    vzeroupper787; AVX512BW-NEXT:    retq788  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 64789  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 64790  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias791  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>792  %zextd.vec = shufflevector <16 x i8> %in.vec.trunc, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 17, i32 1, i32 19, i32 2, i32 21, i32 3, i32 23, i32 4, i32 25, i32 5, i32 27, i32 6, i32 29, i32 7, i32 31>793  %out.bytevec.padded = shufflevector <16 x i8> %zextd.vec, <16 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>794  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 64795  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias796  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 64797  ret void798}799 800define void @vec128_v16i8_to_v4i32_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {801; SSE2-LABEL: vec128_v16i8_to_v4i32_factor4:802; SSE2:       # %bb.0:803; SSE2-NEXT:    movdqa (%rdi), %xmm0804; SSE2-NEXT:    paddb (%rsi), %xmm0805; SSE2-NEXT:    pxor %xmm1, %xmm1806; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]807; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]808; SSE2-NEXT:    paddb (%rdx), %xmm0809; SSE2-NEXT:    movdqa %xmm0, (%rcx)810; SSE2-NEXT:    retq811;812; SSE42-LABEL: vec128_v16i8_to_v4i32_factor4:813; SSE42:       # %bb.0:814; SSE42-NEXT:    movdqa (%rdi), %xmm0815; SSE42-NEXT:    paddb (%rsi), %xmm0816; SSE42-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero817; SSE42-NEXT:    paddb (%rdx), %xmm0818; SSE42-NEXT:    movdqa %xmm0, (%rcx)819; SSE42-NEXT:    retq820;821; AVX-LABEL: vec128_v16i8_to_v4i32_factor4:822; AVX:       # %bb.0:823; AVX-NEXT:    vmovdqa (%rdi), %xmm0824; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm0825; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero826; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm0827; AVX-NEXT:    vmovdqa %xmm0, (%rcx)828; AVX-NEXT:    retq829;830; AVX2-LABEL: vec128_v16i8_to_v4i32_factor4:831; AVX2:       # %bb.0:832; AVX2-NEXT:    vmovdqa (%rdi), %xmm0833; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm0834; AVX2-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero835; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm0836; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)837; AVX2-NEXT:    vzeroupper838; AVX2-NEXT:    retq839;840; AVX512F-LABEL: vec128_v16i8_to_v4i32_factor4:841; AVX512F:       # %bb.0:842; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0843; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0844; AVX512F-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero845; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0846; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)847; AVX512F-NEXT:    vzeroupper848; AVX512F-NEXT:    retq849;850; AVX512BW-LABEL: vec128_v16i8_to_v4i32_factor4:851; AVX512BW:       # %bb.0:852; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0853; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm0854; AVX512BW-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero855; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0856; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)857; AVX512BW-NEXT:    vzeroupper858; AVX512BW-NEXT:    retq859  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 64860  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 64861  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias862  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>863  %zextd.vec = shufflevector <16 x i8> %in.vec.trunc, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 17, i32 18, i32 19, i32 1, i32 21, i32 22, i32 23, i32 2, i32 25, i32 26, i32 27, i32 3, i32 29, i32 30, i32 31>864  %out.bytevec.padded = shufflevector <16 x i8> %zextd.vec, <16 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>865  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 64866  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias867  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 64868  ret void869}870 871define void @vec128_v16i8_to_v2i64_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {872; SSE2-LABEL: vec128_v16i8_to_v2i64_factor8:873; SSE2:       # %bb.0:874; SSE2-NEXT:    movdqa (%rdi), %xmm0875; SSE2-NEXT:    paddb (%rsi), %xmm0876; SSE2-NEXT:    pxor %xmm1, %xmm1877; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]878; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]879; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]880; SSE2-NEXT:    paddb (%rdx), %xmm0881; SSE2-NEXT:    movdqa %xmm0, (%rcx)882; SSE2-NEXT:    retq883;884; SSE42-LABEL: vec128_v16i8_to_v2i64_factor8:885; SSE42:       # %bb.0:886; SSE42-NEXT:    movdqa (%rdi), %xmm0887; SSE42-NEXT:    paddb (%rsi), %xmm0888; SSE42-NEXT:    pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero889; SSE42-NEXT:    paddb (%rdx), %xmm0890; SSE42-NEXT:    movdqa %xmm0, (%rcx)891; SSE42-NEXT:    retq892;893; AVX-LABEL: vec128_v16i8_to_v2i64_factor8:894; AVX:       # %bb.0:895; AVX-NEXT:    vmovdqa (%rdi), %xmm0896; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm0897; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero898; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm0899; AVX-NEXT:    vmovdqa %xmm0, (%rcx)900; AVX-NEXT:    retq901;902; AVX2-LABEL: vec128_v16i8_to_v2i64_factor8:903; AVX2:       # %bb.0:904; AVX2-NEXT:    vmovdqa (%rdi), %xmm0905; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm0906; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero907; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm0908; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)909; AVX2-NEXT:    vzeroupper910; AVX2-NEXT:    retq911;912; AVX512F-LABEL: vec128_v16i8_to_v2i64_factor8:913; AVX512F:       # %bb.0:914; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0915; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0916; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero917; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0918; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)919; AVX512F-NEXT:    vzeroupper920; AVX512F-NEXT:    retq921;922; AVX512BW-LABEL: vec128_v16i8_to_v2i64_factor8:923; AVX512BW:       # %bb.0:924; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0925; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm0926; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero927; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0928; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)929; AVX512BW-NEXT:    vzeroupper930; AVX512BW-NEXT:    retq931  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 64932  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 64933  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias934  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>935  %zextd.vec = shufflevector <16 x i8> %in.vec.trunc, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 1, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>936  %out.bytevec.padded = shufflevector <16 x i8> %zextd.vec, <16 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>937  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 64938  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias939  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 64940  ret void941}942 943define void @vec128_v16i8_to_v1i128_factor16(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {944; SSE-LABEL: vec128_v16i8_to_v1i128_factor16:945; SSE:       # %bb.0:946; SSE-NEXT:    movdqa (%rdi), %xmm0947; SSE-NEXT:    paddb (%rsi), %xmm0948; SSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0949; SSE-NEXT:    paddb (%rdx), %xmm0950; SSE-NEXT:    movdqa %xmm0, (%rcx)951; SSE-NEXT:    retq952;953; AVX-LABEL: vec128_v16i8_to_v1i128_factor16:954; AVX:       # %bb.0:955; AVX-NEXT:    vmovdqa (%rdi), %xmm0956; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm0957; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0958; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm0959; AVX-NEXT:    vmovdqa %xmm0, (%rcx)960; AVX-NEXT:    retq961;962; AVX2-LABEL: vec128_v16i8_to_v1i128_factor16:963; AVX2:       # %bb.0:964; AVX2-NEXT:    vmovdqa (%rdi), %xmm0965; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm0966; AVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0967; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm0968; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)969; AVX2-NEXT:    vzeroupper970; AVX2-NEXT:    retq971;972; AVX512F-LABEL: vec128_v16i8_to_v1i128_factor16:973; AVX512F:       # %bb.0:974; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0975; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0976; AVX512F-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0977; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0978; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)979; AVX512F-NEXT:    vzeroupper980; AVX512F-NEXT:    retq981;982; AVX512BW-LABEL: vec128_v16i8_to_v1i128_factor16:983; AVX512BW:       # %bb.0:984; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0985; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm0986; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0987; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0988; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)989; AVX512BW-NEXT:    vzeroupper990; AVX512BW-NEXT:    retq991  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 64992  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 64993  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias994  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>995  %zextd.vec = shufflevector <16 x i8> %in.vec.trunc, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>996  %out.bytevec.padded = shufflevector <16 x i8> %zextd.vec, <16 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>997  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 64998  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias999  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 641000  ret void1001}1002 1003define void @vec128_v8i16_to_v4i32_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {1004; SSE2-LABEL: vec128_v8i16_to_v4i32_factor2:1005; SSE2:       # %bb.0:1006; SSE2-NEXT:    movdqa (%rdi), %xmm01007; SSE2-NEXT:    paddb (%rsi), %xmm01008; SSE2-NEXT:    pxor %xmm1, %xmm11009; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]1010; SSE2-NEXT:    paddb (%rdx), %xmm01011; SSE2-NEXT:    movdqa %xmm0, (%rcx)1012; SSE2-NEXT:    retq1013;1014; SSE42-LABEL: vec128_v8i16_to_v4i32_factor2:1015; SSE42:       # %bb.0:1016; SSE42-NEXT:    movdqa (%rdi), %xmm01017; SSE42-NEXT:    paddb (%rsi), %xmm01018; SSE42-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero1019; SSE42-NEXT:    paddb (%rdx), %xmm01020; SSE42-NEXT:    movdqa %xmm0, (%rcx)1021; SSE42-NEXT:    retq1022;1023; AVX-LABEL: vec128_v8i16_to_v4i32_factor2:1024; AVX:       # %bb.0:1025; AVX-NEXT:    vmovdqa (%rdi), %xmm01026; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm01027; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero1028; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm01029; AVX-NEXT:    vmovdqa %xmm0, (%rcx)1030; AVX-NEXT:    retq1031;1032; AVX2-LABEL: vec128_v8i16_to_v4i32_factor2:1033; AVX2:       # %bb.0:1034; AVX2-NEXT:    vmovdqa (%rdi), %xmm01035; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm01036; AVX2-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero1037; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm01038; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)1039; AVX2-NEXT:    vzeroupper1040; AVX2-NEXT:    retq1041;1042; AVX512F-LABEL: vec128_v8i16_to_v4i32_factor2:1043; AVX512F:       # %bb.0:1044; AVX512F-NEXT:    vmovdqa (%rdi), %xmm01045; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm01046; AVX512F-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero1047; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm01048; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)1049; AVX512F-NEXT:    vzeroupper1050; AVX512F-NEXT:    retq1051;1052; AVX512BW-LABEL: vec128_v8i16_to_v4i32_factor2:1053; AVX512BW:       # %bb.0:1054; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm01055; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm01056; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero1057; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm01058; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)1059; AVX512BW-NEXT:    vzeroupper1060; AVX512BW-NEXT:    retq1061  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 641062  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 641063  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias1064  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>1065  %in.vec.cast = bitcast <16 x i8> %in.vec.trunc to <8 x i16>1066  %zextd.vec = shufflevector <8 x i16> %in.vec.cast, <8 x i16> zeroinitializer, <8 x i32> <i32 0, i32 9, i32 1, i32 11, i32 2, i32 13, i32 3, i32 15>1067  %out.bytevec = bitcast <8 x i16> %zextd.vec to <16 x i8>1068  %out.bytevec.padded = shufflevector <16 x i8> %out.bytevec, <16 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>1069  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 641070  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias1071  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 641072  ret void1073}1074 1075define void @vec128_v8i16_to_v2i64_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {1076; SSE2-LABEL: vec128_v8i16_to_v2i64_factor4:1077; SSE2:       # %bb.0:1078; SSE2-NEXT:    movdqa (%rdi), %xmm01079; SSE2-NEXT:    paddb (%rsi), %xmm01080; SSE2-NEXT:    pxor %xmm1, %xmm11081; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]1082; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]1083; SSE2-NEXT:    paddb (%rdx), %xmm01084; SSE2-NEXT:    movdqa %xmm0, (%rcx)1085; SSE2-NEXT:    retq1086;1087; SSE42-LABEL: vec128_v8i16_to_v2i64_factor4:1088; SSE42:       # %bb.0:1089; SSE42-NEXT:    movdqa (%rdi), %xmm01090; SSE42-NEXT:    paddb (%rsi), %xmm01091; SSE42-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero1092; SSE42-NEXT:    paddb (%rdx), %xmm01093; SSE42-NEXT:    movdqa %xmm0, (%rcx)1094; SSE42-NEXT:    retq1095;1096; AVX-LABEL: vec128_v8i16_to_v2i64_factor4:1097; AVX:       # %bb.0:1098; AVX-NEXT:    vmovdqa (%rdi), %xmm01099; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm01100; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero1101; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm01102; AVX-NEXT:    vmovdqa %xmm0, (%rcx)1103; AVX-NEXT:    retq1104;1105; AVX2-LABEL: vec128_v8i16_to_v2i64_factor4:1106; AVX2:       # %bb.0:1107; AVX2-NEXT:    vmovdqa (%rdi), %xmm01108; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm01109; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero1110; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm01111; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)1112; AVX2-NEXT:    vzeroupper1113; AVX2-NEXT:    retq1114;1115; AVX512F-LABEL: vec128_v8i16_to_v2i64_factor4:1116; AVX512F:       # %bb.0:1117; AVX512F-NEXT:    vmovdqa (%rdi), %xmm01118; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm01119; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero1120; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm01121; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)1122; AVX512F-NEXT:    vzeroupper1123; AVX512F-NEXT:    retq1124;1125; AVX512BW-LABEL: vec128_v8i16_to_v2i64_factor4:1126; AVX512BW:       # %bb.0:1127; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm01128; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm01129; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero1130; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm01131; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)1132; AVX512BW-NEXT:    vzeroupper1133; AVX512BW-NEXT:    retq1134  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 641135  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 641136  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias1137  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>1138  %in.vec.cast = bitcast <16 x i8> %in.vec.trunc to <8 x i16>1139  %zextd.vec = shufflevector <8 x i16> %in.vec.cast, <8 x i16> zeroinitializer, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 1, i32 13, i32 14, i32 15>1140  %out.bytevec = bitcast <8 x i16> %zextd.vec to <16 x i8>1141  %out.bytevec.padded = shufflevector <16 x i8> %out.bytevec, <16 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>1142  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 641143  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias1144  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 641145  ret void1146}1147 1148define void @vec128_v8i16_to_v1i128_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {1149; SSE2-LABEL: vec128_v8i16_to_v1i128_factor8:1150; SSE2:       # %bb.0:1151; SSE2-NEXT:    movdqa (%rdi), %xmm01152; SSE2-NEXT:    paddb (%rsi), %xmm01153; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm01154; SSE2-NEXT:    paddb (%rdx), %xmm01155; SSE2-NEXT:    movdqa %xmm0, (%rcx)1156; SSE2-NEXT:    retq1157;1158; SSE42-LABEL: vec128_v8i16_to_v1i128_factor8:1159; SSE42:       # %bb.0:1160; SSE42-NEXT:    movdqa (%rdi), %xmm01161; SSE42-NEXT:    paddb (%rsi), %xmm01162; SSE42-NEXT:    pxor %xmm1, %xmm11163; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]1164; SSE42-NEXT:    paddb (%rdx), %xmm11165; SSE42-NEXT:    movdqa %xmm1, (%rcx)1166; SSE42-NEXT:    retq1167;1168; AVX-LABEL: vec128_v8i16_to_v1i128_factor8:1169; AVX:       # %bb.0:1170; AVX-NEXT:    vmovdqa (%rdi), %xmm01171; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm01172; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm11173; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]1174; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm01175; AVX-NEXT:    vmovdqa %xmm0, (%rcx)1176; AVX-NEXT:    retq1177;1178; AVX2-LABEL: vec128_v8i16_to_v1i128_factor8:1179; AVX2:       # %bb.0:1180; AVX2-NEXT:    vmovdqa (%rdi), %xmm01181; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm01182; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm11183; AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]1184; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm01185; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)1186; AVX2-NEXT:    vzeroupper1187; AVX2-NEXT:    retq1188;1189; AVX512F-LABEL: vec128_v8i16_to_v1i128_factor8:1190; AVX512F:       # %bb.0:1191; AVX512F-NEXT:    vmovdqa (%rdi), %xmm01192; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm01193; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm11194; AVX512F-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]1195; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm01196; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)1197; AVX512F-NEXT:    vzeroupper1198; AVX512F-NEXT:    retq1199;1200; AVX512BW-LABEL: vec128_v8i16_to_v1i128_factor8:1201; AVX512BW:       # %bb.0:1202; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm01203; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm01204; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm11205; AVX512BW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]1206; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm01207; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)1208; AVX512BW-NEXT:    vzeroupper1209; AVX512BW-NEXT:    retq1210  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 641211  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 641212  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias1213  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>1214  %in.vec.cast = bitcast <16 x i8> %in.vec.trunc to <8 x i16>1215  %zextd.vec = shufflevector <8 x i16> %in.vec.cast, <8 x i16> zeroinitializer, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>1216  %out.bytevec = bitcast <8 x i16> %zextd.vec to <16 x i8>1217  %out.bytevec.padded = shufflevector <16 x i8> %out.bytevec, <16 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>1218  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 641219  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias1220  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 641221  ret void1222}1223 1224define void @vec128_v4i32_to_v2i64_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {1225; SSE2-LABEL: vec128_v4i32_to_v2i64_factor2:1226; SSE2:       # %bb.0:1227; SSE2-NEXT:    movdqa (%rdi), %xmm01228; SSE2-NEXT:    paddb (%rsi), %xmm01229; SSE2-NEXT:    pxor %xmm1, %xmm11230; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]1231; SSE2-NEXT:    paddb (%rdx), %xmm01232; SSE2-NEXT:    movdqa %xmm0, (%rcx)1233; SSE2-NEXT:    retq1234;1235; SSE42-LABEL: vec128_v4i32_to_v2i64_factor2:1236; SSE42:       # %bb.0:1237; SSE42-NEXT:    movdqa (%rdi), %xmm01238; SSE42-NEXT:    paddb (%rsi), %xmm01239; SSE42-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero1240; SSE42-NEXT:    paddb (%rdx), %xmm01241; SSE42-NEXT:    movdqa %xmm0, (%rcx)1242; SSE42-NEXT:    retq1243;1244; AVX-LABEL: vec128_v4i32_to_v2i64_factor2:1245; AVX:       # %bb.0:1246; AVX-NEXT:    vmovdqa (%rdi), %xmm01247; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm01248; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero1249; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm01250; AVX-NEXT:    vmovdqa %xmm0, (%rcx)1251; AVX-NEXT:    retq1252;1253; AVX2-LABEL: vec128_v4i32_to_v2i64_factor2:1254; AVX2:       # %bb.0:1255; AVX2-NEXT:    vmovdqa (%rdi), %xmm01256; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm01257; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero1258; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm01259; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)1260; AVX2-NEXT:    vzeroupper1261; AVX2-NEXT:    retq1262;1263; AVX512F-LABEL: vec128_v4i32_to_v2i64_factor2:1264; AVX512F:       # %bb.0:1265; AVX512F-NEXT:    vmovdqa (%rdi), %xmm01266; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm01267; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero1268; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm01269; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)1270; AVX512F-NEXT:    vzeroupper1271; AVX512F-NEXT:    retq1272;1273; AVX512BW-LABEL: vec128_v4i32_to_v2i64_factor2:1274; AVX512BW:       # %bb.0:1275; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm01276; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm01277; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero1278; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm01279; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)1280; AVX512BW-NEXT:    vzeroupper1281; AVX512BW-NEXT:    retq1282  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 641283  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 641284  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias1285  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>1286  %in.vec.cast = bitcast <16 x i8> %in.vec.trunc to <4 x i32>1287  %zextd.vec = shufflevector <4 x i32> %in.vec.cast, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 1, i32 7>1288  %out.bytevec = bitcast <4 x i32> %zextd.vec to <16 x i8>1289  %out.bytevec.padded = shufflevector <16 x i8> %out.bytevec, <16 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>1290  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 641291  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias1292  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 641293  ret void1294}1295 1296define void @vec128_v4i32_to_v1i128_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {1297; SSE2-LABEL: vec128_v4i32_to_v1i128_factor4:1298; SSE2:       # %bb.0:1299; SSE2-NEXT:    movdqa (%rdi), %xmm01300; SSE2-NEXT:    paddb (%rsi), %xmm01301; SSE2-NEXT:    xorps %xmm1, %xmm11302; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]1303; SSE2-NEXT:    paddb (%rdx), %xmm11304; SSE2-NEXT:    movdqa %xmm1, (%rcx)1305; SSE2-NEXT:    retq1306;1307; SSE42-LABEL: vec128_v4i32_to_v1i128_factor4:1308; SSE42:       # %bb.0:1309; SSE42-NEXT:    movdqa (%rdi), %xmm01310; SSE42-NEXT:    paddb (%rsi), %xmm01311; SSE42-NEXT:    pxor %xmm1, %xmm11312; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7]1313; SSE42-NEXT:    paddb (%rdx), %xmm11314; SSE42-NEXT:    movdqa %xmm1, (%rcx)1315; SSE42-NEXT:    retq1316;1317; AVX-LABEL: vec128_v4i32_to_v1i128_factor4:1318; AVX:       # %bb.0:1319; AVX-NEXT:    vmovdqa (%rdi), %xmm01320; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm01321; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm11322; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]1323; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm01324; AVX-NEXT:    vmovdqa %xmm0, (%rcx)1325; AVX-NEXT:    retq1326;1327; AVX2-LABEL: vec128_v4i32_to_v1i128_factor4:1328; AVX2:       # %bb.0:1329; AVX2-NEXT:    vmovdqa (%rdi), %xmm01330; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm01331; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm11332; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]1333; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm01334; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)1335; AVX2-NEXT:    vzeroupper1336; AVX2-NEXT:    retq1337;1338; AVX512F-LABEL: vec128_v4i32_to_v1i128_factor4:1339; AVX512F:       # %bb.0:1340; AVX512F-NEXT:    vmovdqa (%rdi), %xmm01341; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm01342; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm11343; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]1344; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm01345; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)1346; AVX512F-NEXT:    vzeroupper1347; AVX512F-NEXT:    retq1348;1349; AVX512BW-LABEL: vec128_v4i32_to_v1i128_factor4:1350; AVX512BW:       # %bb.0:1351; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm01352; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm01353; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm11354; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]1355; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm01356; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)1357; AVX512BW-NEXT:    vzeroupper1358; AVX512BW-NEXT:    retq1359  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 641360  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 641361  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias1362  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>1363  %in.vec.cast = bitcast <16 x i8> %in.vec.trunc to <4 x i32>1364  %zextd.vec = shufflevector <4 x i32> %in.vec.cast, <4 x i32> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 6, i32 7>1365  %out.bytevec = bitcast <4 x i32> %zextd.vec to <16 x i8>1366  %out.bytevec.padded = shufflevector <16 x i8> %out.bytevec, <16 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>1367  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 641368  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias1369  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 641370  ret void1371}1372 1373define void @vec128_v2i64_to_v1i128_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {1374; SSE-LABEL: vec128_v2i64_to_v1i128_factor2:1375; SSE:       # %bb.0:1376; SSE-NEXT:    movdqa (%rdi), %xmm01377; SSE-NEXT:    paddb (%rsi), %xmm01378; SSE-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero1379; SSE-NEXT:    paddb (%rdx), %xmm01380; SSE-NEXT:    movdqa %xmm0, (%rcx)1381; SSE-NEXT:    retq1382;1383; AVX-LABEL: vec128_v2i64_to_v1i128_factor2:1384; AVX:       # %bb.0:1385; AVX-NEXT:    vmovdqa (%rdi), %xmm01386; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm01387; AVX-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero1388; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm01389; AVX-NEXT:    vmovdqa %xmm0, (%rcx)1390; AVX-NEXT:    retq1391;1392; AVX2-LABEL: vec128_v2i64_to_v1i128_factor2:1393; AVX2:       # %bb.0:1394; AVX2-NEXT:    vmovdqa (%rdi), %xmm01395; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm01396; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero1397; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm01398; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)1399; AVX2-NEXT:    vzeroupper1400; AVX2-NEXT:    retq1401;1402; AVX512F-LABEL: vec128_v2i64_to_v1i128_factor2:1403; AVX512F:       # %bb.0:1404; AVX512F-NEXT:    vmovdqa (%rdi), %xmm01405; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm01406; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero1407; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm01408; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)1409; AVX512F-NEXT:    vzeroupper1410; AVX512F-NEXT:    retq1411;1412; AVX512BW-LABEL: vec128_v2i64_to_v1i128_factor2:1413; AVX512BW:       # %bb.0:1414; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm01415; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm01416; AVX512BW-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero1417; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm01418; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)1419; AVX512BW-NEXT:    vzeroupper1420; AVX512BW-NEXT:    retq1421  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 641422  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 641423  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias1424  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>1425  %in.vec.cast = bitcast <16 x i8> %in.vec.trunc to <2 x i64>1426  %zextd.vec = shufflevector <2 x i64> %in.vec.cast, <2 x i64> zeroinitializer, <2 x i32> <i32 0, i32 3>1427  %out.bytevec = bitcast <2 x i64> %zextd.vec to <16 x i8>1428  %out.bytevec.padded = shufflevector <16 x i8> %out.bytevec, <16 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>1429  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 641430  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias1431  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 641432  ret void1433}1434 1435define void @vec256_v32i8_to_v16i16_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {1436; SSE2-LABEL: vec256_v32i8_to_v16i16_factor2:1437; SSE2:       # %bb.0:1438; SSE2-NEXT:    movdqa (%rdi), %xmm01439; SSE2-NEXT:    paddb (%rsi), %xmm01440; SSE2-NEXT:    pxor %xmm1, %xmm11441; SSE2-NEXT:    movdqa %xmm0, %xmm21442; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]1443; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]1444; SSE2-NEXT:    paddb 16(%rdx), %xmm01445; SSE2-NEXT:    paddb (%rdx), %xmm21446; SSE2-NEXT:    movdqa %xmm2, (%rcx)1447; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)1448; SSE2-NEXT:    retq1449;1450; SSE42-LABEL: vec256_v32i8_to_v16i16_factor2:1451; SSE42:       # %bb.0:1452; SSE42-NEXT:    movdqa (%rdi), %xmm01453; SSE42-NEXT:    paddb (%rsi), %xmm01454; SSE42-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1455; SSE42-NEXT:    pxor %xmm2, %xmm21456; SSE42-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]1457; SSE42-NEXT:    paddb 16(%rdx), %xmm01458; SSE42-NEXT:    paddb (%rdx), %xmm11459; SSE42-NEXT:    movdqa %xmm1, (%rcx)1460; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)1461; SSE42-NEXT:    retq1462;1463; AVX-LABEL: vec256_v32i8_to_v16i16_factor2:1464; AVX:       # %bb.0:1465; AVX-NEXT:    vmovdqa (%rdi), %xmm01466; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm01467; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1468; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm21469; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]1470; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm01471; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm11472; AVX-NEXT:    vmovdqa %xmm1, (%rcx)1473; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)1474; AVX-NEXT:    retq1475;1476; AVX2-LABEL: vec256_v32i8_to_v16i16_factor2:1477; AVX2:       # %bb.0:1478; AVX2-NEXT:    vmovdqa (%rdi), %xmm01479; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm01480; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero1481; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm01482; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)1483; AVX2-NEXT:    vzeroupper1484; AVX2-NEXT:    retq1485;1486; AVX512F-LABEL: vec256_v32i8_to_v16i16_factor2:1487; AVX512F:       # %bb.0:1488; AVX512F-NEXT:    vmovdqa (%rdi), %xmm01489; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm01490; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero1491; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm01492; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)1493; AVX512F-NEXT:    vzeroupper1494; AVX512F-NEXT:    retq1495;1496; AVX512BW-LABEL: vec256_v32i8_to_v16i16_factor2:1497; AVX512BW:       # %bb.0:1498; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm01499; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm01500; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero1501; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm01502; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)1503; AVX512BW-NEXT:    vzeroupper1504; AVX512BW-NEXT:    retq1505  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 641506  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 641507  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias1508  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>1509  %zextd.vec = shufflevector <32 x i8> %in.vec.trunc, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 33, i32 1, i32 35, i32 2, i32 37, i32 3, i32 39, i32 4, i32 41, i32 5, i32 43, i32 6, i32 45, i32 7, i32 47, i32 8, i32 49, i32 9, i32 51, i32 10, i32 53, i32 11, i32 55, i32 12, i32 57, i32 13, i32 59, i32 14, i32 61, i32 15, i32 63>1510  %out.bytevec.padded = shufflevector <32 x i8> %zextd.vec, <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>1511  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 641512  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias1513  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 641514  ret void1515}1516 1517define void @vec256_v32i8_to_v8i32_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {1518; SSE2-LABEL: vec256_v32i8_to_v8i32_factor4:1519; SSE2:       # %bb.0:1520; SSE2-NEXT:    movdqa (%rdi), %xmm01521; SSE2-NEXT:    paddb (%rsi), %xmm01522; SSE2-NEXT:    pxor %xmm1, %xmm11523; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]1524; SSE2-NEXT:    movdqa %xmm0, %xmm21525; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]1526; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]1527; SSE2-NEXT:    paddb 16(%rdx), %xmm01528; SSE2-NEXT:    paddb (%rdx), %xmm21529; SSE2-NEXT:    movdqa %xmm2, (%rcx)1530; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)1531; SSE2-NEXT:    retq1532;1533; SSE42-LABEL: vec256_v32i8_to_v8i32_factor4:1534; SSE42:       # %bb.0:1535; SSE42-NEXT:    movdqa (%rdi), %xmm01536; SSE42-NEXT:    paddb (%rsi), %xmm01537; SSE42-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero1538; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]1539; SSE42-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero1540; SSE42-NEXT:    paddb 16(%rdx), %xmm01541; SSE42-NEXT:    paddb (%rdx), %xmm11542; SSE42-NEXT:    movdqa %xmm1, (%rcx)1543; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)1544; SSE42-NEXT:    retq1545;1546; AVX-LABEL: vec256_v32i8_to_v8i32_factor4:1547; AVX:       # %bb.0:1548; AVX-NEXT:    vmovdqa (%rdi), %xmm01549; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm01550; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero1551; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]1552; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero1553; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm01554; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm11555; AVX-NEXT:    vmovdqa %xmm1, (%rcx)1556; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)1557; AVX-NEXT:    retq1558;1559; AVX2-LABEL: vec256_v32i8_to_v8i32_factor4:1560; AVX2:       # %bb.0:1561; AVX2-NEXT:    vmovdqa (%rdi), %xmm01562; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm01563; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero1564; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm01565; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)1566; AVX2-NEXT:    vzeroupper1567; AVX2-NEXT:    retq1568;1569; AVX512F-LABEL: vec256_v32i8_to_v8i32_factor4:1570; AVX512F:       # %bb.0:1571; AVX512F-NEXT:    vmovdqa (%rdi), %xmm01572; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm01573; AVX512F-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero1574; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm01575; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)1576; AVX512F-NEXT:    vzeroupper1577; AVX512F-NEXT:    retq1578;1579; AVX512BW-LABEL: vec256_v32i8_to_v8i32_factor4:1580; AVX512BW:       # %bb.0:1581; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm01582; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm01583; AVX512BW-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero1584; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm01585; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)1586; AVX512BW-NEXT:    vzeroupper1587; AVX512BW-NEXT:    retq1588  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 641589  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 641590  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias1591  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>1592  %zextd.vec = shufflevector <32 x i8> %in.vec.trunc, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 33, i32 34, i32 35, i32 1, i32 37, i32 38, i32 39, i32 2, i32 41, i32 42, i32 43, i32 3, i32 45, i32 46, i32 47, i32 4, i32 49, i32 50, i32 51, i32 5, i32 53, i32 54, i32 55, i32 6, i32 57, i32 58, i32 59, i32 7, i32 61, i32 62, i32 63>1593  %out.bytevec.padded = shufflevector <32 x i8> %zextd.vec, <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>1594  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 641595  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias1596  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 641597  ret void1598}1599 1600define void @vec256_v32i8_to_v4i64_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {1601; SSE2-LABEL: vec256_v32i8_to_v4i64_factor8:1602; SSE2:       # %bb.0:1603; SSE2-NEXT:    movdqa (%rdi), %xmm01604; SSE2-NEXT:    paddb (%rsi), %xmm01605; SSE2-NEXT:    pxor %xmm1, %xmm11606; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]1607; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]1608; SSE2-NEXT:    movdqa %xmm0, %xmm21609; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]1610; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]1611; SSE2-NEXT:    paddb 16(%rdx), %xmm01612; SSE2-NEXT:    paddb (%rdx), %xmm21613; SSE2-NEXT:    movdqa %xmm2, (%rcx)1614; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)1615; SSE2-NEXT:    retq1616;1617; SSE42-LABEL: vec256_v32i8_to_v4i64_factor8:1618; SSE42:       # %bb.0:1619; SSE42-NEXT:    movdqa (%rdi), %xmm01620; SSE42-NEXT:    paddb (%rsi), %xmm01621; SSE42-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero1622; SSE42-NEXT:    psrld $16, %xmm01623; SSE42-NEXT:    pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero1624; SSE42-NEXT:    paddb 16(%rdx), %xmm01625; SSE42-NEXT:    paddb (%rdx), %xmm11626; SSE42-NEXT:    movdqa %xmm1, (%rcx)1627; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)1628; SSE42-NEXT:    retq1629;1630; AVX-LABEL: vec256_v32i8_to_v4i64_factor8:1631; AVX:       # %bb.0:1632; AVX-NEXT:    vmovdqa (%rdi), %xmm01633; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm01634; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero1635; AVX-NEXT:    vpsrld $16, %xmm0, %xmm01636; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero1637; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm01638; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm11639; AVX-NEXT:    vmovdqa %xmm1, (%rcx)1640; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)1641; AVX-NEXT:    retq1642;1643; AVX2-LABEL: vec256_v32i8_to_v4i64_factor8:1644; AVX2:       # %bb.0:1645; AVX2-NEXT:    vmovdqa (%rdi), %xmm01646; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm01647; AVX2-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero1648; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm01649; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)1650; AVX2-NEXT:    vzeroupper1651; AVX2-NEXT:    retq1652;1653; AVX512F-LABEL: vec256_v32i8_to_v4i64_factor8:1654; AVX512F:       # %bb.0:1655; AVX512F-NEXT:    vmovdqa (%rdi), %xmm01656; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm01657; AVX512F-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero1658; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm01659; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)1660; AVX512F-NEXT:    vzeroupper1661; AVX512F-NEXT:    retq1662;1663; AVX512BW-LABEL: vec256_v32i8_to_v4i64_factor8:1664; AVX512BW:       # %bb.0:1665; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm01666; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm01667; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero1668; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm01669; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)1670; AVX512BW-NEXT:    vzeroupper1671; AVX512BW-NEXT:    retq1672  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 641673  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 641674  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias1675  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>1676  %zextd.vec = shufflevector <32 x i8> %in.vec.trunc, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 1, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 2, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 3, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>1677  %out.bytevec.padded = shufflevector <32 x i8> %zextd.vec, <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>1678  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 641679  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias1680  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 641681  ret void1682}1683 1684define void @vec256_v32i8_to_v2i128_factor16(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {1685; SSE2-LABEL: vec256_v32i8_to_v2i128_factor16:1686; SSE2:       # %bb.0:1687; SSE2-NEXT:    movdqa (%rdi), %xmm01688; SSE2-NEXT:    paddb (%rsi), %xmm01689; SSE2-NEXT:    movd {{.*#+}} xmm1 = [255,0,0,0]1690; SSE2-NEXT:    pand %xmm0, %xmm11691; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1]1692; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero1693; SSE2-NEXT:    paddb 16(%rdx), %xmm01694; SSE2-NEXT:    paddb (%rdx), %xmm11695; SSE2-NEXT:    movdqa %xmm1, (%rcx)1696; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)1697; SSE2-NEXT:    retq1698;1699; SSE42-LABEL: vec256_v32i8_to_v2i128_factor16:1700; SSE42:       # %bb.0:1701; SSE42-NEXT:    movdqa (%rdi), %xmm01702; SSE42-NEXT:    paddb (%rsi), %xmm01703; SSE42-NEXT:    pmovzxbq {{.*#+}} xmm1 = [255,0]1704; SSE42-NEXT:    pand %xmm0, %xmm11705; SSE42-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1]1706; SSE42-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero1707; SSE42-NEXT:    paddb 16(%rdx), %xmm01708; SSE42-NEXT:    paddb (%rdx), %xmm11709; SSE42-NEXT:    movdqa %xmm1, (%rcx)1710; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)1711; SSE42-NEXT:    retq1712;1713; AVX-LABEL: vec256_v32i8_to_v2i128_factor16:1714; AVX:       # %bb.0:1715; AVX-NEXT:    vmovdqa (%rdi), %xmm01716; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm01717; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm11718; AVX-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1]1719; AVX-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero1720; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm01721; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm11722; AVX-NEXT:    vmovdqa %xmm1, (%rcx)1723; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)1724; AVX-NEXT:    retq1725;1726; AVX2-LABEL: vec256_v32i8_to_v2i128_factor16:1727; AVX2:       # %bb.0:1728; AVX2-NEXT:    vmovdqa (%rdi), %xmm01729; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm01730; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero1731; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]1732; AVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01733; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm01734; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)1735; AVX2-NEXT:    vzeroupper1736; AVX2-NEXT:    retq1737;1738; AVX512F-LABEL: vec256_v32i8_to_v2i128_factor16:1739; AVX512F:       # %bb.0:1740; AVX512F-NEXT:    vmovdqa (%rdi), %xmm01741; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm01742; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero1743; AVX512F-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]1744; AVX512F-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01745; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm01746; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)1747; AVX512F-NEXT:    vzeroupper1748; AVX512F-NEXT:    retq1749;1750; AVX512BW-LABEL: vec256_v32i8_to_v2i128_factor16:1751; AVX512BW:       # %bb.0:1752; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm01753; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm01754; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero1755; AVX512BW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]1756; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01757; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm01758; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)1759; AVX512BW-NEXT:    vzeroupper1760; AVX512BW-NEXT:    retq1761  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 641762  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 641763  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias1764  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>1765  %zextd.vec = shufflevector <32 x i8> %in.vec.trunc, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 1, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>1766  %out.bytevec.padded = shufflevector <32 x i8> %zextd.vec, <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>1767  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 641768  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias1769  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 641770  ret void1771}1772 1773define void @vec256_v32i8_to_v1i256_factor32(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {1774; SSE-LABEL: vec256_v32i8_to_v1i256_factor32:1775; SSE:       # %bb.0:1776; SSE-NEXT:    movdqa (%rdi), %xmm01777; SSE-NEXT:    paddb (%rsi), %xmm01778; SSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm01779; SSE-NEXT:    movaps 16(%rdx), %xmm11780; SSE-NEXT:    paddb (%rdx), %xmm01781; SSE-NEXT:    movaps %xmm1, 16(%rcx)1782; SSE-NEXT:    movdqa %xmm0, (%rcx)1783; SSE-NEXT:    retq1784;1785; AVX-LABEL: vec256_v32i8_to_v1i256_factor32:1786; AVX:       # %bb.0:1787; AVX-NEXT:    vmovdqa (%rdi), %xmm01788; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm01789; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm01790; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm01791; AVX-NEXT:    vmovaps 16(%rdx), %xmm11792; AVX-NEXT:    vmovaps %xmm1, 16(%rcx)1793; AVX-NEXT:    vmovdqa %xmm0, (%rcx)1794; AVX-NEXT:    retq1795;1796; AVX2-LABEL: vec256_v32i8_to_v1i256_factor32:1797; AVX2:       # %bb.0:1798; AVX2-NEXT:    vmovdqa (%rdi), %ymm01799; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm01800; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = [255,0]1801; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm01802; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm01803; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)1804; AVX2-NEXT:    vzeroupper1805; AVX2-NEXT:    retq1806;1807; AVX512F-LABEL: vec256_v32i8_to_v1i256_factor32:1808; AVX512F:       # %bb.0:1809; AVX512F-NEXT:    vmovdqa (%rdi), %ymm01810; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm01811; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm1 = [255,0]1812; AVX512F-NEXT:    vpand %ymm1, %ymm0, %ymm01813; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm01814; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)1815; AVX512F-NEXT:    vzeroupper1816; AVX512F-NEXT:    retq1817;1818; AVX512BW-LABEL: vec256_v32i8_to_v1i256_factor32:1819; AVX512BW:       # %bb.0:1820; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm01821; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm01822; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = [255,0]1823; AVX512BW-NEXT:    vpand %ymm1, %ymm0, %ymm01824; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm01825; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)1826; AVX512BW-NEXT:    vzeroupper1827; AVX512BW-NEXT:    retq1828  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 641829  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 641830  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias1831  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>1832  %zextd.vec = shufflevector <32 x i8> %in.vec.trunc, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>1833  %out.bytevec.padded = shufflevector <32 x i8> %zextd.vec, <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>1834  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 641835  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias1836  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 641837  ret void1838}1839 1840define void @vec256_v16i16_to_v8i32_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {1841; SSE2-LABEL: vec256_v16i16_to_v8i32_factor2:1842; SSE2:       # %bb.0:1843; SSE2-NEXT:    movdqa (%rdi), %xmm01844; SSE2-NEXT:    paddb (%rsi), %xmm01845; SSE2-NEXT:    pxor %xmm1, %xmm11846; SSE2-NEXT:    movdqa %xmm0, %xmm21847; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]1848; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]1849; SSE2-NEXT:    paddb 16(%rdx), %xmm01850; SSE2-NEXT:    paddb (%rdx), %xmm21851; SSE2-NEXT:    movdqa %xmm2, (%rcx)1852; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)1853; SSE2-NEXT:    retq1854;1855; SSE42-LABEL: vec256_v16i16_to_v8i32_factor2:1856; SSE42:       # %bb.0:1857; SSE42-NEXT:    movdqa (%rdi), %xmm01858; SSE42-NEXT:    paddb (%rsi), %xmm01859; SSE42-NEXT:    pxor %xmm1, %xmm11860; SSE42-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero1861; SSE42-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]1862; SSE42-NEXT:    paddb 16(%rdx), %xmm01863; SSE42-NEXT:    paddb (%rdx), %xmm21864; SSE42-NEXT:    movdqa %xmm2, (%rcx)1865; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)1866; SSE42-NEXT:    retq1867;1868; AVX-LABEL: vec256_v16i16_to_v8i32_factor2:1869; AVX:       # %bb.0:1870; AVX-NEXT:    vmovdqa (%rdi), %xmm01871; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm01872; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero1873; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm21874; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]1875; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm01876; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm11877; AVX-NEXT:    vmovdqa %xmm1, (%rcx)1878; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)1879; AVX-NEXT:    retq1880;1881; AVX2-LABEL: vec256_v16i16_to_v8i32_factor2:1882; AVX2:       # %bb.0:1883; AVX2-NEXT:    vmovdqa (%rdi), %xmm01884; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm01885; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1886; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm01887; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)1888; AVX2-NEXT:    vzeroupper1889; AVX2-NEXT:    retq1890;1891; AVX512F-LABEL: vec256_v16i16_to_v8i32_factor2:1892; AVX512F:       # %bb.0:1893; AVX512F-NEXT:    vmovdqa (%rdi), %xmm01894; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm01895; AVX512F-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1896; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm01897; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)1898; AVX512F-NEXT:    vzeroupper1899; AVX512F-NEXT:    retq1900;1901; AVX512BW-LABEL: vec256_v16i16_to_v8i32_factor2:1902; AVX512BW:       # %bb.0:1903; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm01904; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm01905; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1906; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm01907; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)1908; AVX512BW-NEXT:    vzeroupper1909; AVX512BW-NEXT:    retq1910  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 641911  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 641912  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias1913  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>1914  %in.vec.cast = bitcast <32 x i8> %in.vec.trunc to <16 x i16>1915  %zextd.vec = shufflevector <16 x i16> %in.vec.cast, <16 x i16> zeroinitializer, <16 x i32> <i32 0, i32 17, i32 1, i32 19, i32 2, i32 21, i32 3, i32 23, i32 4, i32 25, i32 5, i32 27, i32 6, i32 29, i32 7, i32 31>1916  %out.bytevec = bitcast <16 x i16> %zextd.vec to <32 x i8>1917  %out.bytevec.padded = shufflevector <32 x i8> %out.bytevec, <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>1918  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 641919  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias1920  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 641921  ret void1922}1923 1924define void @vec256_v16i16_to_v4i64_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {1925; SSE2-LABEL: vec256_v16i16_to_v4i64_factor4:1926; SSE2:       # %bb.0:1927; SSE2-NEXT:    movdqa (%rdi), %xmm01928; SSE2-NEXT:    paddb (%rsi), %xmm01929; SSE2-NEXT:    pxor %xmm1, %xmm11930; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]1931; SSE2-NEXT:    movdqa %xmm0, %xmm21932; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]1933; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]1934; SSE2-NEXT:    paddb 16(%rdx), %xmm01935; SSE2-NEXT:    paddb (%rdx), %xmm21936; SSE2-NEXT:    movdqa %xmm2, (%rcx)1937; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)1938; SSE2-NEXT:    retq1939;1940; SSE42-LABEL: vec256_v16i16_to_v4i64_factor4:1941; SSE42:       # %bb.0:1942; SSE42-NEXT:    movdqa (%rdi), %xmm01943; SSE42-NEXT:    paddb (%rsi), %xmm01944; SSE42-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero1945; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]1946; SSE42-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero1947; SSE42-NEXT:    paddb 16(%rdx), %xmm01948; SSE42-NEXT:    paddb (%rdx), %xmm11949; SSE42-NEXT:    movdqa %xmm1, (%rcx)1950; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)1951; SSE42-NEXT:    retq1952;1953; AVX-LABEL: vec256_v16i16_to_v4i64_factor4:1954; AVX:       # %bb.0:1955; AVX-NEXT:    vmovdqa (%rdi), %xmm01956; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm01957; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero1958; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]1959; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero1960; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm01961; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm11962; AVX-NEXT:    vmovdqa %xmm1, (%rcx)1963; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)1964; AVX-NEXT:    retq1965;1966; AVX2-LABEL: vec256_v16i16_to_v4i64_factor4:1967; AVX2:       # %bb.0:1968; AVX2-NEXT:    vmovdqa (%rdi), %xmm01969; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm01970; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero1971; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm01972; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)1973; AVX2-NEXT:    vzeroupper1974; AVX2-NEXT:    retq1975;1976; AVX512F-LABEL: vec256_v16i16_to_v4i64_factor4:1977; AVX512F:       # %bb.0:1978; AVX512F-NEXT:    vmovdqa (%rdi), %xmm01979; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm01980; AVX512F-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero1981; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm01982; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)1983; AVX512F-NEXT:    vzeroupper1984; AVX512F-NEXT:    retq1985;1986; AVX512BW-LABEL: vec256_v16i16_to_v4i64_factor4:1987; AVX512BW:       # %bb.0:1988; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm01989; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm01990; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero1991; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm01992; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)1993; AVX512BW-NEXT:    vzeroupper1994; AVX512BW-NEXT:    retq1995  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 641996  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 641997  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias1998  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>1999  %in.vec.cast = bitcast <32 x i8> %in.vec.trunc to <16 x i16>2000  %zextd.vec = shufflevector <16 x i16> %in.vec.cast, <16 x i16> zeroinitializer, <16 x i32> <i32 0, i32 17, i32 18, i32 19, i32 1, i32 21, i32 22, i32 23, i32 2, i32 25, i32 26, i32 27, i32 3, i32 29, i32 30, i32 31>2001  %out.bytevec = bitcast <16 x i16> %zextd.vec to <32 x i8>2002  %out.bytevec.padded = shufflevector <32 x i8> %out.bytevec, <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>2003  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 642004  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias2005  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 642006  ret void2007}2008 2009define void @vec256_v16i16_to_v2i128_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {2010; SSE2-LABEL: vec256_v16i16_to_v2i128_factor8:2011; SSE2:       # %bb.0:2012; SSE2-NEXT:    movdqa (%rdi), %xmm02013; SSE2-NEXT:    paddb (%rsi), %xmm02014; SSE2-NEXT:    movd {{.*#+}} xmm1 = [65535,0,0,0]2015; SSE2-NEXT:    pand %xmm0, %xmm12016; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]2017; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero2018; SSE2-NEXT:    paddb 16(%rdx), %xmm02019; SSE2-NEXT:    paddb (%rdx), %xmm12020; SSE2-NEXT:    movdqa %xmm1, (%rcx)2021; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)2022; SSE2-NEXT:    retq2023;2024; SSE42-LABEL: vec256_v16i16_to_v2i128_factor8:2025; SSE42:       # %bb.0:2026; SSE42-NEXT:    movdqa (%rdi), %xmm02027; SSE42-NEXT:    paddb (%rsi), %xmm02028; SSE42-NEXT:    pxor %xmm1, %xmm12029; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]2030; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]2031; SSE42-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero2032; SSE42-NEXT:    paddb 16(%rdx), %xmm02033; SSE42-NEXT:    paddb (%rdx), %xmm12034; SSE42-NEXT:    movdqa %xmm1, (%rcx)2035; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)2036; SSE42-NEXT:    retq2037;2038; AVX-LABEL: vec256_v16i16_to_v2i128_factor8:2039; AVX:       # %bb.0:2040; AVX-NEXT:    vmovdqa (%rdi), %xmm02041; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm02042; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm12043; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]2044; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]2045; AVX-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero2046; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm02047; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm12048; AVX-NEXT:    vmovdqa %xmm1, (%rcx)2049; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)2050; AVX-NEXT:    retq2051;2052; AVX2-LABEL: vec256_v16i16_to_v2i128_factor8:2053; AVX2:       # %bb.0:2054; AVX2-NEXT:    vmovdqa (%rdi), %xmm02055; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm02056; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero2057; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]2058; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm12059; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3,4,5,6,7],ymm0[8],ymm1[9,10,11,12,13,14,15]2060; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm02061; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)2062; AVX2-NEXT:    vzeroupper2063; AVX2-NEXT:    retq2064;2065; AVX512F-LABEL: vec256_v16i16_to_v2i128_factor8:2066; AVX512F:       # %bb.0:2067; AVX512F-NEXT:    vmovdqa (%rdi), %xmm02068; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm02069; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero2070; AVX512F-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]2071; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm12072; AVX512F-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3,4,5,6,7],ymm0[8],ymm1[9,10,11,12,13,14,15]2073; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm02074; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)2075; AVX512F-NEXT:    vzeroupper2076; AVX512F-NEXT:    retq2077;2078; AVX512BW-LABEL: vec256_v16i16_to_v2i128_factor8:2079; AVX512BW:       # %bb.0:2080; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm02081; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm02082; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [16,1,2,3,4,5,6,7,17,9,10,11,12,13,14,15]2083; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm22084; AVX512BW-NEXT:    vpermt2w %ymm0, %ymm1, %ymm22085; AVX512BW-NEXT:    vpaddb (%rdx), %zmm2, %zmm02086; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)2087; AVX512BW-NEXT:    vzeroupper2088; AVX512BW-NEXT:    retq2089  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 642090  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 642091  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias2092  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>2093  %in.vec.cast = bitcast <32 x i8> %in.vec.trunc to <16 x i16>2094  %zextd.vec = shufflevector <16 x i16> %in.vec.cast, <16 x i16> zeroinitializer, <16 x i32> <i32 0, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 1, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>2095  %out.bytevec = bitcast <16 x i16> %zextd.vec to <32 x i8>2096  %out.bytevec.padded = shufflevector <32 x i8> %out.bytevec, <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>2097  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 642098  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias2099  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 642100  ret void2101}2102 2103define void @vec256_v16i16_to_v1i256_factor16(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {2104; SSE2-LABEL: vec256_v16i16_to_v1i256_factor16:2105; SSE2:       # %bb.0:2106; SSE2-NEXT:    movdqa (%rdi), %xmm02107; SSE2-NEXT:    paddb (%rsi), %xmm02108; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm02109; SSE2-NEXT:    movaps 16(%rdx), %xmm12110; SSE2-NEXT:    paddb (%rdx), %xmm02111; SSE2-NEXT:    movaps %xmm1, 16(%rcx)2112; SSE2-NEXT:    movdqa %xmm0, (%rcx)2113; SSE2-NEXT:    retq2114;2115; SSE42-LABEL: vec256_v16i16_to_v1i256_factor16:2116; SSE42:       # %bb.0:2117; SSE42-NEXT:    movdqa (%rdi), %xmm02118; SSE42-NEXT:    paddb (%rsi), %xmm02119; SSE42-NEXT:    pxor %xmm1, %xmm12120; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]2121; SSE42-NEXT:    movaps 16(%rdx), %xmm02122; SSE42-NEXT:    paddb (%rdx), %xmm12123; SSE42-NEXT:    movaps %xmm0, 16(%rcx)2124; SSE42-NEXT:    movdqa %xmm1, (%rcx)2125; SSE42-NEXT:    retq2126;2127; AVX-LABEL: vec256_v16i16_to_v1i256_factor16:2128; AVX:       # %bb.0:2129; AVX-NEXT:    vmovdqa (%rdi), %xmm02130; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm02131; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm12132; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]2133; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm02134; AVX-NEXT:    vmovaps 16(%rdx), %xmm12135; AVX-NEXT:    vmovaps %xmm1, 16(%rcx)2136; AVX-NEXT:    vmovdqa %xmm0, (%rcx)2137; AVX-NEXT:    retq2138;2139; AVX2-LABEL: vec256_v16i16_to_v1i256_factor16:2140; AVX2:       # %bb.0:2141; AVX2-NEXT:    vmovdqa (%rdi), %ymm02142; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm02143; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = [65535,0,0,0]2144; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm02145; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm02146; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)2147; AVX2-NEXT:    vzeroupper2148; AVX2-NEXT:    retq2149;2150; AVX512F-LABEL: vec256_v16i16_to_v1i256_factor16:2151; AVX512F:       # %bb.0:2152; AVX512F-NEXT:    vmovdqa (%rdi), %ymm02153; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm02154; AVX512F-NEXT:    vmovd {{.*#+}} xmm1 = [65535,0,0,0]2155; AVX512F-NEXT:    vpand %ymm1, %ymm0, %ymm02156; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm02157; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)2158; AVX512F-NEXT:    vzeroupper2159; AVX512F-NEXT:    retq2160;2161; AVX512BW-LABEL: vec256_v16i16_to_v1i256_factor16:2162; AVX512BW:       # %bb.0:2163; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm02164; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm02165; AVX512BW-NEXT:    vmovd {{.*#+}} xmm1 = [65535,0,0,0]2166; AVX512BW-NEXT:    vpand %ymm1, %ymm0, %ymm02167; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm02168; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)2169; AVX512BW-NEXT:    vzeroupper2170; AVX512BW-NEXT:    retq2171  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 642172  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 642173  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias2174  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>2175  %in.vec.cast = bitcast <32 x i8> %in.vec.trunc to <16 x i16>2176  %zextd.vec = shufflevector <16 x i16> %in.vec.cast, <16 x i16> zeroinitializer, <16 x i32> <i32 0, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>2177  %out.bytevec = bitcast <16 x i16> %zextd.vec to <32 x i8>2178  %out.bytevec.padded = shufflevector <32 x i8> %out.bytevec, <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>2179  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 642180  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias2181  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 642182  ret void2183}2184 2185define void @vec256_v8i32_to_v4i64_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {2186; SSE2-LABEL: vec256_v8i32_to_v4i64_factor2:2187; SSE2:       # %bb.0:2188; SSE2-NEXT:    movdqa (%rdi), %xmm02189; SSE2-NEXT:    paddb (%rsi), %xmm02190; SSE2-NEXT:    pxor %xmm1, %xmm12191; SSE2-NEXT:    movdqa %xmm0, %xmm22192; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]2193; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]2194; SSE2-NEXT:    paddb 16(%rdx), %xmm02195; SSE2-NEXT:    paddb (%rdx), %xmm22196; SSE2-NEXT:    movdqa %xmm2, (%rcx)2197; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)2198; SSE2-NEXT:    retq2199;2200; SSE42-LABEL: vec256_v8i32_to_v4i64_factor2:2201; SSE42:       # %bb.0:2202; SSE42-NEXT:    movdqa (%rdi), %xmm02203; SSE42-NEXT:    paddb (%rsi), %xmm02204; SSE42-NEXT:    pxor %xmm1, %xmm12205; SSE42-NEXT:    pmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero2206; SSE42-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]2207; SSE42-NEXT:    paddb 16(%rdx), %xmm02208; SSE42-NEXT:    paddb (%rdx), %xmm22209; SSE42-NEXT:    movdqa %xmm2, (%rcx)2210; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)2211; SSE42-NEXT:    retq2212;2213; AVX-LABEL: vec256_v8i32_to_v4i64_factor2:2214; AVX:       # %bb.0:2215; AVX-NEXT:    vmovdqa (%rdi), %xmm02216; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm02217; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero2218; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm22219; AVX-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]2220; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm02221; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm12222; AVX-NEXT:    vmovdqa %xmm1, (%rcx)2223; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)2224; AVX-NEXT:    retq2225;2226; AVX2-LABEL: vec256_v8i32_to_v4i64_factor2:2227; AVX2:       # %bb.0:2228; AVX2-NEXT:    vmovdqa (%rdi), %xmm02229; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm02230; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero2231; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm02232; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)2233; AVX2-NEXT:    vzeroupper2234; AVX2-NEXT:    retq2235;2236; AVX512F-LABEL: vec256_v8i32_to_v4i64_factor2:2237; AVX512F:       # %bb.0:2238; AVX512F-NEXT:    vmovdqa (%rdi), %xmm02239; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm02240; AVX512F-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero2241; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm02242; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)2243; AVX512F-NEXT:    vzeroupper2244; AVX512F-NEXT:    retq2245;2246; AVX512BW-LABEL: vec256_v8i32_to_v4i64_factor2:2247; AVX512BW:       # %bb.0:2248; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm02249; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm02250; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero2251; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm02252; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)2253; AVX512BW-NEXT:    vzeroupper2254; AVX512BW-NEXT:    retq2255  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 642256  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 642257  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias2258  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>2259  %in.vec.cast = bitcast <32 x i8> %in.vec.trunc to <8 x i32>2260  %zextd.vec = shufflevector <8 x i32> %in.vec.cast, <8 x i32> zeroinitializer, <8 x i32> <i32 0, i32 9, i32 1, i32 11, i32 2, i32 13, i32 3, i32 15>2261  %out.bytevec = bitcast <8 x i32> %zextd.vec to <32 x i8>2262  %out.bytevec.padded = shufflevector <32 x i8> %out.bytevec, <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>2263  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 642264  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias2265  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 642266  ret void2267}2268 2269define void @vec256_v8i32_to_v2i128_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {2270; SSE2-LABEL: vec256_v8i32_to_v2i128_factor4:2271; SSE2:       # %bb.0:2272; SSE2-NEXT:    movdqa (%rdi), %xmm02273; SSE2-NEXT:    paddb (%rsi), %xmm02274; SSE2-NEXT:    xorps %xmm1, %xmm12275; SSE2-NEXT:    xorps %xmm2, %xmm22276; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3]2277; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,0],xmm1[1,0]2278; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[2,3]2279; SSE2-NEXT:    paddb 16(%rdx), %xmm02280; SSE2-NEXT:    paddb (%rdx), %xmm22281; SSE2-NEXT:    movdqa %xmm2, (%rcx)2282; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)2283; SSE2-NEXT:    retq2284;2285; SSE42-LABEL: vec256_v8i32_to_v2i128_factor4:2286; SSE42:       # %bb.0:2287; SSE42-NEXT:    movdqa (%rdi), %xmm02288; SSE42-NEXT:    paddb (%rsi), %xmm02289; SSE42-NEXT:    pxor %xmm1, %xmm12290; SSE42-NEXT:    pxor %xmm2, %xmm22291; SSE42-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1],xmm2[2,3,4,5,6,7]2292; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]2293; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]2294; SSE42-NEXT:    paddb 16(%rdx), %xmm02295; SSE42-NEXT:    paddb (%rdx), %xmm22296; SSE42-NEXT:    movdqa %xmm2, (%rcx)2297; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)2298; SSE42-NEXT:    retq2299;2300; AVX-LABEL: vec256_v8i32_to_v2i128_factor4:2301; AVX:       # %bb.0:2302; AVX-NEXT:    vmovdqa (%rdi), %xmm02303; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm02304; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm12305; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7]2306; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[1],zero,zero,zero2307; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm02308; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm12309; AVX-NEXT:    vmovdqa %xmm1, (%rcx)2310; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)2311; AVX-NEXT:    retq2312;2313; AVX2-SLOW-LABEL: vec256_v8i32_to_v2i128_factor4:2314; AVX2-SLOW:       # %bb.0:2315; AVX2-SLOW-NEXT:    vmovdqa (%rdi), %xmm02316; AVX2-SLOW-NEXT:    vpaddb (%rsi), %xmm0, %xmm02317; AVX2-SLOW-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero2318; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]2319; AVX2-SLOW-NEXT:    vpxor %xmm1, %xmm1, %xmm12320; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]2321; AVX2-SLOW-NEXT:    vpaddb (%rdx), %ymm0, %ymm02322; AVX2-SLOW-NEXT:    vmovdqa %ymm0, (%rcx)2323; AVX2-SLOW-NEXT:    vzeroupper2324; AVX2-SLOW-NEXT:    retq2325;2326; AVX2-FAST-PERLANE-LABEL: vec256_v8i32_to_v2i128_factor4:2327; AVX2-FAST-PERLANE:       # %bb.0:2328; AVX2-FAST-PERLANE-NEXT:    vmovdqa (%rdi), %xmm02329; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rsi), %xmm0, %xmm02330; AVX2-FAST-PERLANE-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero2331; AVX2-FAST-PERLANE-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]2332; AVX2-FAST-PERLANE-NEXT:    vpxor %xmm1, %xmm1, %xmm12333; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]2334; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rdx), %ymm0, %ymm02335; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm0, (%rcx)2336; AVX2-FAST-PERLANE-NEXT:    vzeroupper2337; AVX2-FAST-PERLANE-NEXT:    retq2338;2339; AVX2-FAST-LABEL: vec256_v8i32_to_v2i128_factor4:2340; AVX2-FAST:       # %bb.0:2341; AVX2-FAST-NEXT:    vmovdqa (%rdi), %ymm02342; AVX2-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm02343; AVX2-FAST-NEXT:    vpmovsxbq {{.*#+}} ymm1 = [0,0,1,0]2344; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm1, %ymm02345; AVX2-FAST-NEXT:    vpxor %xmm1, %xmm1, %xmm12346; AVX2-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]2347; AVX2-FAST-NEXT:    vpaddb (%rdx), %ymm0, %ymm02348; AVX2-FAST-NEXT:    vmovdqa %ymm0, (%rcx)2349; AVX2-FAST-NEXT:    vzeroupper2350; AVX2-FAST-NEXT:    retq2351;2352; AVX512F-LABEL: vec256_v8i32_to_v2i128_factor4:2353; AVX512F:       # %bb.0:2354; AVX512F-NEXT:    vmovdqa (%rdi), %ymm02355; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm02356; AVX512F-NEXT:    movb $17, %al2357; AVX512F-NEXT:    kmovw %eax, %k12358; AVX512F-NEXT:    vpexpandd %ymm0, %ymm0 {%k1} {z}2359; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm02360; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)2361; AVX512F-NEXT:    vzeroupper2362; AVX512F-NEXT:    retq2363;2364; AVX512BW-LABEL: vec256_v8i32_to_v2i128_factor4:2365; AVX512BW:       # %bb.0:2366; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm02367; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm02368; AVX512BW-NEXT:    movb $17, %al2369; AVX512BW-NEXT:    kmovd %eax, %k12370; AVX512BW-NEXT:    vpexpandd %ymm0, %ymm0 {%k1} {z}2371; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm02372; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)2373; AVX512BW-NEXT:    vzeroupper2374; AVX512BW-NEXT:    retq2375  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 642376  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 642377  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias2378  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>2379  %in.vec.cast = bitcast <32 x i8> %in.vec.trunc to <8 x i32>2380  %zextd.vec = shufflevector <8 x i32> %in.vec.cast, <8 x i32> zeroinitializer, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 1, i32 13, i32 14, i32 15>2381  %out.bytevec = bitcast <8 x i32> %zextd.vec to <32 x i8>2382  %out.bytevec.padded = shufflevector <32 x i8> %out.bytevec, <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>2383  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 642384  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias2385  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 642386  ret void2387}2388 2389define void @vec256_v8i32_to_v1i256_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {2390; SSE2-LABEL: vec256_v8i32_to_v1i256_factor8:2391; SSE2:       # %bb.0:2392; SSE2-NEXT:    movdqa (%rdi), %xmm02393; SSE2-NEXT:    paddb (%rsi), %xmm02394; SSE2-NEXT:    xorps %xmm1, %xmm12395; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]2396; SSE2-NEXT:    movaps 16(%rdx), %xmm02397; SSE2-NEXT:    paddb (%rdx), %xmm12398; SSE2-NEXT:    movaps %xmm0, 16(%rcx)2399; SSE2-NEXT:    movdqa %xmm1, (%rcx)2400; SSE2-NEXT:    retq2401;2402; SSE42-LABEL: vec256_v8i32_to_v1i256_factor8:2403; SSE42:       # %bb.0:2404; SSE42-NEXT:    movdqa (%rdi), %xmm02405; SSE42-NEXT:    paddb (%rsi), %xmm02406; SSE42-NEXT:    pxor %xmm1, %xmm12407; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7]2408; SSE42-NEXT:    movaps 16(%rdx), %xmm02409; SSE42-NEXT:    paddb (%rdx), %xmm12410; SSE42-NEXT:    movaps %xmm0, 16(%rcx)2411; SSE42-NEXT:    movdqa %xmm1, (%rcx)2412; SSE42-NEXT:    retq2413;2414; AVX-LABEL: vec256_v8i32_to_v1i256_factor8:2415; AVX:       # %bb.0:2416; AVX-NEXT:    vmovdqa (%rdi), %xmm02417; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm02418; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm12419; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]2420; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm02421; AVX-NEXT:    vmovaps 16(%rdx), %xmm12422; AVX-NEXT:    vmovaps %xmm1, 16(%rcx)2423; AVX-NEXT:    vmovdqa %xmm0, (%rcx)2424; AVX-NEXT:    retq2425;2426; AVX2-LABEL: vec256_v8i32_to_v1i256_factor8:2427; AVX2:       # %bb.0:2428; AVX2-NEXT:    vmovdqa (%rdi), %ymm02429; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm02430; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm12431; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]2432; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm02433; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)2434; AVX2-NEXT:    vzeroupper2435; AVX2-NEXT:    retq2436;2437; AVX512F-LABEL: vec256_v8i32_to_v1i256_factor8:2438; AVX512F:       # %bb.0:2439; AVX512F-NEXT:    vmovdqa (%rdi), %ymm02440; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm02441; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm12442; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]2443; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm02444; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)2445; AVX512F-NEXT:    vzeroupper2446; AVX512F-NEXT:    retq2447;2448; AVX512BW-LABEL: vec256_v8i32_to_v1i256_factor8:2449; AVX512BW:       # %bb.0:2450; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm02451; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm02452; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm12453; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]2454; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm02455; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)2456; AVX512BW-NEXT:    vzeroupper2457; AVX512BW-NEXT:    retq2458  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 642459  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 642460  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias2461  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>2462  %in.vec.cast = bitcast <32 x i8> %in.vec.trunc to <8 x i32>2463  %zextd.vec = shufflevector <8 x i32> %in.vec.cast, <8 x i32> zeroinitializer, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>2464  %out.bytevec = bitcast <8 x i32> %zextd.vec to <32 x i8>2465  %out.bytevec.padded = shufflevector <32 x i8> %out.bytevec, <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>2466  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 642467  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias2468  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 642469  ret void2470}2471 2472define void @vec256_v4i64_to_v2i128_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {2473; SSE-LABEL: vec256_v4i64_to_v2i128_factor2:2474; SSE:       # %bb.0:2475; SSE-NEXT:    movdqa (%rdi), %xmm02476; SSE-NEXT:    paddb (%rsi), %xmm02477; SSE-NEXT:    movq {{.*#+}} xmm1 = xmm0[0],zero2478; SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero2479; SSE-NEXT:    paddb 16(%rdx), %xmm02480; SSE-NEXT:    paddb (%rdx), %xmm12481; SSE-NEXT:    movdqa %xmm1, (%rcx)2482; SSE-NEXT:    movdqa %xmm0, 16(%rcx)2483; SSE-NEXT:    retq2484;2485; AVX-LABEL: vec256_v4i64_to_v2i128_factor2:2486; AVX:       # %bb.0:2487; AVX-NEXT:    vmovdqa (%rdi), %xmm02488; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm02489; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm02490; AVX-NEXT:    vxorpd %xmm1, %xmm1, %xmm12491; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[3],ymm1[3]2492; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm12493; AVX-NEXT:    vpaddb 16(%rdx), %xmm1, %xmm12494; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm02495; AVX-NEXT:    vmovdqa %xmm0, (%rcx)2496; AVX-NEXT:    vmovdqa %xmm1, 16(%rcx)2497; AVX-NEXT:    vzeroupper2498; AVX-NEXT:    retq2499;2500; AVX2-LABEL: vec256_v4i64_to_v2i128_factor2:2501; AVX2:       # %bb.0:2502; AVX2-NEXT:    vmovdqa (%rdi), %ymm02503; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm02504; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]2505; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm12506; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]2507; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm02508; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)2509; AVX2-NEXT:    vzeroupper2510; AVX2-NEXT:    retq2511;2512; AVX512F-LABEL: vec256_v4i64_to_v2i128_factor2:2513; AVX512F:       # %bb.0:2514; AVX512F-NEXT:    vmovdqa (%rdi), %ymm02515; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm02516; AVX512F-NEXT:    movb $5, %al2517; AVX512F-NEXT:    kmovw %eax, %k12518; AVX512F-NEXT:    vpexpandq %ymm0, %ymm0 {%k1} {z}2519; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm02520; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)2521; AVX512F-NEXT:    vzeroupper2522; AVX512F-NEXT:    retq2523;2524; AVX512BW-LABEL: vec256_v4i64_to_v2i128_factor2:2525; AVX512BW:       # %bb.0:2526; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm02527; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm02528; AVX512BW-NEXT:    movb $5, %al2529; AVX512BW-NEXT:    kmovd %eax, %k12530; AVX512BW-NEXT:    vpexpandq %ymm0, %ymm0 {%k1} {z}2531; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm02532; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)2533; AVX512BW-NEXT:    vzeroupper2534; AVX512BW-NEXT:    retq2535  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 642536  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 642537  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias2538  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>2539  %in.vec.cast = bitcast <32 x i8> %in.vec.trunc to <4 x i64>2540  %zextd.vec = shufflevector <4 x i64> %in.vec.cast, <4 x i64> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 1, i32 7>2541  %out.bytevec = bitcast <4 x i64> %zextd.vec to <32 x i8>2542  %out.bytevec.padded = shufflevector <32 x i8> %out.bytevec, <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>2543  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 642544  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias2545  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 642546  ret void2547}2548 2549define void @vec256_v4i64_to_v1i256_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {2550; SSE-LABEL: vec256_v4i64_to_v1i256_factor4:2551; SSE:       # %bb.0:2552; SSE-NEXT:    movdqa (%rdi), %xmm02553; SSE-NEXT:    paddb (%rsi), %xmm02554; SSE-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero2555; SSE-NEXT:    movaps 16(%rdx), %xmm12556; SSE-NEXT:    paddb (%rdx), %xmm02557; SSE-NEXT:    movaps %xmm1, 16(%rcx)2558; SSE-NEXT:    movdqa %xmm0, (%rcx)2559; SSE-NEXT:    retq2560;2561; AVX-LABEL: vec256_v4i64_to_v1i256_factor4:2562; AVX:       # %bb.0:2563; AVX-NEXT:    vmovdqa (%rdi), %xmm02564; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm02565; AVX-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero2566; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm02567; AVX-NEXT:    vmovaps 16(%rdx), %xmm12568; AVX-NEXT:    vmovaps %xmm1, 16(%rcx)2569; AVX-NEXT:    vmovdqa %xmm0, (%rcx)2570; AVX-NEXT:    retq2571;2572; AVX2-LABEL: vec256_v4i64_to_v1i256_factor4:2573; AVX2:       # %bb.0:2574; AVX2-NEXT:    vmovdqa (%rdi), %ymm02575; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm02576; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero2577; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm02578; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)2579; AVX2-NEXT:    vzeroupper2580; AVX2-NEXT:    retq2581;2582; AVX512F-LABEL: vec256_v4i64_to_v1i256_factor4:2583; AVX512F:       # %bb.0:2584; AVX512F-NEXT:    vmovdqa (%rdi), %ymm02585; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm02586; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero2587; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm02588; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)2589; AVX512F-NEXT:    vzeroupper2590; AVX512F-NEXT:    retq2591;2592; AVX512BW-LABEL: vec256_v4i64_to_v1i256_factor4:2593; AVX512BW:       # %bb.0:2594; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm02595; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm02596; AVX512BW-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero2597; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm02598; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)2599; AVX512BW-NEXT:    vzeroupper2600; AVX512BW-NEXT:    retq2601  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 642602  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 642603  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias2604  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>2605  %in.vec.cast = bitcast <32 x i8> %in.vec.trunc to <4 x i64>2606  %zextd.vec = shufflevector <4 x i64> %in.vec.cast, <4 x i64> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 6, i32 7>2607  %out.bytevec = bitcast <4 x i64> %zextd.vec to <32 x i8>2608  %out.bytevec.padded = shufflevector <32 x i8> %out.bytevec, <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>2609  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 642610  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias2611  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 642612  ret void2613}2614 2615define void @vec256_v2i128_to_v1i256_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {2616; SSE-LABEL: vec256_v2i128_to_v1i256_factor2:2617; SSE:       # %bb.0:2618; SSE-NEXT:    movdqa (%rdi), %xmm02619; SSE-NEXT:    paddb (%rsi), %xmm02620; SSE-NEXT:    movaps 16(%rdx), %xmm12621; SSE-NEXT:    paddb (%rdx), %xmm02622; SSE-NEXT:    movaps %xmm1, 16(%rcx)2623; SSE-NEXT:    movdqa %xmm0, (%rcx)2624; SSE-NEXT:    retq2625;2626; AVX-LABEL: vec256_v2i128_to_v1i256_factor2:2627; AVX:       # %bb.0:2628; AVX-NEXT:    vmovdqa (%rdi), %xmm02629; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm02630; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm02631; AVX-NEXT:    vmovaps 16(%rdx), %xmm12632; AVX-NEXT:    vmovaps %xmm1, 16(%rcx)2633; AVX-NEXT:    vmovdqa %xmm0, (%rcx)2634; AVX-NEXT:    retq2635;2636; AVX2-LABEL: vec256_v2i128_to_v1i256_factor2:2637; AVX2:       # %bb.0:2638; AVX2-NEXT:    vmovdqa (%rdi), %xmm02639; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm02640; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm02641; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)2642; AVX2-NEXT:    vzeroupper2643; AVX2-NEXT:    retq2644;2645; AVX512F-LABEL: vec256_v2i128_to_v1i256_factor2:2646; AVX512F:       # %bb.0:2647; AVX512F-NEXT:    vmovdqa (%rdi), %xmm02648; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm02649; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm02650; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)2651; AVX512F-NEXT:    vzeroupper2652; AVX512F-NEXT:    retq2653;2654; AVX512BW-LABEL: vec256_v2i128_to_v1i256_factor2:2655; AVX512BW:       # %bb.0:2656; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm02657; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm02658; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm02659; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)2660; AVX512BW-NEXT:    vzeroupper2661; AVX512BW-NEXT:    retq2662  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 642663  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 642664  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias2665  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>2666  %in.vec.cast = bitcast <32 x i8> %in.vec.trunc to <2 x i128>2667  %zextd.vec = shufflevector <2 x i128> %in.vec.cast, <2 x i128> zeroinitializer, <2 x i32> <i32 0, i32 3>2668  %out.bytevec = bitcast <2 x i128> %zextd.vec to <32 x i8>2669  %out.bytevec.padded = shufflevector <32 x i8> %out.bytevec, <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>2670  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 642671  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias2672  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 642673  ret void2674}2675 2676define void @vec384_v48i8_to_v24i16_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {2677; SSE2-LABEL: vec384_v48i8_to_v24i16_factor2:2678; SSE2:       # %bb.0:2679; SSE2-NEXT:    movdqa (%rdi), %xmm02680; SSE2-NEXT:    movdqa 16(%rdi), %xmm12681; SSE2-NEXT:    paddb (%rsi), %xmm02682; SSE2-NEXT:    paddb 16(%rsi), %xmm12683; SSE2-NEXT:    pxor %xmm2, %xmm22684; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]2685; SSE2-NEXT:    movdqa %xmm0, %xmm32686; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]2687; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]2688; SSE2-NEXT:    paddb 16(%rdx), %xmm02689; SSE2-NEXT:    paddb (%rdx), %xmm32690; SSE2-NEXT:    paddb 32(%rdx), %xmm12691; SSE2-NEXT:    movdqa %xmm1, 32(%rcx)2692; SSE2-NEXT:    movdqa %xmm3, (%rcx)2693; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)2694; SSE2-NEXT:    retq2695;2696; SSE42-LABEL: vec384_v48i8_to_v24i16_factor2:2697; SSE42:       # %bb.0:2698; SSE42-NEXT:    movdqa (%rdi), %xmm02699; SSE42-NEXT:    movdqa 16(%rdi), %xmm12700; SSE42-NEXT:    paddb (%rsi), %xmm02701; SSE42-NEXT:    paddb 16(%rsi), %xmm12702; SSE42-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero2703; SSE42-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero2704; SSE42-NEXT:    pxor %xmm3, %xmm32705; SSE42-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm3[8],xmm0[9],xmm3[9],xmm0[10],xmm3[10],xmm0[11],xmm3[11],xmm0[12],xmm3[12],xmm0[13],xmm3[13],xmm0[14],xmm3[14],xmm0[15],xmm3[15]2706; SSE42-NEXT:    paddb 16(%rdx), %xmm02707; SSE42-NEXT:    paddb (%rdx), %xmm22708; SSE42-NEXT:    paddb 32(%rdx), %xmm12709; SSE42-NEXT:    movdqa %xmm1, 32(%rcx)2710; SSE42-NEXT:    movdqa %xmm2, (%rcx)2711; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)2712; SSE42-NEXT:    retq2713;2714; AVX-LABEL: vec384_v48i8_to_v24i16_factor2:2715; AVX:       # %bb.0:2716; AVX-NEXT:    vmovdqa (%rdi), %xmm02717; AVX-NEXT:    vmovdqa 16(%rdi), %xmm12718; AVX-NEXT:    vpaddb 16(%rsi), %xmm1, %xmm12719; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm02720; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero2721; AVX-NEXT:    vpxor %xmm3, %xmm3, %xmm32722; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm3[8],xmm0[9],xmm3[9],xmm0[10],xmm3[10],xmm0[11],xmm3[11],xmm0[12],xmm3[12],xmm0[13],xmm3[13],xmm0[14],xmm3[14],xmm0[15],xmm3[15]2723; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero2724; AVX-NEXT:    vpaddb 32(%rdx), %xmm1, %xmm12725; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm02726; AVX-NEXT:    vpaddb (%rdx), %xmm2, %xmm22727; AVX-NEXT:    vmovdqa %xmm2, (%rcx)2728; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)2729; AVX-NEXT:    vmovdqa %xmm1, 32(%rcx)2730; AVX-NEXT:    retq2731;2732; AVX2-LABEL: vec384_v48i8_to_v24i16_factor2:2733; AVX2:       # %bb.0:2734; AVX2-NEXT:    vmovdqa (%rdi), %ymm02735; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm02736; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero2737; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm02738; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero2739; AVX2-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm02740; AVX2-NEXT:    vpaddb (%rdx), %ymm1, %ymm12741; AVX2-NEXT:    vmovdqa %ymm1, (%rcx)2742; AVX2-NEXT:    vmovdqa %ymm0, 32(%rcx)2743; AVX2-NEXT:    vzeroupper2744; AVX2-NEXT:    retq2745;2746; AVX512F-LABEL: vec384_v48i8_to_v24i16_factor2:2747; AVX512F:       # %bb.0:2748; AVX512F-NEXT:    vmovdqa (%rdi), %ymm02749; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm02750; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero2751; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm02752; AVX512F-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero2753; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm02754; AVX512F-NEXT:    vpaddb (%rdx), %ymm1, %ymm12755; AVX512F-NEXT:    vmovdqa %ymm1, (%rcx)2756; AVX512F-NEXT:    vmovdqa %ymm0, 32(%rcx)2757; AVX512F-NEXT:    vzeroupper2758; AVX512F-NEXT:    retq2759;2760; AVX512BW-LABEL: vec384_v48i8_to_v24i16_factor2:2761; AVX512BW:       # %bb.0:2762; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm02763; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm02764; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero2765; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm02766; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)2767; AVX512BW-NEXT:    vzeroupper2768; AVX512BW-NEXT:    retq2769  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 642770  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 642771  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias2772  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>2773  %zextd.vec = shufflevector <48 x i8> %in.vec.trunc, <48 x i8> zeroinitializer, <48 x i32> <i32 0, i32 49, i32 1, i32 51, i32 2, i32 53, i32 3, i32 55, i32 4, i32 57, i32 5, i32 59, i32 6, i32 61, i32 7, i32 63, i32 8, i32 65, i32 9, i32 67, i32 10, i32 69, i32 11, i32 71, i32 12, i32 73, i32 13, i32 75, i32 14, i32 77, i32 15, i32 79, i32 16, i32 81, i32 17, i32 83, i32 18, i32 85, i32 19, i32 87, i32 20, i32 89, i32 21, i32 91, i32 22, i32 93, i32 23, i32 95>2774  %out.bytevec.padded = shufflevector <48 x i8> %zextd.vec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>2775  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 642776  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias2777  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 642778  ret void2779}2780 2781define void @vec384_v48i8_to_v16i24_factor3(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {2782; SSE2-LABEL: vec384_v48i8_to_v16i24_factor3:2783; SSE2:       # %bb.0:2784; SSE2-NEXT:    movdqa (%rdi), %xmm02785; SSE2-NEXT:    paddb (%rsi), %xmm02786; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[3,3,3,3,4,5,6,7]2787; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,6,5]2788; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm12789; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]2790; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[1,1,2,2,4,5,6,7]2791; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,7,7,7]2792; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm22793; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]2794; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]2795; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,6,6]2796; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm02797; SSE2-NEXT:    paddb (%rdx), %xmm02798; SSE2-NEXT:    paddb 32(%rdx), %xmm22799; SSE2-NEXT:    paddb 16(%rdx), %xmm12800; SSE2-NEXT:    movdqa %xmm1, 16(%rcx)2801; SSE2-NEXT:    movdqa %xmm2, 32(%rcx)2802; SSE2-NEXT:    movdqa %xmm0, (%rcx)2803; SSE2-NEXT:    retq2804;2805; SSE42-LABEL: vec384_v48i8_to_v16i24_factor3:2806; SSE42:       # %bb.0:2807; SSE42-NEXT:    movdqa (%rdi), %xmm02808; SSE42-NEXT:    paddb (%rsi), %xmm02809; SSE42-NEXT:    movdqa %xmm0, %xmm12810; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = zero,xmm1[11],zero,zero,xmm1[12],zero,zero,xmm1[13],zero,zero,xmm1[14],zero,zero,xmm1[15],zero,zero2811; SSE42-NEXT:    movdqa %xmm0, %xmm22812; SSE42-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0],zero,zero,xmm2[1],zero,zero,xmm2[2],zero,zero,xmm2[3],zero,zero,xmm2[4],zero,zero,xmm2[5]2813; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = zero,zero,xmm0[6],zero,zero,xmm0[7],zero,zero,xmm0[8],zero,zero,xmm0[9],zero,zero,xmm0[10],zero2814; SSE42-NEXT:    paddb 16(%rdx), %xmm02815; SSE42-NEXT:    paddb (%rdx), %xmm22816; SSE42-NEXT:    paddb 32(%rdx), %xmm12817; SSE42-NEXT:    movdqa %xmm1, 32(%rcx)2818; SSE42-NEXT:    movdqa %xmm2, (%rcx)2819; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)2820; SSE42-NEXT:    retq2821;2822; AVX-LABEL: vec384_v48i8_to_v16i24_factor3:2823; AVX:       # %bb.0:2824; AVX-NEXT:    vmovdqa (%rdi), %xmm02825; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm02826; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[0],zero,zero,xmm0[1],zero,zero,xmm0[2],zero,zero,xmm0[3],zero,zero,xmm0[4],zero,zero,xmm0[5]2827; AVX-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,xmm0[6],zero,zero,xmm0[7],zero,zero,xmm0[8],zero,zero,xmm0[9],zero,zero,xmm0[10],zero2828; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = zero,xmm0[11],zero,zero,xmm0[12],zero,zero,xmm0[13],zero,zero,xmm0[14],zero,zero,xmm0[15],zero,zero2829; AVX-NEXT:    vpaddb 32(%rdx), %xmm0, %xmm02830; AVX-NEXT:    vpaddb 16(%rdx), %xmm2, %xmm22831; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm12832; AVX-NEXT:    vmovdqa %xmm1, (%rcx)2833; AVX-NEXT:    vmovdqa %xmm2, 16(%rcx)2834; AVX-NEXT:    vmovdqa %xmm0, 32(%rcx)2835; AVX-NEXT:    retq2836;2837; AVX2-LABEL: vec384_v48i8_to_v16i24_factor3:2838; AVX2:       # %bb.0:2839; AVX2-NEXT:    vmovdqa (%rdi), %ymm02840; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm02841; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[0,1,0,1]2842; AVX2-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0],zero,zero,ymm1[1],zero,zero,ymm1[2],zero,zero,ymm1[3],zero,zero,ymm1[4],zero,zero,ymm1[5],zero,zero,ymm1[22],zero,zero,ymm1[23],zero,zero,ymm1[24],zero,zero,ymm1[25],zero,zero,ymm1[26],zero2843; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = zero,xmm0[11],zero,zero,xmm0[12],zero,zero,xmm0[13],zero,zero,xmm0[14],zero,zero,xmm0[15],zero,zero2844; AVX2-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm02845; AVX2-NEXT:    vpaddb (%rdx), %ymm1, %ymm12846; AVX2-NEXT:    vmovdqa %ymm1, (%rcx)2847; AVX2-NEXT:    vmovdqa %ymm0, 32(%rcx)2848; AVX2-NEXT:    vzeroupper2849; AVX2-NEXT:    retq2850;2851; AVX512F-LABEL: vec384_v48i8_to_v16i24_factor3:2852; AVX512F:       # %bb.0:2853; AVX512F-NEXT:    vmovdqa (%rdi), %ymm02854; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm02855; AVX512F-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[0,1,0,1]2856; AVX512F-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0],zero,zero,ymm1[1],zero,zero,ymm1[2],zero,zero,ymm1[3],zero,zero,ymm1[4],zero,zero,ymm1[5],zero,zero,ymm1[22],zero,zero,ymm1[23],zero,zero,ymm1[24],zero,zero,ymm1[25],zero,zero,ymm1[26],zero2857; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = zero,xmm0[11],zero,zero,xmm0[12],zero,zero,xmm0[13],zero,zero,xmm0[14],zero,zero,xmm0[15],zero,zero2858; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm02859; AVX512F-NEXT:    vpaddb (%rdx), %ymm1, %ymm12860; AVX512F-NEXT:    vmovdqa %ymm1, (%rcx)2861; AVX512F-NEXT:    vmovdqa %ymm0, 32(%rcx)2862; AVX512F-NEXT:    vzeroupper2863; AVX512F-NEXT:    retq2864;2865; AVX512BW-LABEL: vec384_v48i8_to_v16i24_factor3:2866; AVX512BW:       # %bb.0:2867; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm02868; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm02869; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [0,0,0,1,1,0,2,2,0,3,3,0,4,4,0,5]2870; AVX512BW-NEXT:    vpermw %ymm0, %ymm1, %ymm12871; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm12872; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = zero,xmm0[11],zero,zero,xmm0[12],zero,zero,xmm0[13],zero,zero,xmm0[14],zero,zero,xmm0[15],zero,zero2873; AVX512BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm02874; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm02875; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)2876; AVX512BW-NEXT:    vzeroupper2877; AVX512BW-NEXT:    retq2878  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 642879  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 642880  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias2881  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>2882  %zextd.vec = shufflevector <48 x i8> %in.vec.trunc, <48 x i8> zeroinitializer, <48 x i32> <i32 0, i32 49, i32 50, i32 1, i32 52, i32 53, i32 2, i32 55, i32 56, i32 3, i32 58, i32 59, i32 4, i32 61, i32 62, i32 5, i32 64, i32 65, i32 6, i32 67, i32 68, i32 7, i32 70, i32 71, i32 8, i32 73, i32 74, i32 9, i32 76, i32 77, i32 10, i32 79, i32 80, i32 11, i32 82, i32 83, i32 12, i32 85, i32 86, i32 13, i32 88, i32 89, i32 14, i32 91, i32 92, i32 15, i32 94, i32 95>2883  %out.bytevec.padded = shufflevector <48 x i8> %zextd.vec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>2884  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 642885  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias2886  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 642887  ret void2888}2889 2890define void @vec384_v48i8_to_v12i32_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {2891; SSE2-LABEL: vec384_v48i8_to_v12i32_factor4:2892; SSE2:       # %bb.0:2893; SSE2-NEXT:    movdqa (%rdi), %xmm02894; SSE2-NEXT:    paddb (%rsi), %xmm02895; SSE2-NEXT:    pxor %xmm1, %xmm12896; SSE2-NEXT:    movdqa %xmm0, %xmm22897; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]2898; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]2899; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]2900; SSE2-NEXT:    movdqa %xmm0, %xmm32901; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]2902; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]2903; SSE2-NEXT:    paddb 16(%rdx), %xmm02904; SSE2-NEXT:    paddb (%rdx), %xmm32905; SSE2-NEXT:    paddb 32(%rdx), %xmm22906; SSE2-NEXT:    movdqa %xmm2, 32(%rcx)2907; SSE2-NEXT:    movdqa %xmm3, (%rcx)2908; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)2909; SSE2-NEXT:    retq2910;2911; SSE42-LABEL: vec384_v48i8_to_v12i32_factor4:2912; SSE42:       # %bb.0:2913; SSE42-NEXT:    movdqa (%rdi), %xmm02914; SSE42-NEXT:    paddb (%rsi), %xmm02915; SSE42-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero2916; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]2917; SSE42-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero2918; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]2919; SSE42-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero2920; SSE42-NEXT:    paddb 16(%rdx), %xmm02921; SSE42-NEXT:    paddb 32(%rdx), %xmm22922; SSE42-NEXT:    paddb (%rdx), %xmm12923; SSE42-NEXT:    movdqa %xmm1, (%rcx)2924; SSE42-NEXT:    movdqa %xmm2, 32(%rcx)2925; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)2926; SSE42-NEXT:    retq2927;2928; AVX-LABEL: vec384_v48i8_to_v12i32_factor4:2929; AVX:       # %bb.0:2930; AVX-NEXT:    vmovdqa (%rdi), %xmm02931; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm02932; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero2933; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]2934; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero2935; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]2936; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero2937; AVX-NEXT:    vpaddb 32(%rdx), %xmm0, %xmm02938; AVX-NEXT:    vpaddb 16(%rdx), %xmm2, %xmm22939; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm12940; AVX-NEXT:    vmovdqa %xmm1, (%rcx)2941; AVX-NEXT:    vmovdqa %xmm2, 16(%rcx)2942; AVX-NEXT:    vmovdqa %xmm0, 32(%rcx)2943; AVX-NEXT:    retq2944;2945; AVX2-SLOW-LABEL: vec384_v48i8_to_v12i32_factor4:2946; AVX2-SLOW:       # %bb.0:2947; AVX2-SLOW-NEXT:    vmovdqa (%rdi), %xmm02948; AVX2-SLOW-NEXT:    vpaddb (%rsi), %xmm0, %xmm02949; AVX2-SLOW-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero2950; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]2951; AVX2-SLOW-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero2952; AVX2-SLOW-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm02953; AVX2-SLOW-NEXT:    vpaddb (%rdx), %ymm1, %ymm12954; AVX2-SLOW-NEXT:    vmovdqa %ymm1, (%rcx)2955; AVX2-SLOW-NEXT:    vmovdqa %ymm0, 32(%rcx)2956; AVX2-SLOW-NEXT:    vzeroupper2957; AVX2-SLOW-NEXT:    retq2958;2959; AVX2-FAST-PERLANE-LABEL: vec384_v48i8_to_v12i32_factor4:2960; AVX2-FAST-PERLANE:       # %bb.0:2961; AVX2-FAST-PERLANE-NEXT:    vmovdqa (%rdi), %xmm02962; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rsi), %xmm0, %xmm02963; AVX2-FAST-PERLANE-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero2964; AVX2-FAST-PERLANE-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero2965; AVX2-FAST-PERLANE-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm02966; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rdx), %ymm1, %ymm12967; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm1, (%rcx)2968; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm0, 32(%rcx)2969; AVX2-FAST-PERLANE-NEXT:    vzeroupper2970; AVX2-FAST-PERLANE-NEXT:    retq2971;2972; AVX2-FAST-LABEL: vec384_v48i8_to_v12i32_factor4:2973; AVX2-FAST:       # %bb.0:2974; AVX2-FAST-NEXT:    vmovdqa (%rdi), %xmm02975; AVX2-FAST-NEXT:    vpaddb (%rsi), %xmm0, %xmm02976; AVX2-FAST-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero2977; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero2978; AVX2-FAST-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm02979; AVX2-FAST-NEXT:    vpaddb (%rdx), %ymm1, %ymm12980; AVX2-FAST-NEXT:    vmovdqa %ymm1, (%rcx)2981; AVX2-FAST-NEXT:    vmovdqa %ymm0, 32(%rcx)2982; AVX2-FAST-NEXT:    vzeroupper2983; AVX2-FAST-NEXT:    retq2984;2985; AVX512F-LABEL: vec384_v48i8_to_v12i32_factor4:2986; AVX512F:       # %bb.0:2987; AVX512F-NEXT:    vmovdqa (%rdi), %xmm02988; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm02989; AVX512F-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero2990; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero2991; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm02992; AVX512F-NEXT:    vpaddb (%rdx), %ymm1, %ymm12993; AVX512F-NEXT:    vmovdqa %ymm1, (%rcx)2994; AVX512F-NEXT:    vmovdqa %ymm0, 32(%rcx)2995; AVX512F-NEXT:    vzeroupper2996; AVX512F-NEXT:    retq2997;2998; AVX512BW-LABEL: vec384_v48i8_to_v12i32_factor4:2999; AVX512BW:       # %bb.0:3000; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm03001; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm03002; AVX512BW-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero3003; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm03004; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)3005; AVX512BW-NEXT:    vzeroupper3006; AVX512BW-NEXT:    retq3007  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 643008  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 643009  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias3010  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>3011  %zextd.vec = shufflevector <48 x i8> %in.vec.trunc, <48 x i8> zeroinitializer, <48 x i32> <i32 0, i32 49, i32 50, i32 51, i32 1, i32 53, i32 54, i32 55, i32 2, i32 57, i32 58, i32 59, i32 3, i32 61, i32 62, i32 63, i32 4, i32 65, i32 66, i32 67, i32 5, i32 69, i32 70, i32 71, i32 6, i32 73, i32 74, i32 75, i32 7, i32 77, i32 78, i32 79, i32 8, i32 81, i32 82, i32 83, i32 9, i32 85, i32 86, i32 87, i32 10, i32 89, i32 90, i32 91, i32 11, i32 93, i32 94, i32 95>3012  %out.bytevec.padded = shufflevector <48 x i8> %zextd.vec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>3013  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 643014  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias3015  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 643016  ret void3017}3018 3019define void @vec384_v48i8_to_v8i48_factor6(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {3020; SSE2-LABEL: vec384_v48i8_to_v8i48_factor6:3021; SSE2:       # %bb.0:3022; SSE2-NEXT:    movdqa (%rdi), %xmm03023; SSE2-NEXT:    paddb (%rsi), %xmm03024; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]3025; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]3026; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm13027; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,1,1]3028; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm23029; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,2]3030; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm03031; SSE2-NEXT:    paddb 16(%rdx), %xmm03032; SSE2-NEXT:    paddb (%rdx), %xmm23033; SSE2-NEXT:    paddb 32(%rdx), %xmm13034; SSE2-NEXT:    movdqa %xmm1, 32(%rcx)3035; SSE2-NEXT:    movdqa %xmm2, (%rcx)3036; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)3037; SSE2-NEXT:    retq3038;3039; SSE42-LABEL: vec384_v48i8_to_v8i48_factor6:3040; SSE42:       # %bb.0:3041; SSE42-NEXT:    movdqa (%rdi), %xmm03042; SSE42-NEXT:    paddb (%rsi), %xmm03043; SSE42-NEXT:    movdqa %xmm0, %xmm13044; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = zero,zero,zero,zero,xmm1[6],zero,zero,zero,zero,zero,xmm1[7],zero,zero,zero,zero,zero3045; SSE42-NEXT:    movdqa %xmm0, %xmm23046; SSE42-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,xmm2[2],zero,zero,zero3047; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = zero,zero,xmm0[3],zero,zero,zero,zero,zero,xmm0[4],zero,zero,zero,zero,zero,xmm0[5],zero3048; SSE42-NEXT:    paddb 16(%rdx), %xmm03049; SSE42-NEXT:    paddb (%rdx), %xmm23050; SSE42-NEXT:    paddb 32(%rdx), %xmm13051; SSE42-NEXT:    movdqa %xmm1, 32(%rcx)3052; SSE42-NEXT:    movdqa %xmm2, (%rcx)3053; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)3054; SSE42-NEXT:    retq3055;3056; AVX-LABEL: vec384_v48i8_to_v8i48_factor6:3057; AVX:       # %bb.0:3058; AVX-NEXT:    vmovdqa (%rdi), %xmm03059; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm03060; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero3061; AVX-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,xmm0[3],zero,zero,zero,zero,zero,xmm0[4],zero,zero,zero,zero,zero,xmm0[5],zero3062; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[6],zero,zero,zero,zero,zero,xmm0[7],zero,zero,zero,zero,zero3063; AVX-NEXT:    vpaddb 32(%rdx), %xmm0, %xmm03064; AVX-NEXT:    vpaddb 16(%rdx), %xmm2, %xmm23065; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm13066; AVX-NEXT:    vmovdqa %xmm1, (%rcx)3067; AVX-NEXT:    vmovdqa %xmm2, 16(%rcx)3068; AVX-NEXT:    vmovdqa %xmm0, 32(%rcx)3069; AVX-NEXT:    retq3070;3071; AVX2-LABEL: vec384_v48i8_to_v8i48_factor6:3072; AVX2:       # %bb.0:3073; AVX2-NEXT:    vmovdqa (%rdi), %xmm03074; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm03075; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,xmm0[6],zero,zero,zero,zero,zero,xmm0[7],zero,zero,zero,zero,zero3076; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,u,3,u,2,u,1,u,4,u,5,u,6,u,5,u]3077; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,0,1]3078; AVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm03079; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm03080; AVX2-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm13081; AVX2-NEXT:    vmovdqa %ymm1, 32(%rcx)3082; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)3083; AVX2-NEXT:    vzeroupper3084; AVX2-NEXT:    retq3085;3086; AVX512F-LABEL: vec384_v48i8_to_v8i48_factor6:3087; AVX512F:       # %bb.0:3088; AVX512F-NEXT:    vmovdqa (%rdi), %xmm03089; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm03090; AVX512F-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,xmm0[6],zero,zero,zero,zero,zero,xmm0[7],zero,zero,zero,zero,zero3091; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,u,3,u,2,u,1,u,4,u,5,u,6,u,5,u]3092; AVX512F-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,0,1]3093; AVX512F-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm03094; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm03095; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm13096; AVX512F-NEXT:    vmovdqa %ymm1, 32(%rcx)3097; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)3098; AVX512F-NEXT:    vzeroupper3099; AVX512F-NEXT:    retq3100;3101; AVX512BW-LABEL: vec384_v48i8_to_v8i48_factor6:3102; AVX512BW:       # %bb.0:3103; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm03104; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm03105; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero3106; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,3,0,1,4,0,2,5,0,3,0,1,4,0,2,5]3107; AVX512BW-NEXT:    # ymm2 = mem[0,1,0,1]3108; AVX512BW-NEXT:    vpermw %ymm1, %ymm2, %ymm13109; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm13110; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[6],zero,zero,zero,zero,zero,xmm0[7],zero,zero,zero,zero,zero3111; AVX512BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm03112; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm03113; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)3114; AVX512BW-NEXT:    vzeroupper3115; AVX512BW-NEXT:    retq3116  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 643117  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 643118  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias3119  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>3120  %zextd.vec = shufflevector <48 x i8> %in.vec.trunc, <48 x i8> zeroinitializer, <48 x i32> <i32 0, i32 49, i32 50, i32 51, i32 52, i32 53, i32 1, i32 55, i32 56, i32 57, i32 58, i32 59, i32 2, i32 61, i32 62, i32 63, i32 64, i32 65, i32 3, i32 67, i32 68, i32 69, i32 70, i32 71, i32 4, i32 73, i32 74, i32 75, i32 76, i32 77, i32 5, i32 79, i32 80, i32 81, i32 82, i32 83, i32 6, i32 85, i32 86, i32 87, i32 88, i32 89, i32 7, i32 91, i32 92, i32 93, i32 94, i32 95>3121  %out.bytevec.padded = shufflevector <48 x i8> %zextd.vec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>3122  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 643123  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias3124  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 643125  ret void3126}3127 3128define void @vec384_v48i8_to_v6i64_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {3129; SSE2-LABEL: vec384_v48i8_to_v6i64_factor8:3130; SSE2:       # %bb.0:3131; SSE2-NEXT:    movdqa (%rdi), %xmm03132; SSE2-NEXT:    paddb (%rsi), %xmm03133; SSE2-NEXT:    pxor %xmm1, %xmm13134; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]3135; SSE2-NEXT:    movdqa %xmm0, %xmm23136; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]3137; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]3138; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]3139; SSE2-NEXT:    movdqa %xmm0, %xmm33140; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]3141; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]3142; SSE2-NEXT:    paddb 16(%rdx), %xmm03143; SSE2-NEXT:    paddb (%rdx), %xmm33144; SSE2-NEXT:    paddb 32(%rdx), %xmm23145; SSE2-NEXT:    movdqa %xmm2, 32(%rcx)3146; SSE2-NEXT:    movdqa %xmm3, (%rcx)3147; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)3148; SSE2-NEXT:    retq3149;3150; SSE42-LABEL: vec384_v48i8_to_v6i64_factor8:3151; SSE42:       # %bb.0:3152; SSE42-NEXT:    movdqa (%rdi), %xmm03153; SSE42-NEXT:    paddb (%rsi), %xmm03154; SSE42-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero3155; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]3156; SSE42-NEXT:    pmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero3157; SSE42-NEXT:    psrld $16, %xmm03158; SSE42-NEXT:    pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero3159; SSE42-NEXT:    paddb 16(%rdx), %xmm03160; SSE42-NEXT:    paddb 32(%rdx), %xmm23161; SSE42-NEXT:    paddb (%rdx), %xmm13162; SSE42-NEXT:    movdqa %xmm1, (%rcx)3163; SSE42-NEXT:    movdqa %xmm2, 32(%rcx)3164; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)3165; SSE42-NEXT:    retq3166;3167; AVX-LABEL: vec384_v48i8_to_v6i64_factor8:3168; AVX:       # %bb.0:3169; AVX-NEXT:    vmovdqa (%rdi), %xmm03170; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm03171; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero3172; AVX-NEXT:    vpsrld $16, %xmm0, %xmm23173; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero3174; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]3175; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero3176; AVX-NEXT:    vpaddb 32(%rdx), %xmm0, %xmm03177; AVX-NEXT:    vpaddb 16(%rdx), %xmm2, %xmm23178; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm13179; AVX-NEXT:    vmovdqa %xmm1, (%rcx)3180; AVX-NEXT:    vmovdqa %xmm2, 16(%rcx)3181; AVX-NEXT:    vmovdqa %xmm0, 32(%rcx)3182; AVX-NEXT:    retq3183;3184; AVX2-SLOW-LABEL: vec384_v48i8_to_v6i64_factor8:3185; AVX2-SLOW:       # %bb.0:3186; AVX2-SLOW-NEXT:    vmovdqa (%rdi), %xmm03187; AVX2-SLOW-NEXT:    vpaddb (%rsi), %xmm0, %xmm03188; AVX2-SLOW-NEXT:    vpmovzxbq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero3189; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]3190; AVX2-SLOW-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero3191; AVX2-SLOW-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm03192; AVX2-SLOW-NEXT:    vpaddb (%rdx), %ymm1, %ymm13193; AVX2-SLOW-NEXT:    vmovdqa %ymm1, (%rcx)3194; AVX2-SLOW-NEXT:    vmovdqa %ymm0, 32(%rcx)3195; AVX2-SLOW-NEXT:    vzeroupper3196; AVX2-SLOW-NEXT:    retq3197;3198; AVX2-FAST-PERLANE-LABEL: vec384_v48i8_to_v6i64_factor8:3199; AVX2-FAST-PERLANE:       # %bb.0:3200; AVX2-FAST-PERLANE-NEXT:    vmovdqa (%rdi), %xmm03201; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rsi), %xmm0, %xmm03202; AVX2-FAST-PERLANE-NEXT:    vpmovzxbq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero3203; AVX2-FAST-PERLANE-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4],zero,zero,zero,zero,zero,zero,zero,xmm0[5],zero,zero,zero,zero,zero,zero,zero3204; AVX2-FAST-PERLANE-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm03205; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rdx), %ymm1, %ymm13206; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm1, (%rcx)3207; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm0, 32(%rcx)3208; AVX2-FAST-PERLANE-NEXT:    vzeroupper3209; AVX2-FAST-PERLANE-NEXT:    retq3210;3211; AVX2-FAST-LABEL: vec384_v48i8_to_v6i64_factor8:3212; AVX2-FAST:       # %bb.0:3213; AVX2-FAST-NEXT:    vmovdqa (%rdi), %xmm03214; AVX2-FAST-NEXT:    vpaddb (%rsi), %xmm0, %xmm03215; AVX2-FAST-NEXT:    vpmovzxbq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero3216; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4],zero,zero,zero,zero,zero,zero,zero,xmm0[5],zero,zero,zero,zero,zero,zero,zero3217; AVX2-FAST-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm03218; AVX2-FAST-NEXT:    vpaddb (%rdx), %ymm1, %ymm13219; AVX2-FAST-NEXT:    vmovdqa %ymm1, (%rcx)3220; AVX2-FAST-NEXT:    vmovdqa %ymm0, 32(%rcx)3221; AVX2-FAST-NEXT:    vzeroupper3222; AVX2-FAST-NEXT:    retq3223;3224; AVX512F-LABEL: vec384_v48i8_to_v6i64_factor8:3225; AVX512F:       # %bb.0:3226; AVX512F-NEXT:    vmovdqa (%rdi), %xmm03227; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm03228; AVX512F-NEXT:    vpmovzxbq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero3229; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4],zero,zero,zero,zero,zero,zero,zero,xmm0[5],zero,zero,zero,zero,zero,zero,zero3230; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm03231; AVX512F-NEXT:    vpaddb (%rdx), %ymm1, %ymm13232; AVX512F-NEXT:    vmovdqa %ymm1, (%rcx)3233; AVX512F-NEXT:    vmovdqa %ymm0, 32(%rcx)3234; AVX512F-NEXT:    vzeroupper3235; AVX512F-NEXT:    retq3236;3237; AVX512BW-LABEL: vec384_v48i8_to_v6i64_factor8:3238; AVX512BW:       # %bb.0:3239; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm03240; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm03241; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero,xmm0[4],zero,zero,zero,zero,zero,zero,zero,xmm0[5],zero,zero,zero,zero,zero,zero,zero,xmm0[6],zero,zero,zero,zero,zero,zero,zero,xmm0[7],zero,zero,zero,zero,zero,zero,zero3242; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm03243; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)3244; AVX512BW-NEXT:    vzeroupper3245; AVX512BW-NEXT:    retq3246  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 643247  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 643248  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias3249  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>3250  %zextd.vec = shufflevector <48 x i8> %in.vec.trunc, <48 x i8> zeroinitializer, <48 x i32> <i32 0, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 1, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 2, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 3, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 4, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 5, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95>3251  %out.bytevec.padded = shufflevector <48 x i8> %zextd.vec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>3252  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 643253  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias3254  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 643255  ret void3256}3257 3258define void @vec384_v48i8_to_v4i96_factor12(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {3259; SSE2-LABEL: vec384_v48i8_to_v4i96_factor12:3260; SSE2:       # %bb.0:3261; SSE2-NEXT:    movdqa (%rdi), %xmm03262; SSE2-NEXT:    paddb (%rsi), %xmm03263; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]3264; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero3265; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,xmm1[0,1,2,3,4,5,6,7,8,9,10,11]3266; SSE2-NEXT:    movdqa %xmm0, %xmm23267; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2]3268; SSE2-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero3269; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2,3,4,5,6,7]3270; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]3271; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]3272; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1]3273; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm03274; SSE2-NEXT:    paddb (%rdx), %xmm03275; SSE2-NEXT:    paddb 16(%rdx), %xmm23276; SSE2-NEXT:    paddb 32(%rdx), %xmm13277; SSE2-NEXT:    movdqa %xmm1, 32(%rcx)3278; SSE2-NEXT:    movdqa %xmm2, 16(%rcx)3279; SSE2-NEXT:    movdqa %xmm0, (%rcx)3280; SSE2-NEXT:    retq3281;3282; SSE42-LABEL: vec384_v48i8_to_v4i96_factor12:3283; SSE42:       # %bb.0:3284; SSE42-NEXT:    movdqa (%rdi), %xmm03285; SSE42-NEXT:    paddb (%rsi), %xmm03286; SSE42-NEXT:    movdqa %xmm0, %xmm13287; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = zero,zero,zero,zero,xmm1[3],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero3288; SSE42-NEXT:    movdqa %xmm0, %xmm23289; SSE42-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero3290; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero3291; SSE42-NEXT:    paddb 16(%rdx), %xmm03292; SSE42-NEXT:    paddb (%rdx), %xmm23293; SSE42-NEXT:    paddb 32(%rdx), %xmm13294; SSE42-NEXT:    movdqa %xmm1, 32(%rcx)3295; SSE42-NEXT:    movdqa %xmm2, (%rcx)3296; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)3297; SSE42-NEXT:    retq3298;3299; AVX-LABEL: vec384_v48i8_to_v4i96_factor12:3300; AVX:       # %bb.0:3301; AVX-NEXT:    vmovdqa (%rdi), %xmm03302; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm03303; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero3304; AVX-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero3305; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero3306; AVX-NEXT:    vpaddb 32(%rdx), %xmm0, %xmm03307; AVX-NEXT:    vpaddb 16(%rdx), %xmm2, %xmm23308; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm13309; AVX-NEXT:    vmovdqa %xmm1, (%rcx)3310; AVX-NEXT:    vmovdqa %xmm2, 16(%rcx)3311; AVX-NEXT:    vmovdqa %xmm0, 32(%rcx)3312; AVX-NEXT:    retq3313;3314; AVX2-LABEL: vec384_v48i8_to_v4i96_factor12:3315; AVX2:       # %bb.0:3316; AVX2-NEXT:    vmovdqa (%rdi), %xmm03317; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm03318; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero3319; AVX2-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero3320; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]3321; AVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm03322; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm03323; AVX2-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm13324; AVX2-NEXT:    vmovdqa %ymm1, 32(%rcx)3325; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)3326; AVX2-NEXT:    vzeroupper3327; AVX2-NEXT:    retq3328;3329; AVX512F-LABEL: vec384_v48i8_to_v4i96_factor12:3330; AVX512F:       # %bb.0:3331; AVX512F-NEXT:    vmovdqa (%rdi), %xmm03332; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm03333; AVX512F-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero3334; AVX512F-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero3335; AVX512F-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]3336; AVX512F-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm03337; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm03338; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm13339; AVX512F-NEXT:    vmovdqa %ymm1, 32(%rcx)3340; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)3341; AVX512F-NEXT:    vzeroupper3342; AVX512F-NEXT:    retq3343;3344; AVX512BW-LABEL: vec384_v48i8_to_v4i96_factor12:3345; AVX512BW:       # %bb.0:3346; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm03347; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm03348; AVX512BW-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero3349; AVX512BW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]3350; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm13351; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero3352; AVX512BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm03353; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm03354; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)3355; AVX512BW-NEXT:    vzeroupper3356; AVX512BW-NEXT:    retq3357  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 643358  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 643359  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias3360  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>3361  %zextd.vec = shufflevector <48 x i8> %in.vec.trunc, <48 x i8> zeroinitializer, <48 x i32> <i32 0, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 1, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 2, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 3, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95>3362  %out.bytevec.padded = shufflevector <48 x i8> %zextd.vec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>3363  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 643364  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias3365  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 643366  ret void3367}3368 3369define void @vec384_v48i8_to_v3i128_factor16(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {3370; SSE2-LABEL: vec384_v48i8_to_v3i128_factor16:3371; SSE2:       # %bb.0:3372; SSE2-NEXT:    movdqa (%rdi), %xmm03373; SSE2-NEXT:    paddb (%rsi), %xmm03374; SSE2-NEXT:    movd {{.*#+}} xmm1 = [255,0,0,0]3375; SSE2-NEXT:    pand %xmm0, %xmm13376; SSE2-NEXT:    movdqa %xmm0, %xmm23377; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2]3378; SSE2-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero3379; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1]3380; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero3381; SSE2-NEXT:    paddb 16(%rdx), %xmm03382; SSE2-NEXT:    paddb 32(%rdx), %xmm23383; SSE2-NEXT:    paddb (%rdx), %xmm13384; SSE2-NEXT:    movdqa %xmm1, (%rcx)3385; SSE2-NEXT:    movdqa %xmm2, 32(%rcx)3386; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)3387; SSE2-NEXT:    retq3388;3389; SSE42-LABEL: vec384_v48i8_to_v3i128_factor16:3390; SSE42:       # %bb.0:3391; SSE42-NEXT:    movdqa (%rdi), %xmm03392; SSE42-NEXT:    paddb (%rsi), %xmm03393; SSE42-NEXT:    pmovzxbq {{.*#+}} xmm1 = [255,0]3394; SSE42-NEXT:    pand %xmm0, %xmm13395; SSE42-NEXT:    movdqa %xmm0, %xmm23396; SSE42-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2]3397; SSE42-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero3398; SSE42-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1]3399; SSE42-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero3400; SSE42-NEXT:    paddb 16(%rdx), %xmm03401; SSE42-NEXT:    paddb 32(%rdx), %xmm23402; SSE42-NEXT:    paddb (%rdx), %xmm13403; SSE42-NEXT:    movdqa %xmm1, (%rcx)3404; SSE42-NEXT:    movdqa %xmm2, 32(%rcx)3405; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)3406; SSE42-NEXT:    retq3407;3408; AVX-LABEL: vec384_v48i8_to_v3i128_factor16:3409; AVX:       # %bb.0:3410; AVX-NEXT:    vmovdqa (%rdi), %xmm03411; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm03412; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm13413; AVX-NEXT:    vpslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1]3414; AVX-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm2[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero3415; AVX-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2]3416; AVX-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero3417; AVX-NEXT:    vpaddb 32(%rdx), %xmm0, %xmm03418; AVX-NEXT:    vpaddb 16(%rdx), %xmm2, %xmm23419; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm13420; AVX-NEXT:    vmovdqa %xmm1, (%rcx)3421; AVX-NEXT:    vmovdqa %xmm2, 16(%rcx)3422; AVX-NEXT:    vmovdqa %xmm0, 32(%rcx)3423; AVX-NEXT:    retq3424;3425; AVX2-SLOW-LABEL: vec384_v48i8_to_v3i128_factor16:3426; AVX2-SLOW:       # %bb.0:3427; AVX2-SLOW-NEXT:    vmovdqa (%rdi), %xmm03428; AVX2-SLOW-NEXT:    vpaddb (%rsi), %xmm0, %xmm03429; AVX2-SLOW-NEXT:    vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2]3430; AVX2-SLOW-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero3431; AVX2-SLOW-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero3432; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]3433; AVX2-SLOW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm03434; AVX2-SLOW-NEXT:    vpaddb (%rdx), %ymm0, %ymm03435; AVX2-SLOW-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm13436; AVX2-SLOW-NEXT:    vmovdqa %ymm1, 32(%rcx)3437; AVX2-SLOW-NEXT:    vmovdqa %ymm0, (%rcx)3438; AVX2-SLOW-NEXT:    vzeroupper3439; AVX2-SLOW-NEXT:    retq3440;3441; AVX2-FAST-PERLANE-LABEL: vec384_v48i8_to_v3i128_factor16:3442; AVX2-FAST-PERLANE:       # %bb.0:3443; AVX2-FAST-PERLANE-NEXT:    vmovdqa (%rdi), %xmm03444; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rsi), %xmm0, %xmm03445; AVX2-FAST-PERLANE-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[2],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero3446; AVX2-FAST-PERLANE-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero3447; AVX2-FAST-PERLANE-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]3448; AVX2-FAST-PERLANE-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm03449; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rdx), %ymm0, %ymm03450; AVX2-FAST-PERLANE-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm13451; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm1, 32(%rcx)3452; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm0, (%rcx)3453; AVX2-FAST-PERLANE-NEXT:    vzeroupper3454; AVX2-FAST-PERLANE-NEXT:    retq3455;3456; AVX2-FAST-LABEL: vec384_v48i8_to_v3i128_factor16:3457; AVX2-FAST:       # %bb.0:3458; AVX2-FAST-NEXT:    vmovdqa (%rdi), %xmm03459; AVX2-FAST-NEXT:    vpaddb (%rsi), %xmm0, %xmm03460; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[2],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero3461; AVX2-FAST-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero3462; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]3463; AVX2-FAST-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm03464; AVX2-FAST-NEXT:    vpaddb (%rdx), %ymm0, %ymm03465; AVX2-FAST-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm13466; AVX2-FAST-NEXT:    vmovdqa %ymm1, 32(%rcx)3467; AVX2-FAST-NEXT:    vmovdqa %ymm0, (%rcx)3468; AVX2-FAST-NEXT:    vzeroupper3469; AVX2-FAST-NEXT:    retq3470;3471; AVX512F-LABEL: vec384_v48i8_to_v3i128_factor16:3472; AVX512F:       # %bb.0:3473; AVX512F-NEXT:    vmovdqa (%rdi), %xmm03474; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm03475; AVX512F-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[2],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero3476; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero3477; AVX512F-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]3478; AVX512F-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm03479; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm03480; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm13481; AVX512F-NEXT:    vmovdqa %ymm1, 32(%rcx)3482; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)3483; AVX512F-NEXT:    vzeroupper3484; AVX512F-NEXT:    retq3485;3486; AVX512BW-LABEL: vec384_v48i8_to_v3i128_factor16:3487; AVX512BW:       # %bb.0:3488; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm03489; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm03490; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero3491; AVX512BW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]3492; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm13493; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero3494; AVX512BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm03495; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm03496; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)3497; AVX512BW-NEXT:    vzeroupper3498; AVX512BW-NEXT:    retq3499  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 643500  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 643501  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias3502  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>3503  %zextd.vec = shufflevector <48 x i8> %in.vec.trunc, <48 x i8> zeroinitializer, <48 x i32> <i32 0, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 1, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 2, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95>3504  %out.bytevec.padded = shufflevector <48 x i8> %zextd.vec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>3505  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 643506  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias3507  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 643508  ret void3509}3510 3511define void @vec384_v48i8_to_v2i192_factor24(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {3512; SSE2-LABEL: vec384_v48i8_to_v2i192_factor24:3513; SSE2:       # %bb.0:3514; SSE2-NEXT:    movdqa (%rdi), %xmm03515; SSE2-NEXT:    paddb (%rsi), %xmm03516; SSE2-NEXT:    movd {{.*#+}} xmm1 = [255,0,0,0]3517; SSE2-NEXT:    pand %xmm0, %xmm13518; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1]3519; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero3520; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]3521; SSE2-NEXT:    movaps 32(%rdx), %xmm23522; SSE2-NEXT:    paddb 16(%rdx), %xmm03523; SSE2-NEXT:    paddb (%rdx), %xmm13524; SSE2-NEXT:    movaps %xmm2, 32(%rcx)3525; SSE2-NEXT:    movdqa %xmm1, (%rcx)3526; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)3527; SSE2-NEXT:    retq3528;3529; SSE42-LABEL: vec384_v48i8_to_v2i192_factor24:3530; SSE42:       # %bb.0:3531; SSE42-NEXT:    movdqa (%rdi), %xmm03532; SSE42-NEXT:    paddb (%rsi), %xmm03533; SSE42-NEXT:    movdqa %xmm0, %xmm13534; SSE42-NEXT:    pshufb {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero3535; SSE42-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm03536; SSE42-NEXT:    movaps 32(%rdx), %xmm23537; SSE42-NEXT:    paddb (%rdx), %xmm03538; SSE42-NEXT:    paddb 16(%rdx), %xmm13539; SSE42-NEXT:    movaps %xmm2, 32(%rcx)3540; SSE42-NEXT:    movdqa %xmm1, 16(%rcx)3541; SSE42-NEXT:    movdqa %xmm0, (%rcx)3542; SSE42-NEXT:    retq3543;3544; AVX-LABEL: vec384_v48i8_to_v2i192_factor24:3545; AVX:       # %bb.0:3546; AVX-NEXT:    vmovdqa (%rdi), %xmm03547; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm03548; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm13549; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero3550; AVX-NEXT:    vmovaps 32(%rdx), %ymm23551; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm03552; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm13553; AVX-NEXT:    vmovaps %ymm2, 32(%rcx)3554; AVX-NEXT:    vmovdqa %xmm1, (%rcx)3555; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)3556; AVX-NEXT:    vzeroupper3557; AVX-NEXT:    retq3558;3559; AVX2-LABEL: vec384_v48i8_to_v2i192_factor24:3560; AVX2:       # %bb.0:3561; AVX2-NEXT:    vmovdqa (%rdi), %xmm03562; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm03563; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero3564; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,2,1]3565; AVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm03566; AVX2-NEXT:    vmovaps 32(%rdx), %ymm13567; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm03568; AVX2-NEXT:    vmovaps %ymm1, 32(%rcx)3569; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)3570; AVX2-NEXT:    vzeroupper3571; AVX2-NEXT:    retq3572;3573; AVX512F-LABEL: vec384_v48i8_to_v2i192_factor24:3574; AVX512F:       # %bb.0:3575; AVX512F-NEXT:    vmovdqa (%rdi), %xmm03576; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm03577; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero3578; AVX512F-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,2,1]3579; AVX512F-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm03580; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm03581; AVX512F-NEXT:    vmovaps 32(%rdx), %ymm13582; AVX512F-NEXT:    vmovaps %ymm1, 32(%rcx)3583; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)3584; AVX512F-NEXT:    vzeroupper3585; AVX512F-NEXT:    retq3586;3587; AVX512BW-LABEL: vec384_v48i8_to_v2i192_factor24:3588; AVX512BW:       # %bb.0:3589; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm03590; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm03591; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero3592; AVX512BW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,2,1]3593; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm03594; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm03595; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)3596; AVX512BW-NEXT:    vzeroupper3597; AVX512BW-NEXT:    retq3598  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 643599  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 643600  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias3601  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>3602  %zextd.vec = shufflevector <48 x i8> %in.vec.trunc, <48 x i8> zeroinitializer, <48 x i32> <i32 0, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 1, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95>3603  %out.bytevec.padded = shufflevector <48 x i8> %zextd.vec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>3604  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 643605  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias3606  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 643607  ret void3608}3609 3610define void @vec384_v48i8_to_v1i384_factor48(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {3611; SSE-LABEL: vec384_v48i8_to_v1i384_factor48:3612; SSE:       # %bb.0:3613; SSE-NEXT:    movdqa (%rdi), %xmm03614; SSE-NEXT:    paddb (%rsi), %xmm03615; SSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm03616; SSE-NEXT:    movaps 16(%rdx), %xmm13617; SSE-NEXT:    movaps 32(%rdx), %xmm23618; SSE-NEXT:    paddb (%rdx), %xmm03619; SSE-NEXT:    movaps %xmm1, 16(%rcx)3620; SSE-NEXT:    movaps %xmm2, 32(%rcx)3621; SSE-NEXT:    movdqa %xmm0, (%rcx)3622; SSE-NEXT:    retq3623;3624; AVX-LABEL: vec384_v48i8_to_v1i384_factor48:3625; AVX:       # %bb.0:3626; AVX-NEXT:    vmovdqa (%rdi), %xmm03627; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm03628; AVX-NEXT:    vmovaps 32(%rdx), %ymm13629; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm03630; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm03631; AVX-NEXT:    vmovaps 16(%rdx), %xmm23632; AVX-NEXT:    vmovaps %xmm2, 16(%rcx)3633; AVX-NEXT:    vmovaps %ymm1, 32(%rcx)3634; AVX-NEXT:    vmovdqa %xmm0, (%rcx)3635; AVX-NEXT:    vzeroupper3636; AVX-NEXT:    retq3637;3638; AVX2-LABEL: vec384_v48i8_to_v1i384_factor48:3639; AVX2:       # %bb.0:3640; AVX2-NEXT:    vmovdqa (%rdi), %ymm03641; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm03642; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = [255,0]3643; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm03644; AVX2-NEXT:    vmovaps 32(%rdx), %ymm13645; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm03646; AVX2-NEXT:    vmovaps %ymm1, 32(%rcx)3647; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)3648; AVX2-NEXT:    vzeroupper3649; AVX2-NEXT:    retq3650;3651; AVX512F-LABEL: vec384_v48i8_to_v1i384_factor48:3652; AVX512F:       # %bb.0:3653; AVX512F-NEXT:    vmovdqa (%rdi), %ymm03654; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm03655; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm1 = [255,0]3656; AVX512F-NEXT:    vpand %ymm1, %ymm0, %ymm03657; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm03658; AVX512F-NEXT:    vmovaps 32(%rdx), %ymm13659; AVX512F-NEXT:    vmovaps %ymm1, 32(%rcx)3660; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)3661; AVX512F-NEXT:    vzeroupper3662; AVX512F-NEXT:    retq3663;3664; AVX512BW-LABEL: vec384_v48i8_to_v1i384_factor48:3665; AVX512BW:       # %bb.0:3666; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm03667; AVX512BW-NEXT:    vpaddb (%rsi), %zmm0, %zmm03668; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = [255,0]3669; AVX512BW-NEXT:    vpandq %zmm1, %zmm0, %zmm03670; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm03671; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)3672; AVX512BW-NEXT:    vzeroupper3673; AVX512BW-NEXT:    retq3674  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 643675  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 643676  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias3677  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>3678  %zextd.vec = shufflevector <48 x i8> %in.vec.trunc, <48 x i8> zeroinitializer, <48 x i32> <i32 0, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95>3679  %out.bytevec.padded = shufflevector <48 x i8> %zextd.vec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>3680  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 643681  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias3682  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 643683  ret void3684}3685 3686define void @vec384_v24i16_to_v12i32_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {3687; SSE2-LABEL: vec384_v24i16_to_v12i32_factor2:3688; SSE2:       # %bb.0:3689; SSE2-NEXT:    movdqa (%rdi), %xmm03690; SSE2-NEXT:    movdqa 16(%rdi), %xmm13691; SSE2-NEXT:    paddb (%rsi), %xmm03692; SSE2-NEXT:    paddb 16(%rsi), %xmm13693; SSE2-NEXT:    pxor %xmm2, %xmm23694; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]3695; SSE2-NEXT:    movdqa %xmm0, %xmm33696; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]3697; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]3698; SSE2-NEXT:    paddb 16(%rdx), %xmm03699; SSE2-NEXT:    paddb (%rdx), %xmm33700; SSE2-NEXT:    paddb 32(%rdx), %xmm13701; SSE2-NEXT:    movdqa %xmm1, 32(%rcx)3702; SSE2-NEXT:    movdqa %xmm3, (%rcx)3703; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)3704; SSE2-NEXT:    retq3705;3706; SSE42-LABEL: vec384_v24i16_to_v12i32_factor2:3707; SSE42:       # %bb.0:3708; SSE42-NEXT:    movdqa (%rdi), %xmm03709; SSE42-NEXT:    movdqa 16(%rdi), %xmm13710; SSE42-NEXT:    paddb (%rsi), %xmm03711; SSE42-NEXT:    paddb 16(%rsi), %xmm13712; SSE42-NEXT:    pxor %xmm2, %xmm23713; SSE42-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero3714; SSE42-NEXT:    pmovzxwd {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero3715; SSE42-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]3716; SSE42-NEXT:    paddb 16(%rdx), %xmm03717; SSE42-NEXT:    paddb (%rdx), %xmm33718; SSE42-NEXT:    paddb 32(%rdx), %xmm13719; SSE42-NEXT:    movdqa %xmm1, 32(%rcx)3720; SSE42-NEXT:    movdqa %xmm3, (%rcx)3721; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)3722; SSE42-NEXT:    retq3723;3724; AVX-LABEL: vec384_v24i16_to_v12i32_factor2:3725; AVX:       # %bb.0:3726; AVX-NEXT:    vmovdqa (%rdi), %xmm03727; AVX-NEXT:    vmovdqa 16(%rdi), %xmm13728; AVX-NEXT:    vpaddb 16(%rsi), %xmm1, %xmm13729; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm03730; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero3731; AVX-NEXT:    vpxor %xmm3, %xmm3, %xmm33732; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]3733; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero3734; AVX-NEXT:    vpaddb 32(%rdx), %xmm1, %xmm13735; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm03736; AVX-NEXT:    vpaddb (%rdx), %xmm2, %xmm23737; AVX-NEXT:    vmovdqa %xmm2, (%rcx)3738; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)3739; AVX-NEXT:    vmovdqa %xmm1, 32(%rcx)3740; AVX-NEXT:    retq3741;3742; AVX2-LABEL: vec384_v24i16_to_v12i32_factor2:3743; AVX2:       # %bb.0:3744; AVX2-NEXT:    vmovdqa (%rdi), %ymm03745; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm03746; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero3747; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm03748; AVX2-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero3749; AVX2-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm03750; AVX2-NEXT:    vpaddb (%rdx), %ymm1, %ymm13751; AVX2-NEXT:    vmovdqa %ymm1, (%rcx)3752; AVX2-NEXT:    vmovdqa %ymm0, 32(%rcx)3753; AVX2-NEXT:    vzeroupper3754; AVX2-NEXT:    retq3755;3756; AVX512F-LABEL: vec384_v24i16_to_v12i32_factor2:3757; AVX512F:       # %bb.0:3758; AVX512F-NEXT:    vmovdqa (%rdi), %ymm03759; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm03760; AVX512F-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero3761; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm03762; AVX512F-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero3763; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm03764; AVX512F-NEXT:    vpaddb (%rdx), %ymm1, %ymm13765; AVX512F-NEXT:    vmovdqa %ymm1, (%rcx)3766; AVX512F-NEXT:    vmovdqa %ymm0, 32(%rcx)3767; AVX512F-NEXT:    vzeroupper3768; AVX512F-NEXT:    retq3769;3770; AVX512BW-LABEL: vec384_v24i16_to_v12i32_factor2:3771; AVX512BW:       # %bb.0:3772; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm03773; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm03774; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero3775; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm03776; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero3777; AVX512BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm03778; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm03779; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)3780; AVX512BW-NEXT:    vzeroupper3781; AVX512BW-NEXT:    retq3782  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 643783  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 643784  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias3785  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>3786  %in.vec.cast = bitcast <48 x i8> %in.vec.trunc to <24 x i16>3787  %zextd.vec = shufflevector <24 x i16> %in.vec.cast, <24 x i16> zeroinitializer, <24 x i32> <i32 0, i32 25, i32 1, i32 27, i32 2, i32 29, i32 3, i32 31, i32 4, i32 33, i32 5, i32 35, i32 6, i32 37, i32 7, i32 39, i32 8, i32 41, i32 9, i32 43, i32 10, i32 45, i32 11, i32 47>3788  %out.bytevec = bitcast <24 x i16> %zextd.vec to <48 x i8>3789  %out.bytevec.padded = shufflevector <48 x i8> %out.bytevec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>3790  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 643791  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias3792  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 643793  ret void3794}3795 3796define void @vec384_v24i16_to_v8i48_factor3(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {3797; SSE2-LABEL: vec384_v24i16_to_v8i48_factor3:3798; SSE2:       # %bb.0:3799; SSE2-NEXT:    movdqa (%rdi), %xmm03800; SSE2-NEXT:    paddb (%rsi), %xmm03801; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]3802; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm13803; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,1,1]3804; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm23805; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,2]3806; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm03807; SSE2-NEXT:    paddb 16(%rdx), %xmm03808; SSE2-NEXT:    paddb (%rdx), %xmm23809; SSE2-NEXT:    paddb 32(%rdx), %xmm13810; SSE2-NEXT:    movdqa %xmm1, 32(%rcx)3811; SSE2-NEXT:    movdqa %xmm2, (%rcx)3812; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)3813; SSE2-NEXT:    retq3814;3815; SSE42-LABEL: vec384_v24i16_to_v8i48_factor3:3816; SSE42:       # %bb.0:3817; SSE42-NEXT:    movdqa (%rdi), %xmm03818; SSE42-NEXT:    paddb (%rsi), %xmm03819; SSE42-NEXT:    pxor %xmm1, %xmm13820; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]3821; SSE42-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]3822; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,1,1]3823; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0],xmm1[1,2],xmm3[3],xmm1[4,5],xmm3[6],xmm1[7]3824; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,2]3825; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3],xmm0[4],xmm1[5,6],xmm0[7]3826; SSE42-NEXT:    paddb 16(%rdx), %xmm03827; SSE42-NEXT:    paddb (%rdx), %xmm33828; SSE42-NEXT:    paddb 32(%rdx), %xmm23829; SSE42-NEXT:    movdqa %xmm2, 32(%rcx)3830; SSE42-NEXT:    movdqa %xmm3, (%rcx)3831; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)3832; SSE42-NEXT:    retq3833;3834; AVX-LABEL: vec384_v24i16_to_v8i48_factor3:3835; AVX:       # %bb.0:3836; AVX-NEXT:    vmovdqa (%rdi), %xmm03837; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm03838; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,0,1,1]3839; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm23840; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2],xmm1[3],xmm2[4,5],xmm1[6],xmm2[7]3841; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,2]3842; AVX-NEXT:    vpblendw {{.*#+}} xmm3 = xmm2[0],xmm3[1],xmm2[2,3],xmm3[4],xmm2[5,6],xmm3[7]3843; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]3844; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]3845; AVX-NEXT:    vpaddb 32(%rdx), %xmm0, %xmm03846; AVX-NEXT:    vpaddb 16(%rdx), %xmm3, %xmm23847; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm13848; AVX-NEXT:    vmovdqa %xmm1, (%rcx)3849; AVX-NEXT:    vmovdqa %xmm2, 16(%rcx)3850; AVX-NEXT:    vmovdqa %xmm0, 32(%rcx)3851; AVX-NEXT:    retq3852;3853; AVX2-SLOW-LABEL: vec384_v24i16_to_v8i48_factor3:3854; AVX2-SLOW:       # %bb.0:3855; AVX2-SLOW-NEXT:    vmovdqa (%rdi), %ymm03856; AVX2-SLOW-NEXT:    vpaddb (%rsi), %ymm0, %ymm03857; AVX2-SLOW-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [0,0,0,1,1,0,2,2]3858; AVX2-SLOW-NEXT:    vpermd %ymm0, %ymm1, %ymm13859; AVX2-SLOW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm13860; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]3861; AVX2-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm23862; AVX2-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]3863; AVX2-SLOW-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm03864; AVX2-SLOW-NEXT:    vpaddb (%rdx), %ymm1, %ymm13865; AVX2-SLOW-NEXT:    vmovdqa %ymm1, (%rcx)3866; AVX2-SLOW-NEXT:    vmovdqa %ymm0, 32(%rcx)3867; AVX2-SLOW-NEXT:    vzeroupper3868; AVX2-SLOW-NEXT:    retq3869;3870; AVX2-FAST-PERLANE-LABEL: vec384_v24i16_to_v8i48_factor3:3871; AVX2-FAST-PERLANE:       # %bb.0:3872; AVX2-FAST-PERLANE-NEXT:    vmovdqa (%rdi), %ymm03873; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rsi), %ymm0, %ymm03874; AVX2-FAST-PERLANE-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [0,0,0,1,1,0,2,2]3875; AVX2-FAST-PERLANE-NEXT:    vpermd %ymm0, %ymm1, %ymm13876; AVX2-FAST-PERLANE-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm13877; AVX2-FAST-PERLANE-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[12,13],zero,zero,zero,zero,xmm0[14,15],zero,zero,zero,zero3878; AVX2-FAST-PERLANE-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm03879; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rdx), %ymm1, %ymm13880; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm1, (%rcx)3881; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm0, 32(%rcx)3882; AVX2-FAST-PERLANE-NEXT:    vzeroupper3883; AVX2-FAST-PERLANE-NEXT:    retq3884;3885; AVX2-FAST-LABEL: vec384_v24i16_to_v8i48_factor3:3886; AVX2-FAST:       # %bb.0:3887; AVX2-FAST-NEXT:    vmovdqa (%rdi), %ymm03888; AVX2-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm03889; AVX2-FAST-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [0,0,0,1,1,0,2,2]3890; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm1, %ymm13891; AVX2-FAST-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm13892; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[12,13],zero,zero,zero,zero,xmm0[14,15],zero,zero,zero,zero3893; AVX2-FAST-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm03894; AVX2-FAST-NEXT:    vpaddb (%rdx), %ymm1, %ymm13895; AVX2-FAST-NEXT:    vmovdqa %ymm1, (%rcx)3896; AVX2-FAST-NEXT:    vmovdqa %ymm0, 32(%rcx)3897; AVX2-FAST-NEXT:    vzeroupper3898; AVX2-FAST-NEXT:    retq3899;3900; AVX512F-SLOW-LABEL: vec384_v24i16_to_v8i48_factor3:3901; AVX512F-SLOW:       # %bb.0:3902; AVX512F-SLOW-NEXT:    vmovdqa (%rdi), %ymm03903; AVX512F-SLOW-NEXT:    vpaddb (%rsi), %ymm0, %ymm03904; AVX512F-SLOW-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [0,0,0,1,1,0,2,2]3905; AVX512F-SLOW-NEXT:    vpermd %ymm0, %ymm1, %ymm13906; AVX512F-SLOW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm13907; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]3908; AVX512F-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm23909; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]3910; AVX512F-SLOW-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm03911; AVX512F-SLOW-NEXT:    vpaddb (%rdx), %ymm1, %ymm13912; AVX512F-SLOW-NEXT:    vmovdqa %ymm1, (%rcx)3913; AVX512F-SLOW-NEXT:    vmovdqa %ymm0, 32(%rcx)3914; AVX512F-SLOW-NEXT:    vzeroupper3915; AVX512F-SLOW-NEXT:    retq3916;3917; AVX512F-FAST-LABEL: vec384_v24i16_to_v8i48_factor3:3918; AVX512F-FAST:       # %bb.0:3919; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %ymm03920; AVX512F-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm03921; AVX512F-FAST-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [0,0,0,1,1,0,2,2]3922; AVX512F-FAST-NEXT:    vpermd %ymm0, %ymm1, %ymm13923; AVX512F-FAST-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm13924; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[12,13],zero,zero,zero,zero,xmm0[14,15],zero,zero,zero,zero3925; AVX512F-FAST-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm03926; AVX512F-FAST-NEXT:    vpaddb (%rdx), %ymm1, %ymm13927; AVX512F-FAST-NEXT:    vmovdqa %ymm1, (%rcx)3928; AVX512F-FAST-NEXT:    vmovdqa %ymm0, 32(%rcx)3929; AVX512F-FAST-NEXT:    vzeroupper3930; AVX512F-FAST-NEXT:    retq3931;3932; AVX512BW-SLOW-LABEL: vec384_v24i16_to_v8i48_factor3:3933; AVX512BW-SLOW:       # %bb.0:3934; AVX512BW-SLOW-NEXT:    vmovdqa (%rdi), %ymm03935; AVX512BW-SLOW-NEXT:    vpaddb (%rsi), %ymm0, %ymm03936; AVX512BW-SLOW-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [16,1,2,17,4,5,18,7,8,19,10,11,20,13,14,21]3937; AVX512BW-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm23938; AVX512BW-SLOW-NEXT:    vpermt2w %ymm0, %ymm1, %ymm23939; AVX512BW-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]3940; AVX512BW-SLOW-NEXT:    vpxor %xmm1, %xmm1, %xmm13941; AVX512BW-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]3942; AVX512BW-SLOW-NEXT:    vinserti32x4 $2, %xmm0, %zmm2, %zmm03943; AVX512BW-SLOW-NEXT:    vpaddb (%rdx), %zmm0, %zmm03944; AVX512BW-SLOW-NEXT:    vmovdqa64 %zmm0, (%rcx)3945; AVX512BW-SLOW-NEXT:    vzeroupper3946; AVX512BW-SLOW-NEXT:    retq3947;3948; AVX512BW-FAST-LABEL: vec384_v24i16_to_v8i48_factor3:3949; AVX512BW-FAST:       # %bb.0:3950; AVX512BW-FAST-NEXT:    vmovdqa (%rdi), %ymm03951; AVX512BW-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm03952; AVX512BW-FAST-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [16,1,2,17,4,5,18,7,8,19,10,11,20,13,14,21]3953; AVX512BW-FAST-NEXT:    vpxor %xmm2, %xmm2, %xmm23954; AVX512BW-FAST-NEXT:    vpermt2w %ymm0, %ymm1, %ymm23955; AVX512BW-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[12,13],zero,zero,zero,zero,xmm0[14,15],zero,zero,zero,zero3956; AVX512BW-FAST-NEXT:    vinserti32x4 $2, %xmm0, %zmm2, %zmm03957; AVX512BW-FAST-NEXT:    vpaddb (%rdx), %zmm0, %zmm03958; AVX512BW-FAST-NEXT:    vmovdqa64 %zmm0, (%rcx)3959; AVX512BW-FAST-NEXT:    vzeroupper3960; AVX512BW-FAST-NEXT:    retq3961  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 643962  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 643963  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias3964  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>3965  %in.vec.cast = bitcast <48 x i8> %in.vec.trunc to <24 x i16>3966  %zextd.vec = shufflevector <24 x i16> %in.vec.cast, <24 x i16> zeroinitializer, <24 x i32> <i32 0, i32 25, i32 26, i32 1, i32 28, i32 29, i32 2, i32 31, i32 32, i32 3, i32 34, i32 35, i32 4, i32 37, i32 38, i32 5, i32 40, i32 41, i32 6, i32 43, i32 44, i32 7, i32 46, i32 47>3967  %out.bytevec = bitcast <24 x i16> %zextd.vec to <48 x i8>3968  %out.bytevec.padded = shufflevector <48 x i8> %out.bytevec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>3969  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 643970  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias3971  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 643972  ret void3973}3974 3975define void @vec384_v24i16_to_v6i64_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {3976; SSE2-LABEL: vec384_v24i16_to_v6i64_factor4:3977; SSE2:       # %bb.0:3978; SSE2-NEXT:    movdqa (%rdi), %xmm03979; SSE2-NEXT:    paddb (%rsi), %xmm03980; SSE2-NEXT:    pxor %xmm1, %xmm13981; SSE2-NEXT:    movdqa %xmm0, %xmm23982; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]3983; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]3984; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]3985; SSE2-NEXT:    movdqa %xmm0, %xmm33986; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]3987; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]3988; SSE2-NEXT:    paddb 16(%rdx), %xmm03989; SSE2-NEXT:    paddb (%rdx), %xmm33990; SSE2-NEXT:    paddb 32(%rdx), %xmm23991; SSE2-NEXT:    movdqa %xmm2, 32(%rcx)3992; SSE2-NEXT:    movdqa %xmm3, (%rcx)3993; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)3994; SSE2-NEXT:    retq3995;3996; SSE42-LABEL: vec384_v24i16_to_v6i64_factor4:3997; SSE42:       # %bb.0:3998; SSE42-NEXT:    movdqa (%rdi), %xmm03999; SSE42-NEXT:    paddb (%rsi), %xmm04000; SSE42-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero4001; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]4002; SSE42-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero4003; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]4004; SSE42-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero4005; SSE42-NEXT:    paddb 16(%rdx), %xmm04006; SSE42-NEXT:    paddb 32(%rdx), %xmm24007; SSE42-NEXT:    paddb (%rdx), %xmm14008; SSE42-NEXT:    movdqa %xmm1, (%rcx)4009; SSE42-NEXT:    movdqa %xmm2, 32(%rcx)4010; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)4011; SSE42-NEXT:    retq4012;4013; AVX-LABEL: vec384_v24i16_to_v6i64_factor4:4014; AVX:       # %bb.0:4015; AVX-NEXT:    vmovdqa (%rdi), %xmm04016; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm04017; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero4018; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]4019; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero4020; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]4021; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero4022; AVX-NEXT:    vpaddb 32(%rdx), %xmm0, %xmm04023; AVX-NEXT:    vpaddb 16(%rdx), %xmm2, %xmm24024; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm14025; AVX-NEXT:    vmovdqa %xmm1, (%rcx)4026; AVX-NEXT:    vmovdqa %xmm2, 16(%rcx)4027; AVX-NEXT:    vmovdqa %xmm0, 32(%rcx)4028; AVX-NEXT:    retq4029;4030; AVX2-SLOW-LABEL: vec384_v24i16_to_v6i64_factor4:4031; AVX2-SLOW:       # %bb.0:4032; AVX2-SLOW-NEXT:    vmovdqa (%rdi), %xmm04033; AVX2-SLOW-NEXT:    vpaddb (%rsi), %xmm0, %xmm04034; AVX2-SLOW-NEXT:    vpmovzxwq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero4035; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]4036; AVX2-SLOW-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero4037; AVX2-SLOW-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm04038; AVX2-SLOW-NEXT:    vpaddb (%rdx), %ymm1, %ymm14039; AVX2-SLOW-NEXT:    vmovdqa %ymm1, (%rcx)4040; AVX2-SLOW-NEXT:    vmovdqa %ymm0, 32(%rcx)4041; AVX2-SLOW-NEXT:    vzeroupper4042; AVX2-SLOW-NEXT:    retq4043;4044; AVX2-FAST-PERLANE-LABEL: vec384_v24i16_to_v6i64_factor4:4045; AVX2-FAST-PERLANE:       # %bb.0:4046; AVX2-FAST-PERLANE-NEXT:    vmovdqa (%rdi), %xmm04047; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rsi), %xmm0, %xmm04048; AVX2-FAST-PERLANE-NEXT:    vpmovzxwq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero4049; AVX2-FAST-PERLANE-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8,9],zero,zero,zero,zero,zero,zero,xmm0[10,11],zero,zero,zero,zero,zero,zero4050; AVX2-FAST-PERLANE-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm04051; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rdx), %ymm1, %ymm14052; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm1, (%rcx)4053; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm0, 32(%rcx)4054; AVX2-FAST-PERLANE-NEXT:    vzeroupper4055; AVX2-FAST-PERLANE-NEXT:    retq4056;4057; AVX2-FAST-LABEL: vec384_v24i16_to_v6i64_factor4:4058; AVX2-FAST:       # %bb.0:4059; AVX2-FAST-NEXT:    vmovdqa (%rdi), %xmm04060; AVX2-FAST-NEXT:    vpaddb (%rsi), %xmm0, %xmm04061; AVX2-FAST-NEXT:    vpmovzxwq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero4062; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8,9],zero,zero,zero,zero,zero,zero,xmm0[10,11],zero,zero,zero,zero,zero,zero4063; AVX2-FAST-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm04064; AVX2-FAST-NEXT:    vpaddb (%rdx), %ymm1, %ymm14065; AVX2-FAST-NEXT:    vmovdqa %ymm1, (%rcx)4066; AVX2-FAST-NEXT:    vmovdqa %ymm0, 32(%rcx)4067; AVX2-FAST-NEXT:    vzeroupper4068; AVX2-FAST-NEXT:    retq4069;4070; AVX512F-LABEL: vec384_v24i16_to_v6i64_factor4:4071; AVX512F:       # %bb.0:4072; AVX512F-NEXT:    vmovdqa (%rdi), %xmm04073; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm04074; AVX512F-NEXT:    vpmovzxwq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero4075; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8,9],zero,zero,zero,zero,zero,zero,xmm0[10,11],zero,zero,zero,zero,zero,zero4076; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm04077; AVX512F-NEXT:    vpaddb (%rdx), %ymm1, %ymm14078; AVX512F-NEXT:    vmovdqa %ymm1, (%rcx)4079; AVX512F-NEXT:    vmovdqa %ymm0, 32(%rcx)4080; AVX512F-NEXT:    vzeroupper4081; AVX512F-NEXT:    retq4082;4083; AVX512BW-LABEL: vec384_v24i16_to_v6i64_factor4:4084; AVX512BW:       # %bb.0:4085; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm04086; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm04087; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero4088; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8,9],zero,zero,zero,zero,zero,zero,xmm0[10,11],zero,zero,zero,zero,zero,zero4089; AVX512BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm04090; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm04091; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)4092; AVX512BW-NEXT:    vzeroupper4093; AVX512BW-NEXT:    retq4094  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 644095  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 644096  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias4097  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>4098  %in.vec.cast = bitcast <48 x i8> %in.vec.trunc to <24 x i16>4099  %zextd.vec = shufflevector <24 x i16> %in.vec.cast, <24 x i16> zeroinitializer, <24 x i32> <i32 0, i32 25, i32 26, i32 27, i32 1, i32 29, i32 30, i32 31, i32 2, i32 33, i32 34, i32 35, i32 3, i32 37, i32 38, i32 39, i32 4, i32 41, i32 42, i32 43, i32 5, i32 45, i32 46, i32 47>4100  %out.bytevec = bitcast <24 x i16> %zextd.vec to <48 x i8>4101  %out.bytevec.padded = shufflevector <48 x i8> %out.bytevec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>4102  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 644103  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias4104  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 644105  ret void4106}4107 4108define void @vec384_v24i16_to_v4i96_factor6(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {4109; SSE2-LABEL: vec384_v24i16_to_v4i96_factor6:4110; SSE2:       # %bb.0:4111; SSE2-NEXT:    movdqa (%rdi), %xmm04112; SSE2-NEXT:    paddb (%rsi), %xmm04113; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,1,0,1]4114; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero4115; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,xmm1[0,1,2,3,4,5,6,7,8,9,10,11]4116; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm0[0,1,1,3,4,5,6,7]4117; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,1,2,1]4118; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm24119; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]4120; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero4121; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]4122; SSE2-NEXT:    paddb 16(%rdx), %xmm04123; SSE2-NEXT:    paddb (%rdx), %xmm24124; SSE2-NEXT:    paddb 32(%rdx), %xmm14125; SSE2-NEXT:    movdqa %xmm1, 32(%rcx)4126; SSE2-NEXT:    movdqa %xmm2, (%rcx)4127; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)4128; SSE2-NEXT:    retq4129;4130; SSE42-LABEL: vec384_v24i16_to_v4i96_factor6:4131; SSE42:       # %bb.0:4132; SSE42-NEXT:    movdqa (%rdi), %xmm04133; SSE42-NEXT:    paddb (%rsi), %xmm04134; SSE42-NEXT:    pxor %xmm1, %xmm14135; SSE42-NEXT:    movdqa %xmm0, %xmm24136; SSE42-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0,1],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[2,3],zero,zero4137; SSE42-NEXT:    pmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero4138; SSE42-NEXT:    psrld $16, %xmm04139; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4,5,6,7]4140; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm1[0,1,2,3],xmm3[4],xmm1[5,6,7]4141; SSE42-NEXT:    paddb 16(%rdx), %xmm34142; SSE42-NEXT:    paddb 32(%rdx), %xmm04143; SSE42-NEXT:    paddb (%rdx), %xmm24144; SSE42-NEXT:    movdqa %xmm2, (%rcx)4145; SSE42-NEXT:    movdqa %xmm0, 32(%rcx)4146; SSE42-NEXT:    movdqa %xmm3, 16(%rcx)4147; SSE42-NEXT:    retq4148;4149; AVX-LABEL: vec384_v24i16_to_v4i96_factor6:4150; AVX:       # %bb.0:4151; AVX-NEXT:    vmovdqa (%rdi), %xmm04152; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm04153; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[0,1],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[2,3],zero,zero4154; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero4155; AVX-NEXT:    vpxor %xmm3, %xmm3, %xmm34156; AVX-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4],xmm3[5,6,7]4157; AVX-NEXT:    vpsrld $16, %xmm0, %xmm04158; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0,1],xmm0[2],xmm3[3,4,5,6,7]4159; AVX-NEXT:    vpaddb 32(%rdx), %xmm0, %xmm04160; AVX-NEXT:    vpaddb 16(%rdx), %xmm2, %xmm24161; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm14162; AVX-NEXT:    vmovdqa %xmm1, (%rcx)4163; AVX-NEXT:    vmovdqa %xmm2, 16(%rcx)4164; AVX-NEXT:    vmovdqa %xmm0, 32(%rcx)4165; AVX-NEXT:    retq4166;4167; AVX2-SLOW-LABEL: vec384_v24i16_to_v4i96_factor6:4168; AVX2-SLOW:       # %bb.0:4169; AVX2-SLOW-NEXT:    vmovdqa (%rdi), %xmm04170; AVX2-SLOW-NEXT:    vpaddb (%rsi), %xmm0, %xmm04171; AVX2-SLOW-NEXT:    vpsrld $16, %xmm0, %xmm14172; AVX2-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm24173; AVX2-SLOW-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2],xmm2[3,4,5,6,7]4174; AVX2-SLOW-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero4175; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]4176; AVX2-SLOW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm04177; AVX2-SLOW-NEXT:    vpaddb (%rdx), %ymm0, %ymm04178; AVX2-SLOW-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm14179; AVX2-SLOW-NEXT:    vmovdqa %ymm1, 32(%rcx)4180; AVX2-SLOW-NEXT:    vmovdqa %ymm0, (%rcx)4181; AVX2-SLOW-NEXT:    vzeroupper4182; AVX2-SLOW-NEXT:    retq4183;4184; AVX2-FAST-PERLANE-LABEL: vec384_v24i16_to_v4i96_factor6:4185; AVX2-FAST-PERLANE:       # %bb.0:4186; AVX2-FAST-PERLANE-NEXT:    vmovdqa (%rdi), %xmm04187; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rsi), %xmm0, %xmm04188; AVX2-FAST-PERLANE-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,xmm0[6,7],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero4189; AVX2-FAST-PERLANE-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero4190; AVX2-FAST-PERLANE-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]4191; AVX2-FAST-PERLANE-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm04192; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rdx), %ymm0, %ymm04193; AVX2-FAST-PERLANE-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm14194; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm1, 32(%rcx)4195; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm0, (%rcx)4196; AVX2-FAST-PERLANE-NEXT:    vzeroupper4197; AVX2-FAST-PERLANE-NEXT:    retq4198;4199; AVX2-FAST-LABEL: vec384_v24i16_to_v4i96_factor6:4200; AVX2-FAST:       # %bb.0:4201; AVX2-FAST-NEXT:    vmovdqa (%rdi), %xmm04202; AVX2-FAST-NEXT:    vpaddb (%rsi), %xmm0, %xmm04203; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,xmm0[6,7],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero4204; AVX2-FAST-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero4205; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]4206; AVX2-FAST-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm04207; AVX2-FAST-NEXT:    vpaddb (%rdx), %ymm0, %ymm04208; AVX2-FAST-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm14209; AVX2-FAST-NEXT:    vmovdqa %ymm1, 32(%rcx)4210; AVX2-FAST-NEXT:    vmovdqa %ymm0, (%rcx)4211; AVX2-FAST-NEXT:    vzeroupper4212; AVX2-FAST-NEXT:    retq4213;4214; AVX512F-SLOW-LABEL: vec384_v24i16_to_v4i96_factor6:4215; AVX512F-SLOW:       # %bb.0:4216; AVX512F-SLOW-NEXT:    vmovdqa (%rdi), %xmm04217; AVX512F-SLOW-NEXT:    vpaddb (%rsi), %xmm0, %xmm04218; AVX512F-SLOW-NEXT:    vpsrld $16, %xmm0, %xmm14219; AVX512F-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm24220; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2],xmm2[3,4,5,6,7]4221; AVX512F-SLOW-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero4222; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]4223; AVX512F-SLOW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm04224; AVX512F-SLOW-NEXT:    vpaddb (%rdx), %ymm0, %ymm04225; AVX512F-SLOW-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm14226; AVX512F-SLOW-NEXT:    vmovdqa %ymm1, 32(%rcx)4227; AVX512F-SLOW-NEXT:    vmovdqa %ymm0, (%rcx)4228; AVX512F-SLOW-NEXT:    vzeroupper4229; AVX512F-SLOW-NEXT:    retq4230;4231; AVX512F-FAST-LABEL: vec384_v24i16_to_v4i96_factor6:4232; AVX512F-FAST:       # %bb.0:4233; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %xmm04234; AVX512F-FAST-NEXT:    vpaddb (%rsi), %xmm0, %xmm04235; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,xmm0[6,7],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero4236; AVX512F-FAST-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero4237; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]4238; AVX512F-FAST-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm04239; AVX512F-FAST-NEXT:    vpaddb (%rdx), %ymm0, %ymm04240; AVX512F-FAST-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm14241; AVX512F-FAST-NEXT:    vmovdqa %ymm1, 32(%rcx)4242; AVX512F-FAST-NEXT:    vmovdqa %ymm0, (%rcx)4243; AVX512F-FAST-NEXT:    vzeroupper4244; AVX512F-FAST-NEXT:    retq4245;4246; AVX512BW-SLOW-LABEL: vec384_v24i16_to_v4i96_factor6:4247; AVX512BW-SLOW:       # %bb.0:4248; AVX512BW-SLOW-NEXT:    vmovdqa (%rdi), %ymm04249; AVX512BW-SLOW-NEXT:    vpaddb (%rsi), %ymm0, %ymm04250; AVX512BW-SLOW-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [16,1,2,3,4,5,17,7,8,9,10,11,18,13,14,15]4251; AVX512BW-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm24252; AVX512BW-SLOW-NEXT:    vpermt2w %ymm0, %ymm1, %ymm24253; AVX512BW-SLOW-NEXT:    vpsrld $16, %xmm0, %xmm04254; AVX512BW-SLOW-NEXT:    vpxor %xmm1, %xmm1, %xmm14255; AVX512BW-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4,5,6,7]4256; AVX512BW-SLOW-NEXT:    vinserti32x4 $2, %xmm0, %zmm2, %zmm04257; AVX512BW-SLOW-NEXT:    vpaddb (%rdx), %zmm0, %zmm04258; AVX512BW-SLOW-NEXT:    vmovdqa64 %zmm0, (%rcx)4259; AVX512BW-SLOW-NEXT:    vzeroupper4260; AVX512BW-SLOW-NEXT:    retq4261;4262; AVX512BW-FAST-LABEL: vec384_v24i16_to_v4i96_factor6:4263; AVX512BW-FAST:       # %bb.0:4264; AVX512BW-FAST-NEXT:    vmovdqa (%rdi), %ymm04265; AVX512BW-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm04266; AVX512BW-FAST-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [16,1,2,3,4,5,17,7,8,9,10,11,18,13,14,15]4267; AVX512BW-FAST-NEXT:    vpxor %xmm2, %xmm2, %xmm24268; AVX512BW-FAST-NEXT:    vpermt2w %ymm0, %ymm1, %ymm24269; AVX512BW-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[6,7],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero4270; AVX512BW-FAST-NEXT:    vinserti32x4 $2, %xmm0, %zmm2, %zmm04271; AVX512BW-FAST-NEXT:    vpaddb (%rdx), %zmm0, %zmm04272; AVX512BW-FAST-NEXT:    vmovdqa64 %zmm0, (%rcx)4273; AVX512BW-FAST-NEXT:    vzeroupper4274; AVX512BW-FAST-NEXT:    retq4275  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 644276  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 644277  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias4278  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>4279  %in.vec.cast = bitcast <48 x i8> %in.vec.trunc to <24 x i16>4280  %zextd.vec = shufflevector <24 x i16> %in.vec.cast, <24 x i16> zeroinitializer, <24 x i32> <i32 0, i32 25, i32 26, i32 27, i32 28, i32 29, i32 1, i32 31, i32 32, i32 33, i32 34, i32 35, i32 2, i32 37, i32 38, i32 39, i32 40, i32 41, i32 3, i32 43, i32 44, i32 45, i32 46, i32 47>4281  %out.bytevec = bitcast <24 x i16> %zextd.vec to <48 x i8>4282  %out.bytevec.padded = shufflevector <48 x i8> %out.bytevec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>4283  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 644284  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias4285  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 644286  ret void4287}4288 4289define void @vec384_v24i16_to_v3i128_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {4290; SSE2-LABEL: vec384_v24i16_to_v3i128_factor8:4291; SSE2:       # %bb.0:4292; SSE2-NEXT:    movdqa (%rdi), %xmm04293; SSE2-NEXT:    paddb (%rsi), %xmm04294; SSE2-NEXT:    movd {{.*#+}} xmm1 = [65535,0,0,0]4295; SSE2-NEXT:    pand %xmm0, %xmm14296; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,0,0]4297; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]4298; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero4299; SSE2-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero4300; SSE2-NEXT:    paddb 16(%rdx), %xmm24301; SSE2-NEXT:    paddb 32(%rdx), %xmm04302; SSE2-NEXT:    paddb (%rdx), %xmm14303; SSE2-NEXT:    movdqa %xmm1, (%rcx)4304; SSE2-NEXT:    movdqa %xmm0, 32(%rcx)4305; SSE2-NEXT:    movdqa %xmm2, 16(%rcx)4306; SSE2-NEXT:    retq4307;4308; SSE42-LABEL: vec384_v24i16_to_v3i128_factor8:4309; SSE42:       # %bb.0:4310; SSE42-NEXT:    movdqa (%rdi), %xmm04311; SSE42-NEXT:    paddb (%rsi), %xmm04312; SSE42-NEXT:    pxor %xmm1, %xmm14313; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]4314; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,0,0]4315; SSE42-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]4316; SSE42-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero4317; SSE42-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero4318; SSE42-NEXT:    paddb 16(%rdx), %xmm24319; SSE42-NEXT:    paddb 32(%rdx), %xmm04320; SSE42-NEXT:    paddb (%rdx), %xmm14321; SSE42-NEXT:    movdqa %xmm1, (%rcx)4322; SSE42-NEXT:    movdqa %xmm0, 32(%rcx)4323; SSE42-NEXT:    movdqa %xmm2, 16(%rcx)4324; SSE42-NEXT:    retq4325;4326; AVX-LABEL: vec384_v24i16_to_v3i128_factor8:4327; AVX:       # %bb.0:4328; AVX-NEXT:    vmovdqa (%rdi), %xmm04329; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm04330; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm14331; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]4332; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[0,0,0,0]4333; AVX-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero4334; AVX-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]4335; AVX-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero4336; AVX-NEXT:    vpaddb 32(%rdx), %xmm0, %xmm04337; AVX-NEXT:    vpaddb 16(%rdx), %xmm2, %xmm24338; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm14339; AVX-NEXT:    vmovdqa %xmm1, (%rcx)4340; AVX-NEXT:    vmovdqa %xmm2, 16(%rcx)4341; AVX-NEXT:    vmovdqa %xmm0, 32(%rcx)4342; AVX-NEXT:    retq4343;4344; AVX2-SLOW-LABEL: vec384_v24i16_to_v3i128_factor8:4345; AVX2-SLOW:       # %bb.0:4346; AVX2-SLOW-NEXT:    vpxor %xmm0, %xmm0, %xmm04347; AVX2-SLOW-NEXT:    vmovdqa (%rdi), %xmm14348; AVX2-SLOW-NEXT:    vpaddb (%rsi), %xmm1, %xmm14349; AVX2-SLOW-NEXT:    vpslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4,5]4350; AVX2-SLOW-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero4351; AVX2-SLOW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero4352; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]4353; AVX2-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15]4354; AVX2-SLOW-NEXT:    vpaddb (%rdx), %ymm0, %ymm04355; AVX2-SLOW-NEXT:    vpaddb 32(%rdx), %ymm2, %ymm14356; AVX2-SLOW-NEXT:    vmovdqa %ymm1, 32(%rcx)4357; AVX2-SLOW-NEXT:    vmovdqa %ymm0, (%rcx)4358; AVX2-SLOW-NEXT:    vzeroupper4359; AVX2-SLOW-NEXT:    retq4360;4361; AVX2-FAST-PERLANE-LABEL: vec384_v24i16_to_v3i128_factor8:4362; AVX2-FAST-PERLANE:       # %bb.0:4363; AVX2-FAST-PERLANE-NEXT:    vpxor %xmm0, %xmm0, %xmm04364; AVX2-FAST-PERLANE-NEXT:    vmovdqa (%rdi), %xmm14365; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rsi), %xmm1, %xmm14366; AVX2-FAST-PERLANE-NEXT:    vpshufb {{.*#+}} xmm2 = xmm1[4,5],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero4367; AVX2-FAST-PERLANE-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero4368; AVX2-FAST-PERLANE-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]4369; AVX2-FAST-PERLANE-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15]4370; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rdx), %ymm0, %ymm04371; AVX2-FAST-PERLANE-NEXT:    vpaddb 32(%rdx), %ymm2, %ymm14372; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm1, 32(%rcx)4373; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm0, (%rcx)4374; AVX2-FAST-PERLANE-NEXT:    vzeroupper4375; AVX2-FAST-PERLANE-NEXT:    retq4376;4377; AVX2-FAST-LABEL: vec384_v24i16_to_v3i128_factor8:4378; AVX2-FAST:       # %bb.0:4379; AVX2-FAST-NEXT:    vpxor %xmm0, %xmm0, %xmm04380; AVX2-FAST-NEXT:    vmovdqa (%rdi), %xmm14381; AVX2-FAST-NEXT:    vpaddb (%rsi), %xmm1, %xmm14382; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm1[4,5],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero4383; AVX2-FAST-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero4384; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]4385; AVX2-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15]4386; AVX2-FAST-NEXT:    vpaddb (%rdx), %ymm0, %ymm04387; AVX2-FAST-NEXT:    vpaddb 32(%rdx), %ymm2, %ymm14388; AVX2-FAST-NEXT:    vmovdqa %ymm1, 32(%rcx)4389; AVX2-FAST-NEXT:    vmovdqa %ymm0, (%rcx)4390; AVX2-FAST-NEXT:    vzeroupper4391; AVX2-FAST-NEXT:    retq4392;4393; AVX512F-SLOW-LABEL: vec384_v24i16_to_v3i128_factor8:4394; AVX512F-SLOW:       # %bb.0:4395; AVX512F-SLOW-NEXT:    vmovdqa (%rdi), %xmm04396; AVX512F-SLOW-NEXT:    vpaddb (%rsi), %xmm0, %xmm04397; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]4398; AVX512F-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm24399; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]4400; AVX512F-SLOW-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero4401; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]4402; AVX512F-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm24403; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3,4,5,6,7],ymm0[8],ymm2[9,10,11,12,13,14,15]4404; AVX512F-SLOW-NEXT:    vpaddb (%rdx), %ymm0, %ymm04405; AVX512F-SLOW-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm14406; AVX512F-SLOW-NEXT:    vmovdqa %ymm1, 32(%rcx)4407; AVX512F-SLOW-NEXT:    vmovdqa %ymm0, (%rcx)4408; AVX512F-SLOW-NEXT:    vzeroupper4409; AVX512F-SLOW-NEXT:    retq4410;4411; AVX512F-FAST-LABEL: vec384_v24i16_to_v3i128_factor8:4412; AVX512F-FAST:       # %bb.0:4413; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %xmm04414; AVX512F-FAST-NEXT:    vpaddb (%rsi), %xmm0, %xmm04415; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[4,5],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero4416; AVX512F-FAST-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero4417; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]4418; AVX512F-FAST-NEXT:    vpxor %xmm2, %xmm2, %xmm24419; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3,4,5,6,7],ymm0[8],ymm2[9,10,11,12,13,14,15]4420; AVX512F-FAST-NEXT:    vpaddb (%rdx), %ymm0, %ymm04421; AVX512F-FAST-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm14422; AVX512F-FAST-NEXT:    vmovdqa %ymm1, 32(%rcx)4423; AVX512F-FAST-NEXT:    vmovdqa %ymm0, (%rcx)4424; AVX512F-FAST-NEXT:    vzeroupper4425; AVX512F-FAST-NEXT:    retq4426;4427; AVX512BW-SLOW-LABEL: vec384_v24i16_to_v3i128_factor8:4428; AVX512BW-SLOW:       # %bb.0:4429; AVX512BW-SLOW-NEXT:    vmovdqa (%rdi), %ymm04430; AVX512BW-SLOW-NEXT:    vpaddb (%rsi), %ymm0, %ymm04431; AVX512BW-SLOW-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [16,1,2,3,4,5,6,7,17,9,10,11,12,13,14,15]4432; AVX512BW-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm24433; AVX512BW-SLOW-NEXT:    vpermt2w %ymm0, %ymm1, %ymm24434; AVX512BW-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]4435; AVX512BW-SLOW-NEXT:    vpxor %xmm1, %xmm1, %xmm14436; AVX512BW-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]4437; AVX512BW-SLOW-NEXT:    vinserti32x4 $2, %xmm0, %zmm2, %zmm04438; AVX512BW-SLOW-NEXT:    vpaddb (%rdx), %zmm0, %zmm04439; AVX512BW-SLOW-NEXT:    vmovdqa64 %zmm0, (%rcx)4440; AVX512BW-SLOW-NEXT:    vzeroupper4441; AVX512BW-SLOW-NEXT:    retq4442;4443; AVX512BW-FAST-LABEL: vec384_v24i16_to_v3i128_factor8:4444; AVX512BW-FAST:       # %bb.0:4445; AVX512BW-FAST-NEXT:    vmovdqa (%rdi), %ymm04446; AVX512BW-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm04447; AVX512BW-FAST-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [16,1,2,3,4,5,6,7,17,9,10,11,12,13,14,15]4448; AVX512BW-FAST-NEXT:    vpxor %xmm2, %xmm2, %xmm24449; AVX512BW-FAST-NEXT:    vpermt2w %ymm0, %ymm1, %ymm24450; AVX512BW-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero4451; AVX512BW-FAST-NEXT:    vinserti32x4 $2, %xmm0, %zmm2, %zmm04452; AVX512BW-FAST-NEXT:    vpaddb (%rdx), %zmm0, %zmm04453; AVX512BW-FAST-NEXT:    vmovdqa64 %zmm0, (%rcx)4454; AVX512BW-FAST-NEXT:    vzeroupper4455; AVX512BW-FAST-NEXT:    retq4456  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 644457  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 644458  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias4459  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>4460  %in.vec.cast = bitcast <48 x i8> %in.vec.trunc to <24 x i16>4461  %zextd.vec = shufflevector <24 x i16> %in.vec.cast, <24 x i16> zeroinitializer, <24 x i32> <i32 0, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 1, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 2, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>4462  %out.bytevec = bitcast <24 x i16> %zextd.vec to <48 x i8>4463  %out.bytevec.padded = shufflevector <48 x i8> %out.bytevec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>4464  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 644465  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias4466  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 644467  ret void4468}4469 4470define void @vec384_v24i16_to_v2i192_factor12(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {4471; SSE2-LABEL: vec384_v24i16_to_v2i192_factor12:4472; SSE2:       # %bb.0:4473; SSE2-NEXT:    movdqa (%rdi), %xmm04474; SSE2-NEXT:    paddb (%rsi), %xmm04475; SSE2-NEXT:    movd {{.*#+}} xmm1 = [65535,0,0,0]4476; SSE2-NEXT:    pand %xmm0, %xmm14477; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]4478; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero4479; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]4480; SSE2-NEXT:    movaps 32(%rdx), %xmm24481; SSE2-NEXT:    paddb 16(%rdx), %xmm04482; SSE2-NEXT:    paddb (%rdx), %xmm14483; SSE2-NEXT:    movaps %xmm2, 32(%rcx)4484; SSE2-NEXT:    movdqa %xmm1, (%rcx)4485; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)4486; SSE2-NEXT:    retq4487;4488; SSE42-LABEL: vec384_v24i16_to_v2i192_factor12:4489; SSE42:       # %bb.0:4490; SSE42-NEXT:    movdqa (%rdi), %xmm04491; SSE42-NEXT:    paddb (%rsi), %xmm04492; SSE42-NEXT:    pxor %xmm1, %xmm14493; SSE42-NEXT:    pxor %xmm2, %xmm24494; SSE42-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3,4,5,6,7]4495; SSE42-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero4496; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4],xmm1[5,6,7]4497; SSE42-NEXT:    movaps 32(%rdx), %xmm14498; SSE42-NEXT:    paddb 16(%rdx), %xmm04499; SSE42-NEXT:    paddb (%rdx), %xmm24500; SSE42-NEXT:    movaps %xmm1, 32(%rcx)4501; SSE42-NEXT:    movdqa %xmm2, (%rcx)4502; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)4503; SSE42-NEXT:    retq4504;4505; AVX-LABEL: vec384_v24i16_to_v2i192_factor12:4506; AVX:       # %bb.0:4507; AVX-NEXT:    vmovdqa (%rdi), %xmm04508; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm04509; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm14510; AVX-NEXT:    vpblendw {{.*#+}} xmm2 = xmm0[0],xmm1[1,2,3,4,5,6,7]4511; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero4512; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4],xmm1[5,6,7]4513; AVX-NEXT:    vmovaps 32(%rdx), %ymm14514; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm04515; AVX-NEXT:    vpaddb (%rdx), %xmm2, %xmm24516; AVX-NEXT:    vmovaps %ymm1, 32(%rcx)4517; AVX-NEXT:    vmovdqa %xmm2, (%rcx)4518; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)4519; AVX-NEXT:    vzeroupper4520; AVX-NEXT:    retq4521;4522; AVX2-LABEL: vec384_v24i16_to_v2i192_factor12:4523; AVX2:       # %bb.0:4524; AVX2-NEXT:    vmovdqa (%rdi), %xmm04525; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm04526; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero4527; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,2,1]4528; AVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm04529; AVX2-NEXT:    vmovaps 32(%rdx), %ymm14530; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm04531; AVX2-NEXT:    vmovaps %ymm1, 32(%rcx)4532; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)4533; AVX2-NEXT:    vzeroupper4534; AVX2-NEXT:    retq4535;4536; AVX512F-LABEL: vec384_v24i16_to_v2i192_factor12:4537; AVX512F:       # %bb.0:4538; AVX512F-NEXT:    vmovdqa (%rdi), %xmm04539; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm04540; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero4541; AVX512F-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,2,1]4542; AVX512F-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm04543; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm04544; AVX512F-NEXT:    vmovaps 32(%rdx), %ymm14545; AVX512F-NEXT:    vmovaps %ymm1, 32(%rcx)4546; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)4547; AVX512F-NEXT:    vzeroupper4548; AVX512F-NEXT:    retq4549;4550; AVX512BW-LABEL: vec384_v24i16_to_v2i192_factor12:4551; AVX512BW:       # %bb.0:4552; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm04553; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm04554; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [16,1,2,3,4,5,6,7,8,9,10,11,17,13,14,15]4555; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm24556; AVX512BW-NEXT:    vpermt2w %ymm0, %ymm1, %ymm24557; AVX512BW-NEXT:    vpaddb (%rdx), %zmm2, %zmm04558; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)4559; AVX512BW-NEXT:    vzeroupper4560; AVX512BW-NEXT:    retq4561  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 644562  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 644563  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias4564  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>4565  %in.vec.cast = bitcast <48 x i8> %in.vec.trunc to <24 x i16>4566  %zextd.vec = shufflevector <24 x i16> %in.vec.cast, <24 x i16> zeroinitializer, <24 x i32> <i32 0, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 1, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>4567  %out.bytevec = bitcast <24 x i16> %zextd.vec to <48 x i8>4568  %out.bytevec.padded = shufflevector <48 x i8> %out.bytevec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>4569  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 644570  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias4571  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 644572  ret void4573}4574 4575define void @vec384_v24i16_to_v1i384_factor24(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {4576; SSE2-LABEL: vec384_v24i16_to_v1i384_factor24:4577; SSE2:       # %bb.0:4578; SSE2-NEXT:    movdqa (%rdi), %xmm04579; SSE2-NEXT:    paddb (%rsi), %xmm04580; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm04581; SSE2-NEXT:    movaps 16(%rdx), %xmm14582; SSE2-NEXT:    movaps 32(%rdx), %xmm24583; SSE2-NEXT:    paddb (%rdx), %xmm04584; SSE2-NEXT:    movaps %xmm1, 16(%rcx)4585; SSE2-NEXT:    movaps %xmm2, 32(%rcx)4586; SSE2-NEXT:    movdqa %xmm0, (%rcx)4587; SSE2-NEXT:    retq4588;4589; SSE42-LABEL: vec384_v24i16_to_v1i384_factor24:4590; SSE42:       # %bb.0:4591; SSE42-NEXT:    movdqa (%rdi), %xmm04592; SSE42-NEXT:    paddb (%rsi), %xmm04593; SSE42-NEXT:    pxor %xmm1, %xmm14594; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]4595; SSE42-NEXT:    movaps 16(%rdx), %xmm04596; SSE42-NEXT:    movaps 32(%rdx), %xmm24597; SSE42-NEXT:    paddb (%rdx), %xmm14598; SSE42-NEXT:    movaps %xmm0, 16(%rcx)4599; SSE42-NEXT:    movaps %xmm2, 32(%rcx)4600; SSE42-NEXT:    movdqa %xmm1, (%rcx)4601; SSE42-NEXT:    retq4602;4603; AVX-LABEL: vec384_v24i16_to_v1i384_factor24:4604; AVX:       # %bb.0:4605; AVX-NEXT:    vmovdqa (%rdi), %xmm04606; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm04607; AVX-NEXT:    vmovaps 32(%rdx), %ymm14608; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm24609; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7]4610; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm04611; AVX-NEXT:    vmovaps 16(%rdx), %xmm24612; AVX-NEXT:    vmovaps %xmm2, 16(%rcx)4613; AVX-NEXT:    vmovaps %ymm1, 32(%rcx)4614; AVX-NEXT:    vmovdqa %xmm0, (%rcx)4615; AVX-NEXT:    vzeroupper4616; AVX-NEXT:    retq4617;4618; AVX2-LABEL: vec384_v24i16_to_v1i384_factor24:4619; AVX2:       # %bb.0:4620; AVX2-NEXT:    vmovdqa (%rdi), %ymm04621; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm04622; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = [65535,0,0,0]4623; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm04624; AVX2-NEXT:    vmovaps 32(%rdx), %ymm14625; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm04626; AVX2-NEXT:    vmovaps %ymm1, 32(%rcx)4627; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)4628; AVX2-NEXT:    vzeroupper4629; AVX2-NEXT:    retq4630;4631; AVX512F-LABEL: vec384_v24i16_to_v1i384_factor24:4632; AVX512F:       # %bb.0:4633; AVX512F-NEXT:    vmovdqa (%rdi), %ymm04634; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm04635; AVX512F-NEXT:    vmovd {{.*#+}} xmm1 = [65535,0,0,0]4636; AVX512F-NEXT:    vpand %ymm1, %ymm0, %ymm04637; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm04638; AVX512F-NEXT:    vmovaps 32(%rdx), %ymm14639; AVX512F-NEXT:    vmovaps %ymm1, 32(%rcx)4640; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)4641; AVX512F-NEXT:    vzeroupper4642; AVX512F-NEXT:    retq4643;4644; AVX512BW-LABEL: vec384_v24i16_to_v1i384_factor24:4645; AVX512BW:       # %bb.0:4646; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm04647; AVX512BW-NEXT:    vpaddb (%rsi), %zmm0, %zmm04648; AVX512BW-NEXT:    vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm04649; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm04650; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)4651; AVX512BW-NEXT:    vzeroupper4652; AVX512BW-NEXT:    retq4653  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 644654  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 644655  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias4656  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>4657  %in.vec.cast = bitcast <48 x i8> %in.vec.trunc to <24 x i16>4658  %zextd.vec = shufflevector <24 x i16> %in.vec.cast, <24 x i16> zeroinitializer, <24 x i32> <i32 0, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>4659  %out.bytevec = bitcast <24 x i16> %zextd.vec to <48 x i8>4660  %out.bytevec.padded = shufflevector <48 x i8> %out.bytevec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>4661  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 644662  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias4663  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 644664  ret void4665}4666 4667define void @vec384_v12i32_to_v6i64_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {4668; SSE2-LABEL: vec384_v12i32_to_v6i64_factor2:4669; SSE2:       # %bb.0:4670; SSE2-NEXT:    movdqa (%rdi), %xmm04671; SSE2-NEXT:    movdqa 16(%rdi), %xmm14672; SSE2-NEXT:    paddb (%rsi), %xmm04673; SSE2-NEXT:    paddb 16(%rsi), %xmm14674; SSE2-NEXT:    pxor %xmm2, %xmm24675; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]4676; SSE2-NEXT:    movdqa %xmm0, %xmm34677; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]4678; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]4679; SSE2-NEXT:    paddb 16(%rdx), %xmm04680; SSE2-NEXT:    paddb (%rdx), %xmm34681; SSE2-NEXT:    paddb 32(%rdx), %xmm14682; SSE2-NEXT:    movdqa %xmm1, 32(%rcx)4683; SSE2-NEXT:    movdqa %xmm3, (%rcx)4684; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)4685; SSE2-NEXT:    retq4686;4687; SSE42-LABEL: vec384_v12i32_to_v6i64_factor2:4688; SSE42:       # %bb.0:4689; SSE42-NEXT:    movdqa (%rdi), %xmm04690; SSE42-NEXT:    movdqa 16(%rdi), %xmm14691; SSE42-NEXT:    paddb (%rsi), %xmm04692; SSE42-NEXT:    paddb 16(%rsi), %xmm14693; SSE42-NEXT:    pxor %xmm2, %xmm24694; SSE42-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero4695; SSE42-NEXT:    pmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero4696; SSE42-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]4697; SSE42-NEXT:    paddb 16(%rdx), %xmm04698; SSE42-NEXT:    paddb (%rdx), %xmm34699; SSE42-NEXT:    paddb 32(%rdx), %xmm14700; SSE42-NEXT:    movdqa %xmm1, 32(%rcx)4701; SSE42-NEXT:    movdqa %xmm3, (%rcx)4702; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)4703; SSE42-NEXT:    retq4704;4705; AVX-LABEL: vec384_v12i32_to_v6i64_factor2:4706; AVX:       # %bb.0:4707; AVX-NEXT:    vmovdqa (%rdi), %xmm04708; AVX-NEXT:    vmovdqa 16(%rdi), %xmm14709; AVX-NEXT:    vpaddb 16(%rsi), %xmm1, %xmm14710; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm04711; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero4712; AVX-NEXT:    vpxor %xmm3, %xmm3, %xmm34713; AVX-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3]4714; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero4715; AVX-NEXT:    vpaddb 32(%rdx), %xmm1, %xmm14716; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm04717; AVX-NEXT:    vpaddb (%rdx), %xmm2, %xmm24718; AVX-NEXT:    vmovdqa %xmm2, (%rcx)4719; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)4720; AVX-NEXT:    vmovdqa %xmm1, 32(%rcx)4721; AVX-NEXT:    retq4722;4723; AVX2-LABEL: vec384_v12i32_to_v6i64_factor2:4724; AVX2:       # %bb.0:4725; AVX2-NEXT:    vmovdqa (%rdi), %ymm04726; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm04727; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero4728; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm04729; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero4730; AVX2-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm04731; AVX2-NEXT:    vpaddb (%rdx), %ymm1, %ymm14732; AVX2-NEXT:    vmovdqa %ymm1, (%rcx)4733; AVX2-NEXT:    vmovdqa %ymm0, 32(%rcx)4734; AVX2-NEXT:    vzeroupper4735; AVX2-NEXT:    retq4736;4737; AVX512F-LABEL: vec384_v12i32_to_v6i64_factor2:4738; AVX512F:       # %bb.0:4739; AVX512F-NEXT:    vmovdqa (%rdi), %ymm04740; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm04741; AVX512F-NEXT:    vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero4742; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm14743; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm14744; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm04745; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)4746; AVX512F-NEXT:    vmovdqa %ymm1, 32(%rcx)4747; AVX512F-NEXT:    vzeroupper4748; AVX512F-NEXT:    retq4749;4750; AVX512BW-LABEL: vec384_v12i32_to_v6i64_factor2:4751; AVX512BW:       # %bb.0:4752; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm04753; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm04754; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero4755; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm04756; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)4757; AVX512BW-NEXT:    vzeroupper4758; AVX512BW-NEXT:    retq4759  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 644760  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 644761  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias4762  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>4763  %in.vec.cast = bitcast <48 x i8> %in.vec.trunc to <12 x i32>4764  %zextd.vec = shufflevector <12 x i32> %in.vec.cast, <12 x i32> zeroinitializer, <12 x i32> <i32 0, i32 13, i32 1, i32 15, i32 2, i32 17, i32 3, i32 19, i32 4, i32 21, i32 5, i32 23>4765  %out.bytevec = bitcast <12 x i32> %zextd.vec to <48 x i8>4766  %out.bytevec.padded = shufflevector <48 x i8> %out.bytevec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>4767  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 644768  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias4769  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 644770  ret void4771}4772 4773define void @vec384_v12i32_to_v4i96_factor3(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {4774; SSE2-LABEL: vec384_v12i32_to_v4i96_factor3:4775; SSE2:       # %bb.0:4776; SSE2-NEXT:    movdqa (%rdi), %xmm04777; SSE2-NEXT:    paddb (%rsi), %xmm04778; SSE2-NEXT:    xorps %xmm1, %xmm14779; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [0,0,4294967295,0]4780; SSE2-NEXT:    pand %xmm0, %xmm24781; SSE2-NEXT:    movdqa %xmm0, %xmm34782; SSE2-NEXT:    psrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero4783; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,0],xmm1[2,3]4784; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]4785; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,0,1,3]4786; SSE2-NEXT:    paddb (%rdx), %xmm04787; SSE2-NEXT:    paddb 32(%rdx), %xmm34788; SSE2-NEXT:    paddb 16(%rdx), %xmm24789; SSE2-NEXT:    movdqa %xmm2, 16(%rcx)4790; SSE2-NEXT:    movdqa %xmm3, 32(%rcx)4791; SSE2-NEXT:    movdqa %xmm0, (%rcx)4792; SSE2-NEXT:    retq4793;4794; SSE42-LABEL: vec384_v12i32_to_v4i96_factor3:4795; SSE42:       # %bb.0:4796; SSE42-NEXT:    movdqa (%rdi), %xmm04797; SSE42-NEXT:    paddb (%rsi), %xmm04798; SSE42-NEXT:    pxor %xmm1, %xmm14799; SSE42-NEXT:    pxor %xmm2, %xmm24800; SSE42-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm0[4,5],xmm2[6,7]4801; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]4802; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm1[0,1],xmm3[2,3],xmm1[4,5,6,7]4803; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]4804; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5],xmm0[6,7]4805; SSE42-NEXT:    paddb (%rdx), %xmm04806; SSE42-NEXT:    paddb 32(%rdx), %xmm34807; SSE42-NEXT:    paddb 16(%rdx), %xmm24808; SSE42-NEXT:    movdqa %xmm2, 16(%rcx)4809; SSE42-NEXT:    movdqa %xmm3, 32(%rcx)4810; SSE42-NEXT:    movdqa %xmm0, (%rcx)4811; SSE42-NEXT:    retq4812;4813; AVX-LABEL: vec384_v12i32_to_v4i96_factor3:4814; AVX:       # %bb.0:4815; AVX-NEXT:    vmovdqa (%rdi), %xmm04816; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm04817; AVX-NEXT:    vinsertps {{.*#+}} xmm1 = xmm0[0],zero,zero,xmm0[1]4818; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm24819; AVX-NEXT:    vpblendw {{.*#+}} xmm3 = xmm2[0,1,2,3],xmm0[4,5],xmm2[6,7]4820; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]4821; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3],xmm2[4,5,6,7]4822; AVX-NEXT:    vpaddb 32(%rdx), %xmm0, %xmm04823; AVX-NEXT:    vpaddb 16(%rdx), %xmm3, %xmm24824; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm14825; AVX-NEXT:    vmovdqa %xmm1, (%rcx)4826; AVX-NEXT:    vmovdqa %xmm2, 16(%rcx)4827; AVX-NEXT:    vmovdqa %xmm0, 32(%rcx)4828; AVX-NEXT:    retq4829;4830; AVX2-SLOW-LABEL: vec384_v12i32_to_v4i96_factor3:4831; AVX2-SLOW:       # %bb.0:4832; AVX2-SLOW-NEXT:    vmovdqa (%rdi), %ymm04833; AVX2-SLOW-NEXT:    vpaddb (%rsi), %ymm0, %ymm04834; AVX2-SLOW-NEXT:    vpxor %xmm1, %xmm1, %xmm14835; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[0,0,2,1]4836; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]4837; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]4838; AVX2-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm24839; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2,3]4840; AVX2-SLOW-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm04841; AVX2-SLOW-NEXT:    vpaddb (%rdx), %ymm1, %ymm14842; AVX2-SLOW-NEXT:    vmovdqa %ymm1, (%rcx)4843; AVX2-SLOW-NEXT:    vmovdqa %ymm0, 32(%rcx)4844; AVX2-SLOW-NEXT:    vzeroupper4845; AVX2-SLOW-NEXT:    retq4846;4847; AVX2-FAST-PERLANE-LABEL: vec384_v12i32_to_v4i96_factor3:4848; AVX2-FAST-PERLANE:       # %bb.0:4849; AVX2-FAST-PERLANE-NEXT:    vmovdqa (%rdi), %ymm04850; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rsi), %ymm0, %ymm04851; AVX2-FAST-PERLANE-NEXT:    vpxor %xmm1, %xmm1, %xmm14852; AVX2-FAST-PERLANE-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[0,0,2,1]4853; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]4854; AVX2-FAST-PERLANE-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero4855; AVX2-FAST-PERLANE-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm04856; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rdx), %ymm1, %ymm14857; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm1, (%rcx)4858; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm0, 32(%rcx)4859; AVX2-FAST-PERLANE-NEXT:    vzeroupper4860; AVX2-FAST-PERLANE-NEXT:    retq4861;4862; AVX2-FAST-LABEL: vec384_v12i32_to_v4i96_factor3:4863; AVX2-FAST:       # %bb.0:4864; AVX2-FAST-NEXT:    vmovdqa (%rdi), %ymm04865; AVX2-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm04866; AVX2-FAST-NEXT:    vpxor %xmm1, %xmm1, %xmm14867; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[0,0,2,1]4868; AVX2-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]4869; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero4870; AVX2-FAST-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm04871; AVX2-FAST-NEXT:    vpaddb (%rdx), %ymm1, %ymm14872; AVX2-FAST-NEXT:    vmovdqa %ymm1, (%rcx)4873; AVX2-FAST-NEXT:    vmovdqa %ymm0, 32(%rcx)4874; AVX2-FAST-NEXT:    vzeroupper4875; AVX2-FAST-NEXT:    retq4876;4877; AVX512F-LABEL: vec384_v12i32_to_v4i96_factor3:4878; AVX512F:       # %bb.0:4879; AVX512F-NEXT:    vmovdqa (%rdi), %ymm04880; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm04881; AVX512F-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [16,1,2,17,4,5,18,7,8,19,10,11,0,0,0,0]4882; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm24883; AVX512F-NEXT:    vpermt2d %zmm0, %zmm1, %zmm24884; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm04885; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm04886; AVX512F-NEXT:    vpaddb (%rdx), %ymm2, %ymm14887; AVX512F-NEXT:    vmovdqa %ymm1, (%rcx)4888; AVX512F-NEXT:    vmovdqa %ymm0, 32(%rcx)4889; AVX512F-NEXT:    vzeroupper4890; AVX512F-NEXT:    retq4891;4892; AVX512BW-SLOW-LABEL: vec384_v12i32_to_v4i96_factor3:4893; AVX512BW-SLOW:       # %bb.0:4894; AVX512BW-SLOW-NEXT:    vmovdqa (%rdi), %ymm04895; AVX512BW-SLOW-NEXT:    vpaddb (%rsi), %ymm0, %ymm04896; AVX512BW-SLOW-NEXT:    movb $73, %al4897; AVX512BW-SLOW-NEXT:    kmovd %eax, %k14898; AVX512BW-SLOW-NEXT:    vpexpandd %ymm0, %ymm1 {%k1} {z}4899; AVX512BW-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]4900; AVX512BW-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm24901; AVX512BW-SLOW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2,3]4902; AVX512BW-SLOW-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm04903; AVX512BW-SLOW-NEXT:    vpaddb (%rdx), %zmm0, %zmm04904; AVX512BW-SLOW-NEXT:    vmovdqa64 %zmm0, (%rcx)4905; AVX512BW-SLOW-NEXT:    vzeroupper4906; AVX512BW-SLOW-NEXT:    retq4907;4908; AVX512BW-FAST-LABEL: vec384_v12i32_to_v4i96_factor3:4909; AVX512BW-FAST:       # %bb.0:4910; AVX512BW-FAST-NEXT:    vmovdqa (%rdi), %ymm04911; AVX512BW-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm04912; AVX512BW-FAST-NEXT:    movb $73, %al4913; AVX512BW-FAST-NEXT:    kmovd %eax, %k14914; AVX512BW-FAST-NEXT:    vpexpandd %ymm0, %ymm1 {%k1} {z}4915; AVX512BW-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero4916; AVX512BW-FAST-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm04917; AVX512BW-FAST-NEXT:    vpaddb (%rdx), %zmm0, %zmm04918; AVX512BW-FAST-NEXT:    vmovdqa64 %zmm0, (%rcx)4919; AVX512BW-FAST-NEXT:    vzeroupper4920; AVX512BW-FAST-NEXT:    retq4921  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 644922  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 644923  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias4924  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>4925  %in.vec.cast = bitcast <48 x i8> %in.vec.trunc to <12 x i32>4926  %zextd.vec = shufflevector <12 x i32> %in.vec.cast, <12 x i32> zeroinitializer, <12 x i32> <i32 0, i32 13, i32 14, i32 1, i32 16, i32 17, i32 2, i32 19, i32 20, i32 3, i32 22, i32 23>4927  %out.bytevec = bitcast <12 x i32> %zextd.vec to <48 x i8>4928  %out.bytevec.padded = shufflevector <48 x i8> %out.bytevec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>4929  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 644930  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias4931  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 644932  ret void4933}4934 4935define void @vec384_v12i32_to_v3i128_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {4936; SSE2-LABEL: vec384_v12i32_to_v3i128_factor4:4937; SSE2:       # %bb.0:4938; SSE2-NEXT:    movdqa (%rdi), %xmm04939; SSE2-NEXT:    paddb (%rsi), %xmm04940; SSE2-NEXT:    xorps %xmm1, %xmm14941; SSE2-NEXT:    xorps %xmm2, %xmm24942; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3]4943; SSE2-NEXT:    movdqa %xmm0, %xmm34944; SSE2-NEXT:    psrldq {{.*#+}} xmm3 = xmm3[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero4945; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,2],xmm1[2,3]4946; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,0],xmm1[1,0]4947; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[2,3]4948; SSE2-NEXT:    paddb 16(%rdx), %xmm04949; SSE2-NEXT:    paddb 32(%rdx), %xmm34950; SSE2-NEXT:    paddb (%rdx), %xmm24951; SSE2-NEXT:    movdqa %xmm2, (%rcx)4952; SSE2-NEXT:    movdqa %xmm3, 32(%rcx)4953; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)4954; SSE2-NEXT:    retq4955;4956; SSE42-LABEL: vec384_v12i32_to_v3i128_factor4:4957; SSE42:       # %bb.0:4958; SSE42-NEXT:    movdqa (%rdi), %xmm04959; SSE42-NEXT:    paddb (%rsi), %xmm04960; SSE42-NEXT:    pxor %xmm1, %xmm14961; SSE42-NEXT:    pxor %xmm2, %xmm24962; SSE42-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1],xmm2[2,3,4,5,6,7]4963; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]4964; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm1[2,3,4,5,6,7]4965; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]4966; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]4967; SSE42-NEXT:    paddb 16(%rdx), %xmm04968; SSE42-NEXT:    paddb 32(%rdx), %xmm34969; SSE42-NEXT:    paddb (%rdx), %xmm24970; SSE42-NEXT:    movdqa %xmm2, (%rcx)4971; SSE42-NEXT:    movdqa %xmm3, 32(%rcx)4972; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)4973; SSE42-NEXT:    retq4974;4975; AVX-LABEL: vec384_v12i32_to_v3i128_factor4:4976; AVX:       # %bb.0:4977; AVX-NEXT:    vmovdqa (%rdi), %xmm04978; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm04979; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm14980; AVX-NEXT:    vpblendw {{.*#+}} xmm2 = xmm0[0,1],xmm1[2,3,4,5,6,7]4981; AVX-NEXT:    vinsertps {{.*#+}} xmm3 = xmm0[1],zero,zero,zero4982; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]4983; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]4984; AVX-NEXT:    vpaddb 32(%rdx), %xmm0, %xmm04985; AVX-NEXT:    vpaddb 16(%rdx), %xmm3, %xmm14986; AVX-NEXT:    vpaddb (%rdx), %xmm2, %xmm24987; AVX-NEXT:    vmovdqa %xmm2, (%rcx)4988; AVX-NEXT:    vmovdqa %xmm1, 16(%rcx)4989; AVX-NEXT:    vmovdqa %xmm0, 32(%rcx)4990; AVX-NEXT:    retq4991;4992; AVX2-SLOW-LABEL: vec384_v12i32_to_v3i128_factor4:4993; AVX2-SLOW:       # %bb.0:4994; AVX2-SLOW-NEXT:    vpxor %xmm0, %xmm0, %xmm04995; AVX2-SLOW-NEXT:    vmovdqa (%rdi), %xmm14996; AVX2-SLOW-NEXT:    vpaddb (%rsi), %xmm1, %xmm14997; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]4998; AVX2-SLOW-NEXT:    vpxor %xmm3, %xmm3, %xmm34999; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3]5000; AVX2-SLOW-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero5001; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]5002; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3],ymm1[4],ymm0[5,6,7]5003; AVX2-SLOW-NEXT:    vpaddb (%rdx), %ymm0, %ymm05004; AVX2-SLOW-NEXT:    vpaddb 32(%rdx), %ymm2, %ymm15005; AVX2-SLOW-NEXT:    vmovdqa %ymm1, 32(%rcx)5006; AVX2-SLOW-NEXT:    vmovdqa %ymm0, (%rcx)5007; AVX2-SLOW-NEXT:    vzeroupper5008; AVX2-SLOW-NEXT:    retq5009;5010; AVX2-FAST-PERLANE-LABEL: vec384_v12i32_to_v3i128_factor4:5011; AVX2-FAST-PERLANE:       # %bb.0:5012; AVX2-FAST-PERLANE-NEXT:    vpxor %xmm0, %xmm0, %xmm05013; AVX2-FAST-PERLANE-NEXT:    vmovdqa (%rdi), %xmm15014; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rsi), %xmm1, %xmm15015; AVX2-FAST-PERLANE-NEXT:    vpshufb {{.*#+}} xmm2 = xmm1[8,9,10,11],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero5016; AVX2-FAST-PERLANE-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero5017; AVX2-FAST-PERLANE-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]5018; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3],ymm1[4],ymm0[5,6,7]5019; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rdx), %ymm0, %ymm05020; AVX2-FAST-PERLANE-NEXT:    vpaddb 32(%rdx), %ymm2, %ymm15021; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm1, 32(%rcx)5022; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm0, (%rcx)5023; AVX2-FAST-PERLANE-NEXT:    vzeroupper5024; AVX2-FAST-PERLANE-NEXT:    retq5025;5026; AVX2-FAST-LABEL: vec384_v12i32_to_v3i128_factor4:5027; AVX2-FAST:       # %bb.0:5028; AVX2-FAST-NEXT:    vmovdqa (%rdi), %ymm05029; AVX2-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm05030; AVX2-FAST-NEXT:    vpxor %xmm1, %xmm1, %xmm15031; AVX2-FAST-NEXT:    vpmovsxbq {{.*#+}} ymm2 = [0,0,1,0]5032; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm2, %ymm25033; AVX2-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3],ymm2[4],ymm1[5,6,7]5034; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8,9,10,11],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero5035; AVX2-FAST-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm05036; AVX2-FAST-NEXT:    vpaddb (%rdx), %ymm1, %ymm15037; AVX2-FAST-NEXT:    vmovdqa %ymm1, (%rcx)5038; AVX2-FAST-NEXT:    vmovdqa %ymm0, 32(%rcx)5039; AVX2-FAST-NEXT:    vzeroupper5040; AVX2-FAST-NEXT:    retq5041;5042; AVX512F-LABEL: vec384_v12i32_to_v3i128_factor4:5043; AVX512F:       # %bb.0:5044; AVX512F-NEXT:    vmovdqa (%rdi), %ymm05045; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm05046; AVX512F-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [16,1,2,3,17,5,6,7,18,9,10,11,0,0,0,0]5047; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm25048; AVX512F-NEXT:    vpermt2d %zmm0, %zmm1, %zmm25049; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm05050; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm05051; AVX512F-NEXT:    vpaddb (%rdx), %ymm2, %ymm15052; AVX512F-NEXT:    vmovdqa %ymm1, (%rcx)5053; AVX512F-NEXT:    vmovdqa %ymm0, 32(%rcx)5054; AVX512F-NEXT:    vzeroupper5055; AVX512F-NEXT:    retq5056;5057; AVX512BW-SLOW-LABEL: vec384_v12i32_to_v3i128_factor4:5058; AVX512BW-SLOW:       # %bb.0:5059; AVX512BW-SLOW-NEXT:    vmovdqa (%rdi), %ymm05060; AVX512BW-SLOW-NEXT:    vpaddb (%rsi), %ymm0, %ymm05061; AVX512BW-SLOW-NEXT:    movb $17, %al5062; AVX512BW-SLOW-NEXT:    kmovd %eax, %k15063; AVX512BW-SLOW-NEXT:    vpexpandd %ymm0, %ymm1 {%k1} {z}5064; AVX512BW-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]5065; AVX512BW-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm25066; AVX512BW-SLOW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]5067; AVX512BW-SLOW-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm05068; AVX512BW-SLOW-NEXT:    vpaddb (%rdx), %zmm0, %zmm05069; AVX512BW-SLOW-NEXT:    vmovdqa64 %zmm0, (%rcx)5070; AVX512BW-SLOW-NEXT:    vzeroupper5071; AVX512BW-SLOW-NEXT:    retq5072;5073; AVX512BW-FAST-LABEL: vec384_v12i32_to_v3i128_factor4:5074; AVX512BW-FAST:       # %bb.0:5075; AVX512BW-FAST-NEXT:    vmovdqa (%rdi), %ymm05076; AVX512BW-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm05077; AVX512BW-FAST-NEXT:    movb $17, %al5078; AVX512BW-FAST-NEXT:    kmovd %eax, %k15079; AVX512BW-FAST-NEXT:    vpexpandd %ymm0, %ymm1 {%k1} {z}5080; AVX512BW-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8,9,10,11],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero5081; AVX512BW-FAST-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm05082; AVX512BW-FAST-NEXT:    vpaddb (%rdx), %zmm0, %zmm05083; AVX512BW-FAST-NEXT:    vmovdqa64 %zmm0, (%rcx)5084; AVX512BW-FAST-NEXT:    vzeroupper5085; AVX512BW-FAST-NEXT:    retq5086  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 645087  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 645088  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias5089  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>5090  %in.vec.cast = bitcast <48 x i8> %in.vec.trunc to <12 x i32>5091  %zextd.vec = shufflevector <12 x i32> %in.vec.cast, <12 x i32> zeroinitializer, <12 x i32> <i32 0, i32 13, i32 14, i32 15, i32 1, i32 17, i32 18, i32 19, i32 2, i32 21, i32 22, i32 23>5092  %out.bytevec = bitcast <12 x i32> %zextd.vec to <48 x i8>5093  %out.bytevec.padded = shufflevector <48 x i8> %out.bytevec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>5094  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 645095  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias5096  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 645097  ret void5098}5099 5100define void @vec384_v12i32_to_v2i192_factor6(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {5101; SSE2-LABEL: vec384_v12i32_to_v2i192_factor6:5102; SSE2:       # %bb.0:5103; SSE2-NEXT:    movdqa (%rdi), %xmm05104; SSE2-NEXT:    paddb (%rsi), %xmm05105; SSE2-NEXT:    xorps %xmm1, %xmm15106; SSE2-NEXT:    xorps %xmm2, %xmm25107; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3]5108; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,0],xmm1[3,0]5109; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0,2]5110; SSE2-NEXT:    movaps 32(%rdx), %xmm05111; SSE2-NEXT:    paddb 16(%rdx), %xmm15112; SSE2-NEXT:    paddb (%rdx), %xmm25113; SSE2-NEXT:    movaps %xmm0, 32(%rcx)5114; SSE2-NEXT:    movdqa %xmm2, (%rcx)5115; SSE2-NEXT:    movdqa %xmm1, 16(%rcx)5116; SSE2-NEXT:    retq5117;5118; SSE42-LABEL: vec384_v12i32_to_v2i192_factor6:5119; SSE42:       # %bb.0:5120; SSE42-NEXT:    movdqa (%rdi), %xmm05121; SSE42-NEXT:    paddb (%rsi), %xmm05122; SSE42-NEXT:    pxor %xmm1, %xmm15123; SSE42-NEXT:    pxor %xmm2, %xmm25124; SSE42-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1],xmm2[2,3,4,5,6,7]5125; SSE42-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero5126; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]5127; SSE42-NEXT:    movaps 32(%rdx), %xmm15128; SSE42-NEXT:    paddb 16(%rdx), %xmm05129; SSE42-NEXT:    paddb (%rdx), %xmm25130; SSE42-NEXT:    movaps %xmm1, 32(%rcx)5131; SSE42-NEXT:    movdqa %xmm2, (%rcx)5132; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)5133; SSE42-NEXT:    retq5134;5135; AVX-LABEL: vec384_v12i32_to_v2i192_factor6:5136; AVX:       # %bb.0:5137; AVX-NEXT:    vmovdqa (%rdi), %xmm05138; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm05139; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm15140; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7]5141; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,zero,xmm0[1],zero5142; AVX-NEXT:    vmovaps 32(%rdx), %ymm25143; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm05144; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm15145; AVX-NEXT:    vmovaps %ymm2, 32(%rcx)5146; AVX-NEXT:    vmovdqa %xmm1, (%rcx)5147; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)5148; AVX-NEXT:    vzeroupper5149; AVX-NEXT:    retq5150;5151; AVX2-SLOW-LABEL: vec384_v12i32_to_v2i192_factor6:5152; AVX2-SLOW:       # %bb.0:5153; AVX2-SLOW-NEXT:    vpxor %xmm0, %xmm0, %xmm05154; AVX2-SLOW-NEXT:    vmovdqa (%rdi), %xmm15155; AVX2-SLOW-NEXT:    vpaddb (%rsi), %xmm1, %xmm15156; AVX2-SLOW-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero5157; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,2,1]5158; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5],ymm1[6],ymm0[7]5159; AVX2-SLOW-NEXT:    vmovaps 32(%rdx), %ymm15160; AVX2-SLOW-NEXT:    vpaddb (%rdx), %ymm0, %ymm05161; AVX2-SLOW-NEXT:    vmovaps %ymm1, 32(%rcx)5162; AVX2-SLOW-NEXT:    vmovdqa %ymm0, (%rcx)5163; AVX2-SLOW-NEXT:    vzeroupper5164; AVX2-SLOW-NEXT:    retq5165;5166; AVX2-FAST-PERLANE-LABEL: vec384_v12i32_to_v2i192_factor6:5167; AVX2-FAST-PERLANE:       # %bb.0:5168; AVX2-FAST-PERLANE-NEXT:    vpxor %xmm0, %xmm0, %xmm05169; AVX2-FAST-PERLANE-NEXT:    vmovdqa (%rdi), %xmm15170; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rsi), %xmm1, %xmm15171; AVX2-FAST-PERLANE-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero5172; AVX2-FAST-PERLANE-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,2,1]5173; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5],ymm1[6],ymm0[7]5174; AVX2-FAST-PERLANE-NEXT:    vmovaps 32(%rdx), %ymm15175; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rdx), %ymm0, %ymm05176; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, 32(%rcx)5177; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm0, (%rcx)5178; AVX2-FAST-PERLANE-NEXT:    vzeroupper5179; AVX2-FAST-PERLANE-NEXT:    retq5180;5181; AVX2-FAST-LABEL: vec384_v12i32_to_v2i192_factor6:5182; AVX2-FAST:       # %bb.0:5183; AVX2-FAST-NEXT:    vmovdqa (%rdi), %ymm05184; AVX2-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm05185; AVX2-FAST-NEXT:    vpxor %xmm1, %xmm1, %xmm15186; AVX2-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,0,1,0,0,0,1,0]5187; AVX2-FAST-NEXT:    # ymm2 = mem[0,1,0,1]5188; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm2, %ymm05189; AVX2-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3,4,5],ymm0[6],ymm1[7]5190; AVX2-FAST-NEXT:    vmovaps 32(%rdx), %ymm15191; AVX2-FAST-NEXT:    vpaddb (%rdx), %ymm0, %ymm05192; AVX2-FAST-NEXT:    vmovaps %ymm1, 32(%rcx)5193; AVX2-FAST-NEXT:    vmovdqa %ymm0, (%rcx)5194; AVX2-FAST-NEXT:    vzeroupper5195; AVX2-FAST-NEXT:    retq5196;5197; AVX512F-LABEL: vec384_v12i32_to_v2i192_factor6:5198; AVX512F:       # %bb.0:5199; AVX512F-NEXT:    vmovdqa (%rdi), %ymm05200; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm05201; AVX512F-NEXT:    movb $65, %al5202; AVX512F-NEXT:    kmovw %eax, %k15203; AVX512F-NEXT:    vpexpandd %ymm0, %ymm0 {%k1} {z}5204; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm05205; AVX512F-NEXT:    vmovaps 32(%rdx), %ymm15206; AVX512F-NEXT:    vmovaps %ymm1, 32(%rcx)5207; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)5208; AVX512F-NEXT:    vzeroupper5209; AVX512F-NEXT:    retq5210;5211; AVX512BW-LABEL: vec384_v12i32_to_v2i192_factor6:5212; AVX512BW:       # %bb.0:5213; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm05214; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm05215; AVX512BW-NEXT:    movb $65, %al5216; AVX512BW-NEXT:    kmovd %eax, %k15217; AVX512BW-NEXT:    vpexpandd %ymm0, %ymm0 {%k1} {z}5218; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm05219; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)5220; AVX512BW-NEXT:    vzeroupper5221; AVX512BW-NEXT:    retq5222  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 645223  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 645224  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias5225  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>5226  %in.vec.cast = bitcast <48 x i8> %in.vec.trunc to <12 x i32>5227  %zextd.vec = shufflevector <12 x i32> %in.vec.cast, <12 x i32> zeroinitializer, <12 x i32> <i32 0, i32 13, i32 14, i32 15, i32 16, i32 17, i32 1, i32 19, i32 20, i32 21, i32 22, i32 23>5228  %out.bytevec = bitcast <12 x i32> %zextd.vec to <48 x i8>5229  %out.bytevec.padded = shufflevector <48 x i8> %out.bytevec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>5230  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 645231  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias5232  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 645233  ret void5234}5235 5236define void @vec384_v12i32_to_v1i384_factor12(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {5237; SSE2-LABEL: vec384_v12i32_to_v1i384_factor12:5238; SSE2:       # %bb.0:5239; SSE2-NEXT:    movdqa (%rdi), %xmm05240; SSE2-NEXT:    paddb (%rsi), %xmm05241; SSE2-NEXT:    xorps %xmm1, %xmm15242; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]5243; SSE2-NEXT:    movaps 16(%rdx), %xmm05244; SSE2-NEXT:    movaps 32(%rdx), %xmm25245; SSE2-NEXT:    paddb (%rdx), %xmm15246; SSE2-NEXT:    movaps %xmm0, 16(%rcx)5247; SSE2-NEXT:    movaps %xmm2, 32(%rcx)5248; SSE2-NEXT:    movdqa %xmm1, (%rcx)5249; SSE2-NEXT:    retq5250;5251; SSE42-LABEL: vec384_v12i32_to_v1i384_factor12:5252; SSE42:       # %bb.0:5253; SSE42-NEXT:    movdqa (%rdi), %xmm05254; SSE42-NEXT:    paddb (%rsi), %xmm05255; SSE42-NEXT:    pxor %xmm1, %xmm15256; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7]5257; SSE42-NEXT:    movaps 16(%rdx), %xmm05258; SSE42-NEXT:    movaps 32(%rdx), %xmm25259; SSE42-NEXT:    paddb (%rdx), %xmm15260; SSE42-NEXT:    movaps %xmm0, 16(%rcx)5261; SSE42-NEXT:    movaps %xmm2, 32(%rcx)5262; SSE42-NEXT:    movdqa %xmm1, (%rcx)5263; SSE42-NEXT:    retq5264;5265; AVX-LABEL: vec384_v12i32_to_v1i384_factor12:5266; AVX:       # %bb.0:5267; AVX-NEXT:    vmovdqa (%rdi), %xmm05268; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm05269; AVX-NEXT:    vmovaps 32(%rdx), %ymm15270; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm25271; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3,4,5,6,7]5272; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm05273; AVX-NEXT:    vmovaps 16(%rdx), %xmm25274; AVX-NEXT:    vmovaps %xmm2, 16(%rcx)5275; AVX-NEXT:    vmovaps %ymm1, 32(%rcx)5276; AVX-NEXT:    vmovdqa %xmm0, (%rcx)5277; AVX-NEXT:    vzeroupper5278; AVX-NEXT:    retq5279;5280; AVX2-LABEL: vec384_v12i32_to_v1i384_factor12:5281; AVX2:       # %bb.0:5282; AVX2-NEXT:    vmovdqa (%rdi), %ymm05283; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm05284; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm15285; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]5286; AVX2-NEXT:    vmovaps 32(%rdx), %ymm15287; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm05288; AVX2-NEXT:    vmovaps %ymm1, 32(%rcx)5289; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)5290; AVX2-NEXT:    vzeroupper5291; AVX2-NEXT:    retq5292;5293; AVX512F-LABEL: vec384_v12i32_to_v1i384_factor12:5294; AVX512F:       # %bb.0:5295; AVX512F-NEXT:    vmovdqa (%rdi), %ymm05296; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm05297; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm15298; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]5299; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm05300; AVX512F-NEXT:    vmovaps 32(%rdx), %ymm15301; AVX512F-NEXT:    vmovaps %ymm1, 32(%rcx)5302; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)5303; AVX512F-NEXT:    vzeroupper5304; AVX512F-NEXT:    retq5305;5306; AVX512BW-LABEL: vec384_v12i32_to_v1i384_factor12:5307; AVX512BW:       # %bb.0:5308; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm05309; AVX512BW-NEXT:    vpaddb (%rsi), %zmm0, %zmm05310; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm15311; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]5312; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm05313; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)5314; AVX512BW-NEXT:    vzeroupper5315; AVX512BW-NEXT:    retq5316  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 645317  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 645318  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias5319  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>5320  %in.vec.cast = bitcast <48 x i8> %in.vec.trunc to <12 x i32>5321  %zextd.vec = shufflevector <12 x i32> %in.vec.cast, <12 x i32> zeroinitializer, <12 x i32> <i32 0, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>5322  %out.bytevec = bitcast <12 x i32> %zextd.vec to <48 x i8>5323  %out.bytevec.padded = shufflevector <48 x i8> %out.bytevec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>5324  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 645325  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias5326  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 645327  ret void5328}5329 5330define void @vec384_v6i64_to_v3i128_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {5331; SSE-LABEL: vec384_v6i64_to_v3i128_factor2:5332; SSE:       # %bb.0:5333; SSE-NEXT:    movdqa (%rdi), %xmm05334; SSE-NEXT:    movdqa 16(%rdi), %xmm15335; SSE-NEXT:    paddb (%rsi), %xmm05336; SSE-NEXT:    paddb 16(%rsi), %xmm15337; SSE-NEXT:    movq {{.*#+}} xmm1 = xmm1[0],zero5338; SSE-NEXT:    movq {{.*#+}} xmm2 = xmm0[0],zero5339; SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero5340; SSE-NEXT:    paddb 16(%rdx), %xmm05341; SSE-NEXT:    paddb (%rdx), %xmm25342; SSE-NEXT:    paddb 32(%rdx), %xmm15343; SSE-NEXT:    movdqa %xmm1, 32(%rcx)5344; SSE-NEXT:    movdqa %xmm2, (%rcx)5345; SSE-NEXT:    movdqa %xmm0, 16(%rcx)5346; SSE-NEXT:    retq5347;5348; AVX-LABEL: vec384_v6i64_to_v3i128_factor2:5349; AVX:       # %bb.0:5350; AVX-NEXT:    vmovdqa (%rdi), %xmm05351; AVX-NEXT:    vmovdqa 16(%rdi), %xmm15352; AVX-NEXT:    vpaddb 16(%rsi), %xmm1, %xmm15353; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm05354; AVX-NEXT:    vxorpd %xmm2, %xmm2, %xmm25355; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm05356; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[3],ymm2[3]5357; AVX-NEXT:    vmovq {{.*#+}} xmm1 = xmm1[0],zero5358; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm25359; AVX-NEXT:    vpaddb 16(%rdx), %xmm2, %xmm25360; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm05361; AVX-NEXT:    vpaddb 32(%rdx), %xmm1, %xmm15362; AVX-NEXT:    vmovdqa %xmm1, 32(%rcx)5363; AVX-NEXT:    vmovdqa %xmm0, (%rcx)5364; AVX-NEXT:    vmovdqa %xmm2, 16(%rcx)5365; AVX-NEXT:    vzeroupper5366; AVX-NEXT:    retq5367;5368; AVX2-LABEL: vec384_v6i64_to_v3i128_factor2:5369; AVX2:       # %bb.0:5370; AVX2-NEXT:    vmovdqa (%rdi), %ymm05371; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm05372; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm15373; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[0,1,1,3]5374; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5],ymm1[6,7]5375; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm05376; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero5377; AVX2-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm05378; AVX2-NEXT:    vpaddb (%rdx), %ymm1, %ymm15379; AVX2-NEXT:    vmovdqa %ymm1, (%rcx)5380; AVX2-NEXT:    vmovdqa %ymm0, 32(%rcx)5381; AVX2-NEXT:    vzeroupper5382; AVX2-NEXT:    retq5383;5384; AVX512F-LABEL: vec384_v6i64_to_v3i128_factor2:5385; AVX512F:       # %bb.0:5386; AVX512F-NEXT:    vmovdqa (%rdi), %ymm05387; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm05388; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm15389; AVX512F-NEXT:    vpmovsxbq {{.*#+}} zmm2 = [0,9,1,11,2,13,0,0]5390; AVX512F-NEXT:    vpermi2q %zmm1, %zmm0, %zmm25391; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm05392; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm05393; AVX512F-NEXT:    vpaddb (%rdx), %ymm2, %ymm15394; AVX512F-NEXT:    vmovdqa %ymm1, (%rcx)5395; AVX512F-NEXT:    vmovdqa %ymm0, 32(%rcx)5396; AVX512F-NEXT:    vzeroupper5397; AVX512F-NEXT:    retq5398;5399; AVX512BW-LABEL: vec384_v6i64_to_v3i128_factor2:5400; AVX512BW:       # %bb.0:5401; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm05402; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm05403; AVX512BW-NEXT:    movb $5, %al5404; AVX512BW-NEXT:    kmovd %eax, %k15405; AVX512BW-NEXT:    vpexpandq %ymm0, %ymm1 {%k1} {z}5406; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm05407; AVX512BW-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero5408; AVX512BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm05409; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm05410; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)5411; AVX512BW-NEXT:    vzeroupper5412; AVX512BW-NEXT:    retq5413  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 645414  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 645415  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias5416  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>5417  %in.vec.cast = bitcast <48 x i8> %in.vec.trunc to <6 x i64>5418  %zextd.vec = shufflevector <6 x i64> %in.vec.cast, <6 x i64> zeroinitializer, <6 x i32> <i32 0, i32 7, i32 1, i32 9, i32 2, i32 11>5419  %out.bytevec = bitcast <6 x i64> %zextd.vec to <48 x i8>5420  %out.bytevec.padded = shufflevector <48 x i8> %out.bytevec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>5421  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 645422  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias5423  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 645424  ret void5425}5426 5427define void @vec384_v6i64_to_v2i192_factor3(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {5428; SSE2-LABEL: vec384_v6i64_to_v2i192_factor3:5429; SSE2:       # %bb.0:5430; SSE2-NEXT:    movdqa (%rdi), %xmm05431; SSE2-NEXT:    paddb (%rsi), %xmm05432; SSE2-NEXT:    pxor %xmm1, %xmm15433; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm1 = xmm1[1],xmm0[1]5434; SSE2-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero5435; SSE2-NEXT:    movaps 32(%rdx), %xmm25436; SSE2-NEXT:    paddb (%rdx), %xmm05437; SSE2-NEXT:    paddb 16(%rdx), %xmm15438; SSE2-NEXT:    movaps %xmm2, 32(%rcx)5439; SSE2-NEXT:    movdqa %xmm1, 16(%rcx)5440; SSE2-NEXT:    movdqa %xmm0, (%rcx)5441; SSE2-NEXT:    retq5442;5443; SSE42-LABEL: vec384_v6i64_to_v2i192_factor3:5444; SSE42:       # %bb.0:5445; SSE42-NEXT:    movdqa (%rdi), %xmm05446; SSE42-NEXT:    paddb (%rsi), %xmm05447; SSE42-NEXT:    pxor %xmm1, %xmm15448; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm0[4,5,6,7]5449; SSE42-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero5450; SSE42-NEXT:    movaps 32(%rdx), %xmm25451; SSE42-NEXT:    paddb (%rdx), %xmm05452; SSE42-NEXT:    paddb 16(%rdx), %xmm15453; SSE42-NEXT:    movaps %xmm2, 32(%rcx)5454; SSE42-NEXT:    movdqa %xmm1, 16(%rcx)5455; SSE42-NEXT:    movdqa %xmm0, (%rcx)5456; SSE42-NEXT:    retq5457;5458; AVX-LABEL: vec384_v6i64_to_v2i192_factor3:5459; AVX:       # %bb.0:5460; AVX-NEXT:    vmovdqa (%rdi), %xmm05461; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm05462; AVX-NEXT:    vperm2f128 {{.*#+}} ymm1 = zero,zero,ymm0[0,1]5463; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]5464; AVX-NEXT:    vmovaps 32(%rdx), %ymm15465; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm25466; AVX-NEXT:    vpaddb 16(%rdx), %xmm2, %xmm25467; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm05468; AVX-NEXT:    vmovaps %ymm1, 32(%rcx)5469; AVX-NEXT:    vmovdqa %xmm0, (%rcx)5470; AVX-NEXT:    vmovdqa %xmm2, 16(%rcx)5471; AVX-NEXT:    vzeroupper5472; AVX-NEXT:    retq5473;5474; AVX2-LABEL: vec384_v6i64_to_v2i192_factor3:5475; AVX2:       # %bb.0:5476; AVX2-NEXT:    vmovdqa (%rdi), %ymm05477; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm05478; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm15479; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,2,1]5480; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3,4,5],ymm0[6,7]5481; AVX2-NEXT:    vmovaps 32(%rdx), %ymm15482; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm05483; AVX2-NEXT:    vmovaps %ymm1, 32(%rcx)5484; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)5485; AVX2-NEXT:    vzeroupper5486; AVX2-NEXT:    retq5487;5488; AVX512F-LABEL: vec384_v6i64_to_v2i192_factor3:5489; AVX512F:       # %bb.0:5490; AVX512F-NEXT:    vmovdqa (%rdi), %ymm05491; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm05492; AVX512F-NEXT:    movb $9, %al5493; AVX512F-NEXT:    kmovw %eax, %k15494; AVX512F-NEXT:    vpexpandq %ymm0, %ymm0 {%k1} {z}5495; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm05496; AVX512F-NEXT:    vmovaps 32(%rdx), %ymm15497; AVX512F-NEXT:    vmovaps %ymm1, 32(%rcx)5498; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)5499; AVX512F-NEXT:    vzeroupper5500; AVX512F-NEXT:    retq5501;5502; AVX512BW-LABEL: vec384_v6i64_to_v2i192_factor3:5503; AVX512BW:       # %bb.0:5504; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm05505; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm05506; AVX512BW-NEXT:    movb $9, %al5507; AVX512BW-NEXT:    kmovd %eax, %k15508; AVX512BW-NEXT:    vpexpandq %ymm0, %ymm0 {%k1} {z}5509; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm05510; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)5511; AVX512BW-NEXT:    vzeroupper5512; AVX512BW-NEXT:    retq5513  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 645514  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 645515  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias5516  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>5517  %in.vec.cast = bitcast <48 x i8> %in.vec.trunc to <6 x i64>5518  %zextd.vec = shufflevector <6 x i64> %in.vec.cast, <6 x i64> zeroinitializer, <6 x i32> <i32 0, i32 7, i32 8, i32 1, i32 10, i32 11>5519  %out.bytevec = bitcast <6 x i64> %zextd.vec to <48 x i8>5520  %out.bytevec.padded = shufflevector <48 x i8> %out.bytevec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>5521  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 645522  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias5523  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 645524  ret void5525}5526 5527define void @vec384_v6i64_to_v1i384_factor6(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {5528; SSE-LABEL: vec384_v6i64_to_v1i384_factor6:5529; SSE:       # %bb.0:5530; SSE-NEXT:    movdqa (%rdi), %xmm05531; SSE-NEXT:    paddb (%rsi), %xmm05532; SSE-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero5533; SSE-NEXT:    movaps 16(%rdx), %xmm15534; SSE-NEXT:    movaps 32(%rdx), %xmm25535; SSE-NEXT:    paddb (%rdx), %xmm05536; SSE-NEXT:    movaps %xmm1, 16(%rcx)5537; SSE-NEXT:    movaps %xmm2, 32(%rcx)5538; SSE-NEXT:    movdqa %xmm0, (%rcx)5539; SSE-NEXT:    retq5540;5541; AVX-LABEL: vec384_v6i64_to_v1i384_factor6:5542; AVX:       # %bb.0:5543; AVX-NEXT:    vmovdqa (%rdi), %xmm05544; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm05545; AVX-NEXT:    vmovaps 32(%rdx), %ymm15546; AVX-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero5547; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm05548; AVX-NEXT:    vmovaps 16(%rdx), %xmm25549; AVX-NEXT:    vmovaps %xmm2, 16(%rcx)5550; AVX-NEXT:    vmovaps %ymm1, 32(%rcx)5551; AVX-NEXT:    vmovdqa %xmm0, (%rcx)5552; AVX-NEXT:    vzeroupper5553; AVX-NEXT:    retq5554;5555; AVX2-LABEL: vec384_v6i64_to_v1i384_factor6:5556; AVX2:       # %bb.0:5557; AVX2-NEXT:    vmovdqa (%rdi), %ymm05558; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm05559; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero5560; AVX2-NEXT:    vmovaps 32(%rdx), %ymm15561; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm05562; AVX2-NEXT:    vmovaps %ymm1, 32(%rcx)5563; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)5564; AVX2-NEXT:    vzeroupper5565; AVX2-NEXT:    retq5566;5567; AVX512F-LABEL: vec384_v6i64_to_v1i384_factor6:5568; AVX512F:       # %bb.0:5569; AVX512F-NEXT:    vmovdqa (%rdi), %ymm05570; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm05571; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero5572; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm05573; AVX512F-NEXT:    vmovaps 32(%rdx), %ymm15574; AVX512F-NEXT:    vmovaps %ymm1, 32(%rcx)5575; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)5576; AVX512F-NEXT:    vzeroupper5577; AVX512F-NEXT:    retq5578;5579; AVX512BW-LABEL: vec384_v6i64_to_v1i384_factor6:5580; AVX512BW:       # %bb.0:5581; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm05582; AVX512BW-NEXT:    vpaddb (%rsi), %zmm0, %zmm05583; AVX512BW-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero5584; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm05585; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)5586; AVX512BW-NEXT:    vzeroupper5587; AVX512BW-NEXT:    retq5588  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 645589  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 645590  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias5591  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>5592  %in.vec.cast = bitcast <48 x i8> %in.vec.trunc to <6 x i64>5593  %zextd.vec = shufflevector <6 x i64> %in.vec.cast, <6 x i64> zeroinitializer, <6 x i32> <i32 0, i32 7, i32 8, i32 9, i32 10, i32 11>5594  %out.bytevec = bitcast <6 x i64> %zextd.vec to <48 x i8>5595  %out.bytevec.padded = shufflevector <48 x i8> %out.bytevec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>5596  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 645597  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias5598  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 645599  ret void5600}5601 5602define void @vec384_v3i128_to_v1i384_factor3(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {5603; SSE-LABEL: vec384_v3i128_to_v1i384_factor3:5604; SSE:       # %bb.0:5605; SSE-NEXT:    movdqa (%rdi), %xmm05606; SSE-NEXT:    paddb (%rsi), %xmm05607; SSE-NEXT:    movaps 16(%rdx), %xmm15608; SSE-NEXT:    movaps 32(%rdx), %xmm25609; SSE-NEXT:    paddb (%rdx), %xmm05610; SSE-NEXT:    movaps %xmm1, 16(%rcx)5611; SSE-NEXT:    movaps %xmm2, 32(%rcx)5612; SSE-NEXT:    movdqa %xmm0, (%rcx)5613; SSE-NEXT:    retq5614;5615; AVX-LABEL: vec384_v3i128_to_v1i384_factor3:5616; AVX:       # %bb.0:5617; AVX-NEXT:    vmovdqa (%rdi), %xmm05618; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm05619; AVX-NEXT:    vmovaps 32(%rdx), %ymm15620; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm05621; AVX-NEXT:    vmovaps 16(%rdx), %xmm25622; AVX-NEXT:    vmovaps %xmm2, 16(%rcx)5623; AVX-NEXT:    vmovaps %ymm1, 32(%rcx)5624; AVX-NEXT:    vmovdqa %xmm0, (%rcx)5625; AVX-NEXT:    vzeroupper5626; AVX-NEXT:    retq5627;5628; AVX2-LABEL: vec384_v3i128_to_v1i384_factor3:5629; AVX2:       # %bb.0:5630; AVX2-NEXT:    vmovdqa (%rdi), %xmm05631; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm05632; AVX2-NEXT:    vmovaps 32(%rdx), %ymm15633; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm05634; AVX2-NEXT:    vmovaps %ymm1, 32(%rcx)5635; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)5636; AVX2-NEXT:    vzeroupper5637; AVX2-NEXT:    retq5638;5639; AVX512F-LABEL: vec384_v3i128_to_v1i384_factor3:5640; AVX512F:       # %bb.0:5641; AVX512F-NEXT:    vmovdqa (%rdi), %xmm05642; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm05643; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm05644; AVX512F-NEXT:    vmovaps 32(%rdx), %ymm15645; AVX512F-NEXT:    vmovaps %ymm1, 32(%rcx)5646; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)5647; AVX512F-NEXT:    vzeroupper5648; AVX512F-NEXT:    retq5649;5650; AVX512BW-LABEL: vec384_v3i128_to_v1i384_factor3:5651; AVX512BW:       # %bb.0:5652; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm05653; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm05654; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm05655; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)5656; AVX512BW-NEXT:    vzeroupper5657; AVX512BW-NEXT:    retq5658  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 645659  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 645660  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias5661  %in.vec.trunc = shufflevector <64 x i8> %in.vec, <64 x i8> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>5662  %in.vec.cast = bitcast <48 x i8> %in.vec.trunc to <3 x i128>5663  %zextd.vec = shufflevector <3 x i128> %in.vec.cast, <3 x i128> zeroinitializer, <3 x i32> <i32 0, i32 4, i32 5>5664  %out.bytevec = bitcast <3 x i128> %zextd.vec to <48 x i8>5665  %out.bytevec.padded = shufflevector <48 x i8> %out.bytevec, <48 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>5666  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 645667  %out.vec = add <64 x i8> %out.bytevec.padded, %out.vec.bias5668  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 645669  ret void5670}5671 5672define void @vec512_v64i8_to_v32i16_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {5673; SSE2-LABEL: vec512_v64i8_to_v32i16_factor2:5674; SSE2:       # %bb.0:5675; SSE2-NEXT:    movdqa (%rdi), %xmm05676; SSE2-NEXT:    movdqa 16(%rdi), %xmm15677; SSE2-NEXT:    paddb (%rsi), %xmm05678; SSE2-NEXT:    paddb 16(%rsi), %xmm15679; SSE2-NEXT:    pxor %xmm2, %xmm25680; SSE2-NEXT:    movdqa %xmm1, %xmm35681; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]5682; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]5683; SSE2-NEXT:    movdqa %xmm0, %xmm45684; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]5685; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]5686; SSE2-NEXT:    paddb 16(%rdx), %xmm05687; SSE2-NEXT:    paddb (%rdx), %xmm45688; SSE2-NEXT:    paddb 48(%rdx), %xmm15689; SSE2-NEXT:    paddb 32(%rdx), %xmm35690; SSE2-NEXT:    movdqa %xmm3, 32(%rcx)5691; SSE2-NEXT:    movdqa %xmm1, 48(%rcx)5692; SSE2-NEXT:    movdqa %xmm4, (%rcx)5693; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)5694; SSE2-NEXT:    retq5695;5696; SSE42-LABEL: vec512_v64i8_to_v32i16_factor2:5697; SSE42:       # %bb.0:5698; SSE42-NEXT:    movdqa (%rdi), %xmm05699; SSE42-NEXT:    movdqa 16(%rdi), %xmm15700; SSE42-NEXT:    paddb (%rsi), %xmm05701; SSE42-NEXT:    paddb 16(%rsi), %xmm15702; SSE42-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero5703; SSE42-NEXT:    pxor %xmm3, %xmm35704; SSE42-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm3[8],xmm1[9],xmm3[9],xmm1[10],xmm3[10],xmm1[11],xmm3[11],xmm1[12],xmm3[12],xmm1[13],xmm3[13],xmm1[14],xmm3[14],xmm1[15],xmm3[15]5705; SSE42-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero5706; SSE42-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm3[8],xmm0[9],xmm3[9],xmm0[10],xmm3[10],xmm0[11],xmm3[11],xmm0[12],xmm3[12],xmm0[13],xmm3[13],xmm0[14],xmm3[14],xmm0[15],xmm3[15]5707; SSE42-NEXT:    paddb 16(%rdx), %xmm05708; SSE42-NEXT:    paddb (%rdx), %xmm45709; SSE42-NEXT:    paddb 48(%rdx), %xmm15710; SSE42-NEXT:    paddb 32(%rdx), %xmm25711; SSE42-NEXT:    movdqa %xmm2, 32(%rcx)5712; SSE42-NEXT:    movdqa %xmm1, 48(%rcx)5713; SSE42-NEXT:    movdqa %xmm4, (%rcx)5714; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)5715; SSE42-NEXT:    retq5716;5717; AVX-LABEL: vec512_v64i8_to_v32i16_factor2:5718; AVX:       # %bb.0:5719; AVX-NEXT:    vmovdqa (%rdi), %xmm05720; AVX-NEXT:    vmovdqa 16(%rdi), %xmm15721; AVX-NEXT:    vpaddb 16(%rsi), %xmm1, %xmm15722; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm05723; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero5724; AVX-NEXT:    vpxor %xmm3, %xmm3, %xmm35725; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm3[8],xmm0[9],xmm3[9],xmm0[10],xmm3[10],xmm0[11],xmm3[11],xmm0[12],xmm3[12],xmm0[13],xmm3[13],xmm0[14],xmm3[14],xmm0[15],xmm3[15]5726; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero5727; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm3[8],xmm1[9],xmm3[9],xmm1[10],xmm3[10],xmm1[11],xmm3[11],xmm1[12],xmm3[12],xmm1[13],xmm3[13],xmm1[14],xmm3[14],xmm1[15],xmm3[15]5728; AVX-NEXT:    vpaddb 48(%rdx), %xmm1, %xmm15729; AVX-NEXT:    vpaddb 32(%rdx), %xmm4, %xmm35730; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm05731; AVX-NEXT:    vpaddb (%rdx), %xmm2, %xmm25732; AVX-NEXT:    vmovdqa %xmm2, (%rcx)5733; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)5734; AVX-NEXT:    vmovdqa %xmm3, 32(%rcx)5735; AVX-NEXT:    vmovdqa %xmm1, 48(%rcx)5736; AVX-NEXT:    retq5737;5738; AVX2-LABEL: vec512_v64i8_to_v32i16_factor2:5739; AVX2:       # %bb.0:5740; AVX2-NEXT:    vmovdqa (%rdi), %ymm05741; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm05742; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero5743; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm05744; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero5745; AVX2-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm05746; AVX2-NEXT:    vpaddb (%rdx), %ymm1, %ymm15747; AVX2-NEXT:    vmovdqa %ymm1, (%rcx)5748; AVX2-NEXT:    vmovdqa %ymm0, 32(%rcx)5749; AVX2-NEXT:    vzeroupper5750; AVX2-NEXT:    retq5751;5752; AVX512F-LABEL: vec512_v64i8_to_v32i16_factor2:5753; AVX512F:       # %bb.0:5754; AVX512F-NEXT:    vmovdqa (%rdi), %ymm05755; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm05756; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero5757; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm05758; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero5759; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm05760; AVX512F-NEXT:    vpaddb (%rdx), %ymm1, %ymm15761; AVX512F-NEXT:    vmovdqa %ymm1, (%rcx)5762; AVX512F-NEXT:    vmovdqa %ymm0, 32(%rcx)5763; AVX512F-NEXT:    vzeroupper5764; AVX512F-NEXT:    retq5765;5766; AVX512BW-LABEL: vec512_v64i8_to_v32i16_factor2:5767; AVX512BW:       # %bb.0:5768; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm05769; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm05770; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero5771; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm05772; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)5773; AVX512BW-NEXT:    vzeroupper5774; AVX512BW-NEXT:    retq5775  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 645776  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 645777  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias5778  %zextd.vec = shufflevector <64 x i8> %in.vec, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 65, i32 1, i32 67, i32 2, i32 69, i32 3, i32 71, i32 4, i32 73, i32 5, i32 75, i32 6, i32 77, i32 7, i32 79, i32 8, i32 81, i32 9, i32 83, i32 10, i32 85, i32 11, i32 87, i32 12, i32 89, i32 13, i32 91, i32 14, i32 93, i32 15, i32 95, i32 16, i32 97, i32 17, i32 99, i32 18, i32 101, i32 19, i32 103, i32 20, i32 105, i32 21, i32 107, i32 22, i32 109, i32 23, i32 111, i32 24, i32 113, i32 25, i32 115, i32 26, i32 117, i32 27, i32 119, i32 28, i32 121, i32 29, i32 123, i32 30, i32 125, i32 31, i32 127>5779  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 645780  %out.vec = add <64 x i8> %zextd.vec, %out.vec.bias5781  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 645782  ret void5783}5784 5785define void @vec512_v64i8_to_v16i32_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {5786; SSE2-LABEL: vec512_v64i8_to_v16i32_factor4:5787; SSE2:       # %bb.0:5788; SSE2-NEXT:    movdqa (%rdi), %xmm05789; SSE2-NEXT:    paddb (%rsi), %xmm05790; SSE2-NEXT:    pxor %xmm1, %xmm15791; SSE2-NEXT:    movdqa %xmm0, %xmm25792; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]5793; SSE2-NEXT:    movdqa %xmm2, %xmm35794; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]5795; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]5796; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]5797; SSE2-NEXT:    movdqa %xmm0, %xmm45798; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]5799; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]5800; SSE2-NEXT:    paddb 16(%rdx), %xmm05801; SSE2-NEXT:    paddb (%rdx), %xmm45802; SSE2-NEXT:    paddb 48(%rdx), %xmm25803; SSE2-NEXT:    paddb 32(%rdx), %xmm35804; SSE2-NEXT:    movdqa %xmm3, 32(%rcx)5805; SSE2-NEXT:    movdqa %xmm2, 48(%rcx)5806; SSE2-NEXT:    movdqa %xmm4, (%rcx)5807; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)5808; SSE2-NEXT:    retq5809;5810; SSE42-LABEL: vec512_v64i8_to_v16i32_factor4:5811; SSE42:       # %bb.0:5812; SSE42-NEXT:    movdqa (%rdi), %xmm05813; SSE42-NEXT:    paddb (%rsi), %xmm05814; SSE42-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero5815; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]5816; SSE42-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero5817; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]5818; SSE42-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero5819; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]5820; SSE42-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero5821; SSE42-NEXT:    paddb 16(%rdx), %xmm05822; SSE42-NEXT:    paddb 48(%rdx), %xmm35823; SSE42-NEXT:    paddb 32(%rdx), %xmm25824; SSE42-NEXT:    paddb (%rdx), %xmm15825; SSE42-NEXT:    movdqa %xmm1, (%rcx)5826; SSE42-NEXT:    movdqa %xmm2, 32(%rcx)5827; SSE42-NEXT:    movdqa %xmm3, 48(%rcx)5828; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)5829; SSE42-NEXT:    retq5830;5831; AVX-LABEL: vec512_v64i8_to_v16i32_factor4:5832; AVX:       # %bb.0:5833; AVX-NEXT:    vmovdqa (%rdi), %xmm05834; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm05835; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero5836; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]5837; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero5838; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]5839; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero5840; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]5841; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero5842; AVX-NEXT:    vpaddb 48(%rdx), %xmm0, %xmm05843; AVX-NEXT:    vpaddb 32(%rdx), %xmm3, %xmm35844; AVX-NEXT:    vpaddb 16(%rdx), %xmm2, %xmm25845; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm15846; AVX-NEXT:    vmovdqa %xmm1, (%rcx)5847; AVX-NEXT:    vmovdqa %xmm2, 16(%rcx)5848; AVX-NEXT:    vmovdqa %xmm3, 32(%rcx)5849; AVX-NEXT:    vmovdqa %xmm0, 48(%rcx)5850; AVX-NEXT:    retq5851;5852; AVX2-LABEL: vec512_v64i8_to_v16i32_factor4:5853; AVX2:       # %bb.0:5854; AVX2-NEXT:    vmovdqa (%rdi), %xmm05855; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm05856; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero5857; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]5858; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero5859; AVX2-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm05860; AVX2-NEXT:    vpaddb (%rdx), %ymm1, %ymm15861; AVX2-NEXT:    vmovdqa %ymm1, (%rcx)5862; AVX2-NEXT:    vmovdqa %ymm0, 32(%rcx)5863; AVX2-NEXT:    vzeroupper5864; AVX2-NEXT:    retq5865;5866; AVX512F-LABEL: vec512_v64i8_to_v16i32_factor4:5867; AVX512F:       # %bb.0:5868; AVX512F-NEXT:    vmovdqa (%rdi), %xmm05869; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm05870; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero5871; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm15872; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm15873; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm05874; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)5875; AVX512F-NEXT:    vmovdqa %ymm1, 32(%rcx)5876; AVX512F-NEXT:    vzeroupper5877; AVX512F-NEXT:    retq5878;5879; AVX512BW-LABEL: vec512_v64i8_to_v16i32_factor4:5880; AVX512BW:       # %bb.0:5881; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm05882; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm05883; AVX512BW-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero5884; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm05885; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)5886; AVX512BW-NEXT:    vzeroupper5887; AVX512BW-NEXT:    retq5888  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 645889  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 645890  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias5891  %zextd.vec = shufflevector <64 x i8> %in.vec, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 65, i32 66, i32 67, i32 1, i32 69, i32 70, i32 71, i32 2, i32 73, i32 74, i32 75, i32 3, i32 77, i32 78, i32 79, i32 4, i32 81, i32 82, i32 83, i32 5, i32 85, i32 86, i32 87, i32 6, i32 89, i32 90, i32 91, i32 7, i32 93, i32 94, i32 95, i32 8, i32 97, i32 98, i32 99, i32 9, i32 101, i32 102, i32 103, i32 10, i32 105, i32 106, i32 107, i32 11, i32 109, i32 110, i32 111, i32 12, i32 113, i32 114, i32 115, i32 13, i32 117, i32 118, i32 119, i32 14, i32 121, i32 122, i32 123, i32 15, i32 125, i32 126, i32 127>5892  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 645893  %out.vec = add <64 x i8> %zextd.vec, %out.vec.bias5894  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 645895  ret void5896}5897 5898define void @vec512_v64i8_to_v8i64_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {5899; SSE2-LABEL: vec512_v64i8_to_v8i64_factor8:5900; SSE2:       # %bb.0:5901; SSE2-NEXT:    movdqa (%rdi), %xmm05902; SSE2-NEXT:    paddb (%rsi), %xmm05903; SSE2-NEXT:    pxor %xmm1, %xmm15904; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]5905; SSE2-NEXT:    movdqa %xmm0, %xmm25906; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]5907; SSE2-NEXT:    movdqa %xmm2, %xmm35908; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]5909; SSE2-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]5910; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]5911; SSE2-NEXT:    movdqa %xmm0, %xmm45912; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]5913; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]5914; SSE2-NEXT:    paddb 16(%rdx), %xmm05915; SSE2-NEXT:    paddb (%rdx), %xmm45916; SSE2-NEXT:    paddb 48(%rdx), %xmm25917; SSE2-NEXT:    paddb 32(%rdx), %xmm35918; SSE2-NEXT:    movdqa %xmm3, 32(%rcx)5919; SSE2-NEXT:    movdqa %xmm2, 48(%rcx)5920; SSE2-NEXT:    movdqa %xmm4, (%rcx)5921; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)5922; SSE2-NEXT:    retq5923;5924; SSE42-LABEL: vec512_v64i8_to_v8i64_factor8:5925; SSE42:       # %bb.0:5926; SSE42-NEXT:    movdqa (%rdi), %xmm05927; SSE42-NEXT:    paddb (%rsi), %xmm05928; SSE42-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero5929; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]5930; SSE42-NEXT:    pmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero5931; SSE42-NEXT:    movdqa %xmm0, %xmm35932; SSE42-NEXT:    psrlq $48, %xmm35933; SSE42-NEXT:    pmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero5934; SSE42-NEXT:    psrld $16, %xmm05935; SSE42-NEXT:    pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero5936; SSE42-NEXT:    paddb 16(%rdx), %xmm05937; SSE42-NEXT:    paddb 48(%rdx), %xmm35938; SSE42-NEXT:    paddb 32(%rdx), %xmm25939; SSE42-NEXT:    paddb (%rdx), %xmm15940; SSE42-NEXT:    movdqa %xmm1, (%rcx)5941; SSE42-NEXT:    movdqa %xmm2, 32(%rcx)5942; SSE42-NEXT:    movdqa %xmm3, 48(%rcx)5943; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)5944; SSE42-NEXT:    retq5945;5946; AVX-LABEL: vec512_v64i8_to_v8i64_factor8:5947; AVX:       # %bb.0:5948; AVX-NEXT:    vmovdqa (%rdi), %xmm05949; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm05950; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero5951; AVX-NEXT:    vpsrld $16, %xmm0, %xmm25952; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero5953; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]5954; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero5955; AVX-NEXT:    vpsrlq $48, %xmm0, %xmm05956; AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero5957; AVX-NEXT:    vpaddb 48(%rdx), %xmm0, %xmm05958; AVX-NEXT:    vpaddb 32(%rdx), %xmm3, %xmm35959; AVX-NEXT:    vpaddb 16(%rdx), %xmm2, %xmm25960; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm15961; AVX-NEXT:    vmovdqa %xmm1, (%rcx)5962; AVX-NEXT:    vmovdqa %xmm2, 16(%rcx)5963; AVX-NEXT:    vmovdqa %xmm3, 32(%rcx)5964; AVX-NEXT:    vmovdqa %xmm0, 48(%rcx)5965; AVX-NEXT:    retq5966;5967; AVX2-LABEL: vec512_v64i8_to_v8i64_factor8:5968; AVX2:       # %bb.0:5969; AVX2-NEXT:    vmovdqa (%rdi), %xmm05970; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm05971; AVX2-NEXT:    vpmovzxbq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero5972; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]5973; AVX2-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero5974; AVX2-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm05975; AVX2-NEXT:    vpaddb (%rdx), %ymm1, %ymm15976; AVX2-NEXT:    vmovdqa %ymm1, (%rcx)5977; AVX2-NEXT:    vmovdqa %ymm0, 32(%rcx)5978; AVX2-NEXT:    vzeroupper5979; AVX2-NEXT:    retq5980;5981; AVX512F-LABEL: vec512_v64i8_to_v8i64_factor8:5982; AVX512F:       # %bb.0:5983; AVX512F-NEXT:    vmovdqa (%rdi), %xmm05984; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm05985; AVX512F-NEXT:    vpmovzxbq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero,xmm0[4],zero,zero,zero,zero,zero,zero,zero,xmm0[5],zero,zero,zero,zero,zero,zero,zero,xmm0[6],zero,zero,zero,zero,zero,zero,zero,xmm0[7],zero,zero,zero,zero,zero,zero,zero5986; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm15987; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm15988; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm05989; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)5990; AVX512F-NEXT:    vmovdqa %ymm1, 32(%rcx)5991; AVX512F-NEXT:    vzeroupper5992; AVX512F-NEXT:    retq5993;5994; AVX512BW-LABEL: vec512_v64i8_to_v8i64_factor8:5995; AVX512BW:       # %bb.0:5996; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm05997; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm05998; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero,xmm0[4],zero,zero,zero,zero,zero,zero,zero,xmm0[5],zero,zero,zero,zero,zero,zero,zero,xmm0[6],zero,zero,zero,zero,zero,zero,zero,xmm0[7],zero,zero,zero,zero,zero,zero,zero5999; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm06000; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)6001; AVX512BW-NEXT:    vzeroupper6002; AVX512BW-NEXT:    retq6003  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 646004  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 646005  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias6006  %zextd.vec = shufflevector <64 x i8> %in.vec, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 1, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 2, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 3, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 4, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 5, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 6, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 7, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>6007  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 646008  %out.vec = add <64 x i8> %zextd.vec, %out.vec.bias6009  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 646010  ret void6011}6012 6013define void @vec512_v64i8_to_v4i128_factor16(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {6014; SSE2-LABEL: vec512_v64i8_to_v4i128_factor16:6015; SSE2:       # %bb.0:6016; SSE2-NEXT:    movdqa (%rdi), %xmm06017; SSE2-NEXT:    paddb (%rsi), %xmm06018; SSE2-NEXT:    movd {{.*#+}} xmm1 = [255,0,0,0]6019; SSE2-NEXT:    pand %xmm0, %xmm16020; SSE2-NEXT:    movdqa %xmm0, %xmm26021; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2]6022; SSE2-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6023; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,0,0]6024; SSE2-NEXT:    psrldq {{.*#+}} xmm3 = xmm3[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6025; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1]6026; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6027; SSE2-NEXT:    paddb 16(%rdx), %xmm06028; SSE2-NEXT:    paddb 48(%rdx), %xmm36029; SSE2-NEXT:    paddb 32(%rdx), %xmm26030; SSE2-NEXT:    paddb (%rdx), %xmm16031; SSE2-NEXT:    movdqa %xmm1, (%rcx)6032; SSE2-NEXT:    movdqa %xmm2, 32(%rcx)6033; SSE2-NEXT:    movdqa %xmm3, 48(%rcx)6034; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)6035; SSE2-NEXT:    retq6036;6037; SSE42-LABEL: vec512_v64i8_to_v4i128_factor16:6038; SSE42:       # %bb.0:6039; SSE42-NEXT:    movdqa (%rdi), %xmm06040; SSE42-NEXT:    paddb (%rsi), %xmm06041; SSE42-NEXT:    pmovzxbq {{.*#+}} xmm1 = [255,0]6042; SSE42-NEXT:    pand %xmm0, %xmm16043; SSE42-NEXT:    movdqa %xmm0, %xmm26044; SSE42-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1,2]6045; SSE42-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6046; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,0,0]6047; SSE42-NEXT:    psrldq {{.*#+}} xmm3 = xmm3[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6048; SSE42-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1]6049; SSE42-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6050; SSE42-NEXT:    paddb 16(%rdx), %xmm06051; SSE42-NEXT:    paddb 48(%rdx), %xmm36052; SSE42-NEXT:    paddb 32(%rdx), %xmm26053; SSE42-NEXT:    paddb (%rdx), %xmm16054; SSE42-NEXT:    movdqa %xmm1, (%rcx)6055; SSE42-NEXT:    movdqa %xmm2, 32(%rcx)6056; SSE42-NEXT:    movdqa %xmm3, 48(%rcx)6057; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)6058; SSE42-NEXT:    retq6059;6060; AVX-LABEL: vec512_v64i8_to_v4i128_factor16:6061; AVX:       # %bb.0:6062; AVX-NEXT:    vmovdqa (%rdi), %xmm06063; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm06064; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm16065; AVX-NEXT:    vpslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1]6066; AVX-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm2[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6067; AVX-NEXT:    vpslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2]6068; AVX-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm3[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6069; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]6070; AVX-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6071; AVX-NEXT:    vpaddb 48(%rdx), %xmm0, %xmm06072; AVX-NEXT:    vpaddb 32(%rdx), %xmm3, %xmm36073; AVX-NEXT:    vpaddb 16(%rdx), %xmm2, %xmm26074; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm16075; AVX-NEXT:    vmovdqa %xmm1, (%rcx)6076; AVX-NEXT:    vmovdqa %xmm2, 16(%rcx)6077; AVX-NEXT:    vmovdqa %xmm3, 32(%rcx)6078; AVX-NEXT:    vmovdqa %xmm0, 48(%rcx)6079; AVX-NEXT:    retq6080;6081; AVX2-SLOW-LABEL: vec512_v64i8_to_v4i128_factor16:6082; AVX2-SLOW:       # %bb.0:6083; AVX2-SLOW-NEXT:    vmovdqa (%rdi), %xmm06084; AVX2-SLOW-NEXT:    vpaddb (%rsi), %xmm0, %xmm06085; AVX2-SLOW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero6086; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]6087; AVX2-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]6088; AVX2-SLOW-NEXT:    # ymm2 = mem[0,1,0,1]6089; AVX2-SLOW-NEXT:    vpand %ymm2, %ymm1, %ymm16090; AVX2-SLOW-NEXT:    vpsrld $16, %xmm0, %xmm06091; AVX2-SLOW-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero6092; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]6093; AVX2-SLOW-NEXT:    vpand %ymm2, %ymm0, %ymm06094; AVX2-SLOW-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm06095; AVX2-SLOW-NEXT:    vpaddb (%rdx), %ymm1, %ymm16096; AVX2-SLOW-NEXT:    vmovdqa %ymm1, (%rcx)6097; AVX2-SLOW-NEXT:    vmovdqa %ymm0, 32(%rcx)6098; AVX2-SLOW-NEXT:    vzeroupper6099; AVX2-SLOW-NEXT:    retq6100;6101; AVX2-FAST-PERLANE-LABEL: vec512_v64i8_to_v4i128_factor16:6102; AVX2-FAST-PERLANE:       # %bb.0:6103; AVX2-FAST-PERLANE-NEXT:    vmovdqa (%rdi), %xmm06104; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rsi), %xmm0, %xmm06105; AVX2-FAST-PERLANE-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero6106; AVX2-FAST-PERLANE-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]6107; AVX2-FAST-PERLANE-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]6108; AVX2-FAST-PERLANE-NEXT:    # ymm2 = mem[0,1,0,1]6109; AVX2-FAST-PERLANE-NEXT:    vpand %ymm2, %ymm1, %ymm16110; AVX2-FAST-PERLANE-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,u,u,u,u,u,u,u,3,u,u,u,u,u,u,u]6111; AVX2-FAST-PERLANE-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]6112; AVX2-FAST-PERLANE-NEXT:    vpand %ymm2, %ymm0, %ymm06113; AVX2-FAST-PERLANE-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm06114; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rdx), %ymm1, %ymm16115; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm1, (%rcx)6116; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm0, 32(%rcx)6117; AVX2-FAST-PERLANE-NEXT:    vzeroupper6118; AVX2-FAST-PERLANE-NEXT:    retq6119;6120; AVX2-FAST-LABEL: vec512_v64i8_to_v4i128_factor16:6121; AVX2-FAST:       # %bb.0:6122; AVX2-FAST-NEXT:    vmovdqa (%rdi), %xmm06123; AVX2-FAST-NEXT:    vpaddb (%rsi), %xmm0, %xmm06124; AVX2-FAST-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero6125; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]6126; AVX2-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]6127; AVX2-FAST-NEXT:    # ymm2 = mem[0,1,0,1]6128; AVX2-FAST-NEXT:    vpand %ymm2, %ymm1, %ymm16129; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,u,u,u,u,u,u,u,3,u,u,u,u,u,u,u]6130; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]6131; AVX2-FAST-NEXT:    vpand %ymm2, %ymm0, %ymm06132; AVX2-FAST-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm06133; AVX2-FAST-NEXT:    vpaddb (%rdx), %ymm1, %ymm16134; AVX2-FAST-NEXT:    vmovdqa %ymm1, (%rcx)6135; AVX2-FAST-NEXT:    vmovdqa %ymm0, 32(%rcx)6136; AVX2-FAST-NEXT:    vzeroupper6137; AVX2-FAST-NEXT:    retq6138;6139; AVX512F-SLOW-LABEL: vec512_v64i8_to_v4i128_factor16:6140; AVX512F-SLOW:       # %bb.0:6141; AVX512F-SLOW-NEXT:    vmovdqa (%rdi), %xmm06142; AVX512F-SLOW-NEXT:    vpaddb (%rsi), %xmm0, %xmm06143; AVX512F-SLOW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero6144; AVX512F-SLOW-NEXT:    vpsrld $16, %xmm0, %xmm06145; AVX512F-SLOW-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero6146; AVX512F-SLOW-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm06147; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[0,1,1,3,4,5,5,7]6148; AVX512F-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]6149; AVX512F-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]6150; AVX512F-SLOW-NEXT:    vpandq %zmm1, %zmm0, %zmm06151; AVX512F-SLOW-NEXT:    vextracti64x4 $1, %zmm0, %ymm16152; AVX512F-SLOW-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm16153; AVX512F-SLOW-NEXT:    vpaddb (%rdx), %ymm0, %ymm06154; AVX512F-SLOW-NEXT:    vmovdqa %ymm0, (%rcx)6155; AVX512F-SLOW-NEXT:    vmovdqa %ymm1, 32(%rcx)6156; AVX512F-SLOW-NEXT:    vzeroupper6157; AVX512F-SLOW-NEXT:    retq6158;6159; AVX512F-FAST-LABEL: vec512_v64i8_to_v4i128_factor16:6160; AVX512F-FAST:       # %bb.0:6161; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %xmm06162; AVX512F-FAST-NEXT:    vpaddb (%rsi), %xmm0, %xmm06163; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[2,u,u,u,u,u,u,u,3,u,u,u,u,u,u,u]6164; AVX512F-FAST-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero6165; AVX512F-FAST-NEXT:    vpmovsxbq {{.*#+}} zmm2 = [0,0,1,0,8,0,9,0]6166; AVX512F-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm26167; AVX512F-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]6168; AVX512F-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]6169; AVX512F-FAST-NEXT:    vpandq %zmm0, %zmm2, %zmm06170; AVX512F-FAST-NEXT:    vextracti64x4 $1, %zmm0, %ymm16171; AVX512F-FAST-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm16172; AVX512F-FAST-NEXT:    vpaddb (%rdx), %ymm0, %ymm06173; AVX512F-FAST-NEXT:    vmovdqa %ymm0, (%rcx)6174; AVX512F-FAST-NEXT:    vmovdqa %ymm1, 32(%rcx)6175; AVX512F-FAST-NEXT:    vzeroupper6176; AVX512F-FAST-NEXT:    retq6177;6178; AVX512BW-SLOW-LABEL: vec512_v64i8_to_v4i128_factor16:6179; AVX512BW-SLOW:       # %bb.0:6180; AVX512BW-SLOW-NEXT:    vmovdqa (%rdi), %xmm06181; AVX512BW-SLOW-NEXT:    vpaddb (%rsi), %xmm0, %xmm06182; AVX512BW-SLOW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero6183; AVX512BW-SLOW-NEXT:    vpsrld $16, %xmm0, %xmm06184; AVX512BW-SLOW-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero6185; AVX512BW-SLOW-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm06186; AVX512BW-SLOW-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[0,1,1,3,4,5,5,7]6187; AVX512BW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]6188; AVX512BW-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]6189; AVX512BW-SLOW-NEXT:    vpandq %zmm1, %zmm0, %zmm06190; AVX512BW-SLOW-NEXT:    vpaddb (%rdx), %zmm0, %zmm06191; AVX512BW-SLOW-NEXT:    vmovdqa64 %zmm0, (%rcx)6192; AVX512BW-SLOW-NEXT:    vzeroupper6193; AVX512BW-SLOW-NEXT:    retq6194;6195; AVX512BW-FAST-LABEL: vec512_v64i8_to_v4i128_factor16:6196; AVX512BW-FAST:       # %bb.0:6197; AVX512BW-FAST-NEXT:    vmovdqa (%rdi), %xmm06198; AVX512BW-FAST-NEXT:    vpaddb (%rsi), %xmm0, %xmm06199; AVX512BW-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[2,u,u,u,u,u,u,u,3,u,u,u,u,u,u,u]6200; AVX512BW-FAST-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero6201; AVX512BW-FAST-NEXT:    vpmovsxbq {{.*#+}} zmm2 = [0,0,1,0,8,0,9,0]6202; AVX512BW-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm26203; AVX512BW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]6204; AVX512BW-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]6205; AVX512BW-FAST-NEXT:    vpandq %zmm0, %zmm2, %zmm06206; AVX512BW-FAST-NEXT:    vpaddb (%rdx), %zmm0, %zmm06207; AVX512BW-FAST-NEXT:    vmovdqa64 %zmm0, (%rcx)6208; AVX512BW-FAST-NEXT:    vzeroupper6209; AVX512BW-FAST-NEXT:    retq6210  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 646211  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 646212  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias6213  %zextd.vec = shufflevector <64 x i8> %in.vec, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 1, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 2, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 3, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>6214  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 646215  %out.vec = add <64 x i8> %zextd.vec, %out.vec.bias6216  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 646217  ret void6218}6219 6220define void @vec512_v64i8_to_v2i256_factor32(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {6221; SSE2-LABEL: vec512_v64i8_to_v2i256_factor32:6222; SSE2:       # %bb.0:6223; SSE2-NEXT:    movdqa (%rdi), %xmm06224; SSE2-NEXT:    paddb (%rsi), %xmm06225; SSE2-NEXT:    movd {{.*#+}} xmm1 = [255,0,0,0]6226; SSE2-NEXT:    pand %xmm0, %xmm16227; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1]6228; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6229; SSE2-NEXT:    movaps 16(%rdx), %xmm26230; SSE2-NEXT:    movaps 48(%rdx), %xmm36231; SSE2-NEXT:    paddb 32(%rdx), %xmm06232; SSE2-NEXT:    paddb (%rdx), %xmm16233; SSE2-NEXT:    movaps %xmm3, 48(%rcx)6234; SSE2-NEXT:    movaps %xmm2, 16(%rcx)6235; SSE2-NEXT:    movdqa %xmm1, (%rcx)6236; SSE2-NEXT:    movdqa %xmm0, 32(%rcx)6237; SSE2-NEXT:    retq6238;6239; SSE42-LABEL: vec512_v64i8_to_v2i256_factor32:6240; SSE42:       # %bb.0:6241; SSE42-NEXT:    movdqa (%rdi), %xmm06242; SSE42-NEXT:    paddb (%rsi), %xmm06243; SSE42-NEXT:    pmovzxbq {{.*#+}} xmm1 = [255,0]6244; SSE42-NEXT:    pand %xmm0, %xmm16245; SSE42-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1]6246; SSE42-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6247; SSE42-NEXT:    movaps 16(%rdx), %xmm26248; SSE42-NEXT:    movaps 48(%rdx), %xmm36249; SSE42-NEXT:    paddb 32(%rdx), %xmm06250; SSE42-NEXT:    paddb (%rdx), %xmm16251; SSE42-NEXT:    movaps %xmm3, 48(%rcx)6252; SSE42-NEXT:    movaps %xmm2, 16(%rcx)6253; SSE42-NEXT:    movdqa %xmm1, (%rcx)6254; SSE42-NEXT:    movdqa %xmm0, 32(%rcx)6255; SSE42-NEXT:    retq6256;6257; AVX-LABEL: vec512_v64i8_to_v2i256_factor32:6258; AVX:       # %bb.0:6259; AVX-NEXT:    vmovdqa (%rdi), %xmm06260; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm06261; AVX-NEXT:    vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1]6262; AVX-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6263; AVX-NEXT:    vpaddb 32(%rdx), %xmm1, %xmm16264; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm06265; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm06266; AVX-NEXT:    vmovaps 16(%rdx), %xmm26267; AVX-NEXT:    vmovaps 48(%rdx), %xmm36268; AVX-NEXT:    vmovaps %xmm2, 16(%rcx)6269; AVX-NEXT:    vmovaps %xmm3, 48(%rcx)6270; AVX-NEXT:    vmovdqa %xmm0, (%rcx)6271; AVX-NEXT:    vmovdqa %xmm1, 32(%rcx)6272; AVX-NEXT:    retq6273;6274; AVX2-LABEL: vec512_v64i8_to_v2i256_factor32:6275; AVX2:       # %bb.0:6276; AVX2-NEXT:    vmovdqa (%rdi), %ymm06277; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm06278; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = [255,0]6279; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm16280; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[1],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6281; AVX2-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm06282; AVX2-NEXT:    vpaddb (%rdx), %ymm1, %ymm16283; AVX2-NEXT:    vmovdqa %ymm1, (%rcx)6284; AVX2-NEXT:    vmovdqa %ymm0, 32(%rcx)6285; AVX2-NEXT:    vzeroupper6286; AVX2-NEXT:    retq6287;6288; AVX512F-LABEL: vec512_v64i8_to_v2i256_factor32:6289; AVX512F:       # %bb.0:6290; AVX512F-NEXT:    vmovdqa (%rdi), %ymm06291; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm06292; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm1 = [255,0]6293; AVX512F-NEXT:    vpand %ymm1, %ymm0, %ymm16294; AVX512F-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[1],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6295; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm06296; AVX512F-NEXT:    vpaddb (%rdx), %ymm1, %ymm16297; AVX512F-NEXT:    vmovdqa %ymm1, (%rcx)6298; AVX512F-NEXT:    vmovdqa %ymm0, 32(%rcx)6299; AVX512F-NEXT:    vzeroupper6300; AVX512F-NEXT:    retq6301;6302; AVX512BW-LABEL: vec512_v64i8_to_v2i256_factor32:6303; AVX512BW:       # %bb.0:6304; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm06305; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm06306; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = [255,0]6307; AVX512BW-NEXT:    vpand %ymm1, %ymm0, %ymm16308; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[1],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6309; AVX512BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm06310; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm06311; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)6312; AVX512BW-NEXT:    vzeroupper6313; AVX512BW-NEXT:    retq6314  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 646315  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 646316  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias6317  %zextd.vec = shufflevector <64 x i8> %in.vec, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 1, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>6318  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 646319  %out.vec = add <64 x i8> %zextd.vec, %out.vec.bias6320  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 646321  ret void6322}6323 6324define void @vec512_v64i8_to_v1i512_factor64(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {6325; SSE-LABEL: vec512_v64i8_to_v1i512_factor64:6326; SSE:       # %bb.0:6327; SSE-NEXT:    movdqa (%rdi), %xmm06328; SSE-NEXT:    paddb (%rsi), %xmm06329; SSE-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm06330; SSE-NEXT:    movaps 16(%rdx), %xmm16331; SSE-NEXT:    movaps 32(%rdx), %xmm26332; SSE-NEXT:    movaps 48(%rdx), %xmm36333; SSE-NEXT:    paddb (%rdx), %xmm06334; SSE-NEXT:    movaps %xmm2, 32(%rcx)6335; SSE-NEXT:    movaps %xmm3, 48(%rcx)6336; SSE-NEXT:    movaps %xmm1, 16(%rcx)6337; SSE-NEXT:    movdqa %xmm0, (%rcx)6338; SSE-NEXT:    retq6339;6340; AVX-LABEL: vec512_v64i8_to_v1i512_factor64:6341; AVX:       # %bb.0:6342; AVX-NEXT:    vmovdqa (%rdi), %xmm06343; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm06344; AVX-NEXT:    vmovaps 32(%rdx), %ymm16345; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm06346; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm06347; AVX-NEXT:    vmovaps 16(%rdx), %xmm26348; AVX-NEXT:    vmovaps %xmm2, 16(%rcx)6349; AVX-NEXT:    vmovaps %ymm1, 32(%rcx)6350; AVX-NEXT:    vmovdqa %xmm0, (%rcx)6351; AVX-NEXT:    vzeroupper6352; AVX-NEXT:    retq6353;6354; AVX2-LABEL: vec512_v64i8_to_v1i512_factor64:6355; AVX2:       # %bb.0:6356; AVX2-NEXT:    vmovdqa (%rdi), %ymm06357; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm06358; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = [255,0]6359; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm06360; AVX2-NEXT:    vmovaps 32(%rdx), %ymm16361; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm06362; AVX2-NEXT:    vmovaps %ymm1, 32(%rcx)6363; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)6364; AVX2-NEXT:    vzeroupper6365; AVX2-NEXT:    retq6366;6367; AVX512F-LABEL: vec512_v64i8_to_v1i512_factor64:6368; AVX512F:       # %bb.0:6369; AVX512F-NEXT:    vmovdqa (%rdi), %ymm06370; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm06371; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm1 = [255,0]6372; AVX512F-NEXT:    vpand %ymm1, %ymm0, %ymm06373; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm06374; AVX512F-NEXT:    vmovaps 32(%rdx), %ymm16375; AVX512F-NEXT:    vmovaps %ymm1, 32(%rcx)6376; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)6377; AVX512F-NEXT:    vzeroupper6378; AVX512F-NEXT:    retq6379;6380; AVX512BW-LABEL: vec512_v64i8_to_v1i512_factor64:6381; AVX512BW:       # %bb.0:6382; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm06383; AVX512BW-NEXT:    vpaddb (%rsi), %zmm0, %zmm06384; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm1 = [255,0]6385; AVX512BW-NEXT:    vpandq %zmm1, %zmm0, %zmm06386; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm06387; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)6388; AVX512BW-NEXT:    vzeroupper6389; AVX512BW-NEXT:    retq6390  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 646391  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 646392  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias6393  %zextd.vec = shufflevector <64 x i8> %in.vec, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>6394  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 646395  %out.vec = add <64 x i8> %zextd.vec, %out.vec.bias6396  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 646397  ret void6398}6399 6400define void @vec512_v32i16_to_v16i32_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {6401; SSE2-LABEL: vec512_v32i16_to_v16i32_factor2:6402; SSE2:       # %bb.0:6403; SSE2-NEXT:    movdqa (%rdi), %xmm06404; SSE2-NEXT:    movdqa 16(%rdi), %xmm16405; SSE2-NEXT:    paddb (%rsi), %xmm06406; SSE2-NEXT:    paddb 16(%rsi), %xmm16407; SSE2-NEXT:    pxor %xmm2, %xmm26408; SSE2-NEXT:    movdqa %xmm1, %xmm36409; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]6410; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]6411; SSE2-NEXT:    movdqa %xmm0, %xmm46412; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]6413; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]6414; SSE2-NEXT:    paddb 16(%rdx), %xmm06415; SSE2-NEXT:    paddb (%rdx), %xmm46416; SSE2-NEXT:    paddb 48(%rdx), %xmm16417; SSE2-NEXT:    paddb 32(%rdx), %xmm36418; SSE2-NEXT:    movdqa %xmm3, 32(%rcx)6419; SSE2-NEXT:    movdqa %xmm1, 48(%rcx)6420; SSE2-NEXT:    movdqa %xmm4, (%rcx)6421; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)6422; SSE2-NEXT:    retq6423;6424; SSE42-LABEL: vec512_v32i16_to_v16i32_factor2:6425; SSE42:       # %bb.0:6426; SSE42-NEXT:    movdqa (%rdi), %xmm06427; SSE42-NEXT:    movdqa 16(%rdi), %xmm16428; SSE42-NEXT:    paddb (%rsi), %xmm06429; SSE42-NEXT:    paddb 16(%rsi), %xmm16430; SSE42-NEXT:    pxor %xmm2, %xmm26431; SSE42-NEXT:    pmovzxwd {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero6432; SSE42-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]6433; SSE42-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero6434; SSE42-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]6435; SSE42-NEXT:    paddb 16(%rdx), %xmm06436; SSE42-NEXT:    paddb (%rdx), %xmm46437; SSE42-NEXT:    paddb 48(%rdx), %xmm16438; SSE42-NEXT:    paddb 32(%rdx), %xmm36439; SSE42-NEXT:    movdqa %xmm3, 32(%rcx)6440; SSE42-NEXT:    movdqa %xmm1, 48(%rcx)6441; SSE42-NEXT:    movdqa %xmm4, (%rcx)6442; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)6443; SSE42-NEXT:    retq6444;6445; AVX-LABEL: vec512_v32i16_to_v16i32_factor2:6446; AVX:       # %bb.0:6447; AVX-NEXT:    vmovdqa (%rdi), %xmm06448; AVX-NEXT:    vmovdqa 16(%rdi), %xmm16449; AVX-NEXT:    vpaddb 16(%rsi), %xmm1, %xmm16450; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm06451; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero6452; AVX-NEXT:    vpxor %xmm3, %xmm3, %xmm36453; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]6454; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero6455; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]6456; AVX-NEXT:    vpaddb 48(%rdx), %xmm1, %xmm16457; AVX-NEXT:    vpaddb 32(%rdx), %xmm4, %xmm36458; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm06459; AVX-NEXT:    vpaddb (%rdx), %xmm2, %xmm26460; AVX-NEXT:    vmovdqa %xmm2, (%rcx)6461; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)6462; AVX-NEXT:    vmovdqa %xmm3, 32(%rcx)6463; AVX-NEXT:    vmovdqa %xmm1, 48(%rcx)6464; AVX-NEXT:    retq6465;6466; AVX2-LABEL: vec512_v32i16_to_v16i32_factor2:6467; AVX2:       # %bb.0:6468; AVX2-NEXT:    vmovdqa (%rdi), %ymm06469; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm06470; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero6471; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm06472; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero6473; AVX2-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm06474; AVX2-NEXT:    vpaddb (%rdx), %ymm1, %ymm16475; AVX2-NEXT:    vmovdqa %ymm1, (%rcx)6476; AVX2-NEXT:    vmovdqa %ymm0, 32(%rcx)6477; AVX2-NEXT:    vzeroupper6478; AVX2-NEXT:    retq6479;6480; AVX512F-LABEL: vec512_v32i16_to_v16i32_factor2:6481; AVX512F:       # %bb.0:6482; AVX512F-NEXT:    vmovdqa (%rdi), %ymm06483; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm06484; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero6485; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm16486; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm16487; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm06488; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)6489; AVX512F-NEXT:    vmovdqa %ymm1, 32(%rcx)6490; AVX512F-NEXT:    vzeroupper6491; AVX512F-NEXT:    retq6492;6493; AVX512BW-LABEL: vec512_v32i16_to_v16i32_factor2:6494; AVX512BW:       # %bb.0:6495; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm06496; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm06497; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero6498; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm06499; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)6500; AVX512BW-NEXT:    vzeroupper6501; AVX512BW-NEXT:    retq6502  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 646503  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 646504  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias6505  %in.vec.cast = bitcast <64 x i8> %in.vec to <32 x i16>6506  %zextd.vec = shufflevector <32 x i16> %in.vec.cast, <32 x i16> zeroinitializer, <32 x i32> <i32 0, i32 33, i32 1, i32 35, i32 2, i32 37, i32 3, i32 39, i32 4, i32 41, i32 5, i32 43, i32 6, i32 45, i32 7, i32 47, i32 8, i32 49, i32 9, i32 51, i32 10, i32 53, i32 11, i32 55, i32 12, i32 57, i32 13, i32 59, i32 14, i32 61, i32 15, i32 63>6507  %out.bytevec = bitcast <32 x i16> %zextd.vec to <64 x i8>6508  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 646509  %out.vec = add <64 x i8> %out.bytevec, %out.vec.bias6510  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 646511  ret void6512}6513 6514define void @vec512_v32i16_to_v8i64_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {6515; SSE2-LABEL: vec512_v32i16_to_v8i64_factor4:6516; SSE2:       # %bb.0:6517; SSE2-NEXT:    movdqa (%rdi), %xmm06518; SSE2-NEXT:    paddb (%rsi), %xmm06519; SSE2-NEXT:    pxor %xmm1, %xmm16520; SSE2-NEXT:    movdqa %xmm0, %xmm26521; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]6522; SSE2-NEXT:    movdqa %xmm2, %xmm36523; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]6524; SSE2-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]6525; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]6526; SSE2-NEXT:    movdqa %xmm0, %xmm46527; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]6528; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]6529; SSE2-NEXT:    paddb 16(%rdx), %xmm06530; SSE2-NEXT:    paddb (%rdx), %xmm46531; SSE2-NEXT:    paddb 48(%rdx), %xmm26532; SSE2-NEXT:    paddb 32(%rdx), %xmm36533; SSE2-NEXT:    movdqa %xmm3, 32(%rcx)6534; SSE2-NEXT:    movdqa %xmm2, 48(%rcx)6535; SSE2-NEXT:    movdqa %xmm4, (%rcx)6536; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)6537; SSE2-NEXT:    retq6538;6539; SSE42-LABEL: vec512_v32i16_to_v8i64_factor4:6540; SSE42:       # %bb.0:6541; SSE42-NEXT:    movdqa (%rdi), %xmm06542; SSE42-NEXT:    paddb (%rsi), %xmm06543; SSE42-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero6544; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]6545; SSE42-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero6546; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]6547; SSE42-NEXT:    pmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero6548; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]6549; SSE42-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero6550; SSE42-NEXT:    paddb 16(%rdx), %xmm06551; SSE42-NEXT:    paddb 48(%rdx), %xmm36552; SSE42-NEXT:    paddb 32(%rdx), %xmm26553; SSE42-NEXT:    paddb (%rdx), %xmm16554; SSE42-NEXT:    movdqa %xmm1, (%rcx)6555; SSE42-NEXT:    movdqa %xmm2, 32(%rcx)6556; SSE42-NEXT:    movdqa %xmm3, 48(%rcx)6557; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)6558; SSE42-NEXT:    retq6559;6560; AVX-LABEL: vec512_v32i16_to_v8i64_factor4:6561; AVX:       # %bb.0:6562; AVX-NEXT:    vmovdqa (%rdi), %xmm06563; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm06564; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero6565; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]6566; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero6567; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]6568; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero6569; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]6570; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero6571; AVX-NEXT:    vpaddb 48(%rdx), %xmm0, %xmm06572; AVX-NEXT:    vpaddb 32(%rdx), %xmm3, %xmm36573; AVX-NEXT:    vpaddb 16(%rdx), %xmm2, %xmm26574; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm16575; AVX-NEXT:    vmovdqa %xmm1, (%rcx)6576; AVX-NEXT:    vmovdqa %xmm2, 16(%rcx)6577; AVX-NEXT:    vmovdqa %xmm3, 32(%rcx)6578; AVX-NEXT:    vmovdqa %xmm0, 48(%rcx)6579; AVX-NEXT:    retq6580;6581; AVX2-LABEL: vec512_v32i16_to_v8i64_factor4:6582; AVX2:       # %bb.0:6583; AVX2-NEXT:    vmovdqa (%rdi), %ymm06584; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm06585; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero6586; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]6587; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero6588; AVX2-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm06589; AVX2-NEXT:    vpaddb (%rdx), %ymm1, %ymm16590; AVX2-NEXT:    vmovdqa %ymm1, (%rcx)6591; AVX2-NEXT:    vmovdqa %ymm0, 32(%rcx)6592; AVX2-NEXT:    vzeroupper6593; AVX2-NEXT:    retq6594;6595; AVX512F-LABEL: vec512_v32i16_to_v8i64_factor4:6596; AVX512F:       # %bb.0:6597; AVX512F-NEXT:    vmovdqa (%rdi), %xmm06598; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm06599; AVX512F-NEXT:    vpmovzxwq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero6600; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm16601; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm16602; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm06603; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)6604; AVX512F-NEXT:    vmovdqa %ymm1, 32(%rcx)6605; AVX512F-NEXT:    vzeroupper6606; AVX512F-NEXT:    retq6607;6608; AVX512BW-LABEL: vec512_v32i16_to_v8i64_factor4:6609; AVX512BW:       # %bb.0:6610; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm06611; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm06612; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero6613; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm06614; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)6615; AVX512BW-NEXT:    vzeroupper6616; AVX512BW-NEXT:    retq6617  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 646618  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 646619  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias6620  %in.vec.cast = bitcast <64 x i8> %in.vec to <32 x i16>6621  %zextd.vec = shufflevector <32 x i16> %in.vec.cast, <32 x i16> zeroinitializer, <32 x i32> <i32 0, i32 33, i32 34, i32 35, i32 1, i32 37, i32 38, i32 39, i32 2, i32 41, i32 42, i32 43, i32 3, i32 45, i32 46, i32 47, i32 4, i32 49, i32 50, i32 51, i32 5, i32 53, i32 54, i32 55, i32 6, i32 57, i32 58, i32 59, i32 7, i32 61, i32 62, i32 63>6622  %out.bytevec = bitcast <32 x i16> %zextd.vec to <64 x i8>6623  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 646624  %out.vec = add <64 x i8> %out.bytevec, %out.vec.bias6625  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 646626  ret void6627}6628 6629define void @vec512_v32i16_to_v4i128_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {6630; SSE2-LABEL: vec512_v32i16_to_v4i128_factor8:6631; SSE2:       # %bb.0:6632; SSE2-NEXT:    movdqa (%rdi), %xmm06633; SSE2-NEXT:    paddb (%rsi), %xmm06634; SSE2-NEXT:    movd {{.*#+}} xmm1 = [65535,0,0,0]6635; SSE2-NEXT:    pand %xmm0, %xmm16636; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,1,0,1]6637; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,0,0]6638; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]6639; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6640; SSE2-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6641; SSE2-NEXT:    psrldq {{.*#+}} xmm3 = xmm3[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6642; SSE2-NEXT:    paddb 16(%rdx), %xmm36643; SSE2-NEXT:    paddb 48(%rdx), %xmm26644; SSE2-NEXT:    paddb 32(%rdx), %xmm06645; SSE2-NEXT:    paddb (%rdx), %xmm16646; SSE2-NEXT:    movdqa %xmm1, (%rcx)6647; SSE2-NEXT:    movdqa %xmm0, 32(%rcx)6648; SSE2-NEXT:    movdqa %xmm2, 48(%rcx)6649; SSE2-NEXT:    movdqa %xmm3, 16(%rcx)6650; SSE2-NEXT:    retq6651;6652; SSE42-LABEL: vec512_v32i16_to_v4i128_factor8:6653; SSE42:       # %bb.0:6654; SSE42-NEXT:    movdqa (%rdi), %xmm06655; SSE42-NEXT:    paddb (%rsi), %xmm06656; SSE42-NEXT:    pxor %xmm1, %xmm16657; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]6658; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,1,0,1]6659; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,0,0]6660; SSE42-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]6661; SSE42-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6662; SSE42-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6663; SSE42-NEXT:    psrldq {{.*#+}} xmm3 = xmm3[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6664; SSE42-NEXT:    paddb 16(%rdx), %xmm36665; SSE42-NEXT:    paddb 48(%rdx), %xmm26666; SSE42-NEXT:    paddb 32(%rdx), %xmm06667; SSE42-NEXT:    paddb (%rdx), %xmm16668; SSE42-NEXT:    movdqa %xmm1, (%rcx)6669; SSE42-NEXT:    movdqa %xmm0, 32(%rcx)6670; SSE42-NEXT:    movdqa %xmm2, 48(%rcx)6671; SSE42-NEXT:    movdqa %xmm3, 16(%rcx)6672; SSE42-NEXT:    retq6673;6674; AVX-LABEL: vec512_v32i16_to_v4i128_factor8:6675; AVX:       # %bb.0:6676; AVX-NEXT:    vmovdqa (%rdi), %xmm06677; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm06678; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm16679; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]6680; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[0,0,0,0]6681; AVX-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6682; AVX-NEXT:    vpslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]6683; AVX-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm3[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6684; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]6685; AVX-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6686; AVX-NEXT:    vpaddb 48(%rdx), %xmm0, %xmm06687; AVX-NEXT:    vpaddb 32(%rdx), %xmm3, %xmm36688; AVX-NEXT:    vpaddb 16(%rdx), %xmm2, %xmm26689; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm16690; AVX-NEXT:    vmovdqa %xmm1, (%rcx)6691; AVX-NEXT:    vmovdqa %xmm2, 16(%rcx)6692; AVX-NEXT:    vmovdqa %xmm3, 32(%rcx)6693; AVX-NEXT:    vmovdqa %xmm0, 48(%rcx)6694; AVX-NEXT:    retq6695;6696; AVX2-SLOW-LABEL: vec512_v32i16_to_v4i128_factor8:6697; AVX2-SLOW:       # %bb.0:6698; AVX2-SLOW-NEXT:    vpxor %xmm0, %xmm0, %xmm06699; AVX2-SLOW-NEXT:    vmovdqa (%rdi), %xmm16700; AVX2-SLOW-NEXT:    vpaddb (%rsi), %xmm1, %xmm16701; AVX2-SLOW-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero6702; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,1,3]6703; AVX2-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm0[1,2,3,4,5,6,7],ymm2[8],ymm0[9,10,11,12,13,14,15]6704; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]6705; AVX2-SLOW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero6706; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]6707; AVX2-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15]6708; AVX2-SLOW-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm06709; AVX2-SLOW-NEXT:    vpaddb (%rdx), %ymm2, %ymm16710; AVX2-SLOW-NEXT:    vmovdqa %ymm1, (%rcx)6711; AVX2-SLOW-NEXT:    vmovdqa %ymm0, 32(%rcx)6712; AVX2-SLOW-NEXT:    vzeroupper6713; AVX2-SLOW-NEXT:    retq6714;6715; AVX2-FAST-PERLANE-LABEL: vec512_v32i16_to_v4i128_factor8:6716; AVX2-FAST-PERLANE:       # %bb.0:6717; AVX2-FAST-PERLANE-NEXT:    vpxor %xmm0, %xmm0, %xmm06718; AVX2-FAST-PERLANE-NEXT:    vmovdqa (%rdi), %xmm16719; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rsi), %xmm1, %xmm16720; AVX2-FAST-PERLANE-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero6721; AVX2-FAST-PERLANE-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,1,3]6722; AVX2-FAST-PERLANE-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm0[1,2,3,4,5,6,7],ymm2[8],ymm0[9,10,11,12,13,14,15]6723; AVX2-FAST-PERLANE-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[4,5,u,u,u,u,u,u,6,7,u,u,u,u,u,u]6724; AVX2-FAST-PERLANE-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]6725; AVX2-FAST-PERLANE-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15]6726; AVX2-FAST-PERLANE-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm06727; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rdx), %ymm2, %ymm16728; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm1, (%rcx)6729; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm0, 32(%rcx)6730; AVX2-FAST-PERLANE-NEXT:    vzeroupper6731; AVX2-FAST-PERLANE-NEXT:    retq6732;6733; AVX2-FAST-LABEL: vec512_v32i16_to_v4i128_factor8:6734; AVX2-FAST:       # %bb.0:6735; AVX2-FAST-NEXT:    vpxor %xmm0, %xmm0, %xmm06736; AVX2-FAST-NEXT:    vmovdqa (%rdi), %xmm16737; AVX2-FAST-NEXT:    vpaddb (%rsi), %xmm1, %xmm16738; AVX2-FAST-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero6739; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,1,3]6740; AVX2-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm0[1,2,3,4,5,6,7],ymm2[8],ymm0[9,10,11,12,13,14,15]6741; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[4,5,u,u,u,u,u,u,6,7,u,u,u,u,u,u]6742; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]6743; AVX2-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15]6744; AVX2-FAST-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm06745; AVX2-FAST-NEXT:    vpaddb (%rdx), %ymm2, %ymm16746; AVX2-FAST-NEXT:    vmovdqa %ymm1, (%rcx)6747; AVX2-FAST-NEXT:    vmovdqa %ymm0, 32(%rcx)6748; AVX2-FAST-NEXT:    vzeroupper6749; AVX2-FAST-NEXT:    retq6750;6751; AVX512F-SLOW-LABEL: vec512_v32i16_to_v4i128_factor8:6752; AVX512F-SLOW:       # %bb.0:6753; AVX512F-SLOW-NEXT:    vmovdqa (%rdi), %xmm06754; AVX512F-SLOW-NEXT:    vpaddb (%rsi), %xmm0, %xmm06755; AVX512F-SLOW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero6756; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]6757; AVX512F-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm26758; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3,4,5,6,7],ymm1[8],ymm2[9,10,11,12,13,14,15]6759; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]6760; AVX512F-SLOW-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero6761; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]6762; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3,4,5,6,7],ymm0[8],ymm2[9,10,11,12,13,14,15]6763; AVX512F-SLOW-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm06764; AVX512F-SLOW-NEXT:    vpaddb (%rdx), %ymm1, %ymm16765; AVX512F-SLOW-NEXT:    vmovdqa %ymm1, (%rcx)6766; AVX512F-SLOW-NEXT:    vmovdqa %ymm0, 32(%rcx)6767; AVX512F-SLOW-NEXT:    vzeroupper6768; AVX512F-SLOW-NEXT:    retq6769;6770; AVX512F-FAST-LABEL: vec512_v32i16_to_v4i128_factor8:6771; AVX512F-FAST:       # %bb.0:6772; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %xmm06773; AVX512F-FAST-NEXT:    vpaddb (%rsi), %xmm0, %xmm06774; AVX512F-FAST-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero6775; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]6776; AVX512F-FAST-NEXT:    vpxor %xmm2, %xmm2, %xmm26777; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3,4,5,6,7],ymm1[8],ymm2[9,10,11,12,13,14,15]6778; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,u,u,u,u,u,u,6,7,u,u,u,u,u,u]6779; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]6780; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3,4,5,6,7],ymm0[8],ymm2[9,10,11,12,13,14,15]6781; AVX512F-FAST-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm06782; AVX512F-FAST-NEXT:    vpaddb (%rdx), %ymm1, %ymm16783; AVX512F-FAST-NEXT:    vmovdqa %ymm1, (%rcx)6784; AVX512F-FAST-NEXT:    vmovdqa %ymm0, 32(%rcx)6785; AVX512F-FAST-NEXT:    vzeroupper6786; AVX512F-FAST-NEXT:    retq6787;6788; AVX512BW-LABEL: vec512_v32i16_to_v4i128_factor8:6789; AVX512BW:       # %bb.0:6790; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm06791; AVX512BW-NEXT:    vpaddb (%rsi), %zmm0, %zmm06792; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [32,1,2,3,4,5,6,7,33,9,10,11,12,13,14,15,34,1,2,3,4,5,6,7,35,9,10,11,12,13,14,15]6793; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm26794; AVX512BW-NEXT:    vpermt2w %zmm0, %zmm1, %zmm26795; AVX512BW-NEXT:    vpaddb (%rdx), %zmm2, %zmm06796; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)6797; AVX512BW-NEXT:    vzeroupper6798; AVX512BW-NEXT:    retq6799  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 646800  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 646801  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias6802  %in.vec.cast = bitcast <64 x i8> %in.vec to <32 x i16>6803  %zextd.vec = shufflevector <32 x i16> %in.vec.cast, <32 x i16> zeroinitializer, <32 x i32> <i32 0, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 1, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 2, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 3, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>6804  %out.bytevec = bitcast <32 x i16> %zextd.vec to <64 x i8>6805  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 646806  %out.vec = add <64 x i8> %out.bytevec, %out.vec.bias6807  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 646808  ret void6809}6810 6811define void @vec512_v32i16_to_v2i256_factor16(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {6812; SSE2-LABEL: vec512_v32i16_to_v2i256_factor16:6813; SSE2:       # %bb.0:6814; SSE2-NEXT:    movdqa (%rdi), %xmm06815; SSE2-NEXT:    paddb (%rsi), %xmm06816; SSE2-NEXT:    movd {{.*#+}} xmm1 = [65535,0,0,0]6817; SSE2-NEXT:    pand %xmm0, %xmm16818; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]6819; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6820; SSE2-NEXT:    movaps 16(%rdx), %xmm26821; SSE2-NEXT:    movaps 48(%rdx), %xmm36822; SSE2-NEXT:    paddb 32(%rdx), %xmm06823; SSE2-NEXT:    paddb (%rdx), %xmm16824; SSE2-NEXT:    movaps %xmm3, 48(%rcx)6825; SSE2-NEXT:    movaps %xmm2, 16(%rcx)6826; SSE2-NEXT:    movdqa %xmm1, (%rcx)6827; SSE2-NEXT:    movdqa %xmm0, 32(%rcx)6828; SSE2-NEXT:    retq6829;6830; SSE42-LABEL: vec512_v32i16_to_v2i256_factor16:6831; SSE42:       # %bb.0:6832; SSE42-NEXT:    movdqa (%rdi), %xmm06833; SSE42-NEXT:    paddb (%rsi), %xmm06834; SSE42-NEXT:    pxor %xmm1, %xmm16835; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]6836; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]6837; SSE42-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6838; SSE42-NEXT:    movaps 16(%rdx), %xmm26839; SSE42-NEXT:    movaps 48(%rdx), %xmm36840; SSE42-NEXT:    paddb 32(%rdx), %xmm06841; SSE42-NEXT:    paddb (%rdx), %xmm16842; SSE42-NEXT:    movaps %xmm3, 48(%rcx)6843; SSE42-NEXT:    movaps %xmm2, 16(%rcx)6844; SSE42-NEXT:    movdqa %xmm1, (%rcx)6845; SSE42-NEXT:    movdqa %xmm0, 32(%rcx)6846; SSE42-NEXT:    retq6847;6848; AVX-LABEL: vec512_v32i16_to_v2i256_factor16:6849; AVX:       # %bb.0:6850; AVX-NEXT:    vmovdqa (%rdi), %xmm06851; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm06852; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]6853; AVX-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6854; AVX-NEXT:    vpaddb 32(%rdx), %xmm1, %xmm16855; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm26856; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7]6857; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm06858; AVX-NEXT:    vmovaps 16(%rdx), %xmm26859; AVX-NEXT:    vmovaps 48(%rdx), %xmm36860; AVX-NEXT:    vmovaps %xmm2, 16(%rcx)6861; AVX-NEXT:    vmovaps %xmm3, 48(%rcx)6862; AVX-NEXT:    vmovdqa %xmm0, (%rcx)6863; AVX-NEXT:    vmovdqa %xmm1, 32(%rcx)6864; AVX-NEXT:    retq6865;6866; AVX2-LABEL: vec512_v32i16_to_v2i256_factor16:6867; AVX2:       # %bb.0:6868; AVX2-NEXT:    vmovdqa (%rdi), %ymm06869; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm06870; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = [65535,0,0,0]6871; AVX2-NEXT:    vpand %ymm0, %ymm1, %ymm16872; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[2,3],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6873; AVX2-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm06874; AVX2-NEXT:    vpaddb (%rdx), %ymm1, %ymm16875; AVX2-NEXT:    vmovdqa %ymm1, (%rcx)6876; AVX2-NEXT:    vmovdqa %ymm0, 32(%rcx)6877; AVX2-NEXT:    vzeroupper6878; AVX2-NEXT:    retq6879;6880; AVX512F-LABEL: vec512_v32i16_to_v2i256_factor16:6881; AVX512F:       # %bb.0:6882; AVX512F-NEXT:    vmovdqa (%rdi), %ymm06883; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm06884; AVX512F-NEXT:    vmovd {{.*#+}} xmm1 = [65535,0,0,0]6885; AVX512F-NEXT:    vpand %ymm0, %ymm1, %ymm16886; AVX512F-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[2,3],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6887; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm06888; AVX512F-NEXT:    vpaddb (%rdx), %ymm1, %ymm16889; AVX512F-NEXT:    vmovdqa %ymm1, (%rcx)6890; AVX512F-NEXT:    vmovdqa %ymm0, 32(%rcx)6891; AVX512F-NEXT:    vzeroupper6892; AVX512F-NEXT:    retq6893;6894; AVX512BW-LABEL: vec512_v32i16_to_v2i256_factor16:6895; AVX512BW:       # %bb.0:6896; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm06897; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm06898; AVX512BW-NEXT:    vmovd {{.*#+}} xmm1 = [65535,0,0,0]6899; AVX512BW-NEXT:    vpand %ymm0, %ymm1, %ymm16900; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[2,3],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero6901; AVX512BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm06902; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm06903; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)6904; AVX512BW-NEXT:    vzeroupper6905; AVX512BW-NEXT:    retq6906  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 646907  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 646908  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias6909  %in.vec.cast = bitcast <64 x i8> %in.vec to <32 x i16>6910  %zextd.vec = shufflevector <32 x i16> %in.vec.cast, <32 x i16> zeroinitializer, <32 x i32> <i32 0, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 1, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>6911  %out.bytevec = bitcast <32 x i16> %zextd.vec to <64 x i8>6912  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 646913  %out.vec = add <64 x i8> %out.bytevec, %out.vec.bias6914  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 646915  ret void6916}6917 6918define void @vec512_v32i16_to_v1i512_factor32(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {6919; SSE2-LABEL: vec512_v32i16_to_v1i512_factor32:6920; SSE2:       # %bb.0:6921; SSE2-NEXT:    movdqa (%rdi), %xmm06922; SSE2-NEXT:    paddb (%rsi), %xmm06923; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm06924; SSE2-NEXT:    movaps 16(%rdx), %xmm16925; SSE2-NEXT:    movaps 32(%rdx), %xmm26926; SSE2-NEXT:    movaps 48(%rdx), %xmm36927; SSE2-NEXT:    paddb (%rdx), %xmm06928; SSE2-NEXT:    movaps %xmm2, 32(%rcx)6929; SSE2-NEXT:    movaps %xmm3, 48(%rcx)6930; SSE2-NEXT:    movaps %xmm1, 16(%rcx)6931; SSE2-NEXT:    movdqa %xmm0, (%rcx)6932; SSE2-NEXT:    retq6933;6934; SSE42-LABEL: vec512_v32i16_to_v1i512_factor32:6935; SSE42:       # %bb.0:6936; SSE42-NEXT:    movdqa (%rdi), %xmm06937; SSE42-NEXT:    paddb (%rsi), %xmm06938; SSE42-NEXT:    pxor %xmm1, %xmm16939; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]6940; SSE42-NEXT:    movaps 16(%rdx), %xmm06941; SSE42-NEXT:    movaps 32(%rdx), %xmm26942; SSE42-NEXT:    movaps 48(%rdx), %xmm36943; SSE42-NEXT:    paddb (%rdx), %xmm16944; SSE42-NEXT:    movaps %xmm2, 32(%rcx)6945; SSE42-NEXT:    movaps %xmm3, 48(%rcx)6946; SSE42-NEXT:    movaps %xmm0, 16(%rcx)6947; SSE42-NEXT:    movdqa %xmm1, (%rcx)6948; SSE42-NEXT:    retq6949;6950; AVX-LABEL: vec512_v32i16_to_v1i512_factor32:6951; AVX:       # %bb.0:6952; AVX-NEXT:    vmovdqa (%rdi), %xmm06953; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm06954; AVX-NEXT:    vmovaps 32(%rdx), %ymm16955; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm26956; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7]6957; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm06958; AVX-NEXT:    vmovaps 16(%rdx), %xmm26959; AVX-NEXT:    vmovaps %xmm2, 16(%rcx)6960; AVX-NEXT:    vmovaps %ymm1, 32(%rcx)6961; AVX-NEXT:    vmovdqa %xmm0, (%rcx)6962; AVX-NEXT:    vzeroupper6963; AVX-NEXT:    retq6964;6965; AVX2-LABEL: vec512_v32i16_to_v1i512_factor32:6966; AVX2:       # %bb.0:6967; AVX2-NEXT:    vmovdqa (%rdi), %ymm06968; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm06969; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = [65535,0,0,0]6970; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm06971; AVX2-NEXT:    vmovaps 32(%rdx), %ymm16972; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm06973; AVX2-NEXT:    vmovaps %ymm1, 32(%rcx)6974; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)6975; AVX2-NEXT:    vzeroupper6976; AVX2-NEXT:    retq6977;6978; AVX512F-LABEL: vec512_v32i16_to_v1i512_factor32:6979; AVX512F:       # %bb.0:6980; AVX512F-NEXT:    vmovdqa (%rdi), %ymm06981; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm06982; AVX512F-NEXT:    vmovd {{.*#+}} xmm1 = [65535,0,0,0]6983; AVX512F-NEXT:    vpand %ymm1, %ymm0, %ymm06984; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm06985; AVX512F-NEXT:    vmovaps 32(%rdx), %ymm16986; AVX512F-NEXT:    vmovaps %ymm1, 32(%rcx)6987; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)6988; AVX512F-NEXT:    vzeroupper6989; AVX512F-NEXT:    retq6990;6991; AVX512BW-LABEL: vec512_v32i16_to_v1i512_factor32:6992; AVX512BW:       # %bb.0:6993; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm06994; AVX512BW-NEXT:    vpaddb (%rsi), %zmm0, %zmm06995; AVX512BW-NEXT:    vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm06996; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm06997; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)6998; AVX512BW-NEXT:    vzeroupper6999; AVX512BW-NEXT:    retq7000  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 647001  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 647002  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias7003  %in.vec.cast = bitcast <64 x i8> %in.vec to <32 x i16>7004  %zextd.vec = shufflevector <32 x i16> %in.vec.cast, <32 x i16> zeroinitializer, <32 x i32> <i32 0, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>7005  %out.bytevec = bitcast <32 x i16> %zextd.vec to <64 x i8>7006  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 647007  %out.vec = add <64 x i8> %out.bytevec, %out.vec.bias7008  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 647009  ret void7010}7011 7012define void @vec512_v16i32_to_v8i64_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {7013; SSE2-LABEL: vec512_v16i32_to_v8i64_factor2:7014; SSE2:       # %bb.0:7015; SSE2-NEXT:    movdqa (%rdi), %xmm07016; SSE2-NEXT:    movdqa 16(%rdi), %xmm17017; SSE2-NEXT:    paddb (%rsi), %xmm07018; SSE2-NEXT:    paddb 16(%rsi), %xmm17019; SSE2-NEXT:    pxor %xmm2, %xmm27020; SSE2-NEXT:    movdqa %xmm1, %xmm37021; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]7022; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]7023; SSE2-NEXT:    movdqa %xmm0, %xmm47024; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]7025; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]7026; SSE2-NEXT:    paddb 16(%rdx), %xmm07027; SSE2-NEXT:    paddb (%rdx), %xmm47028; SSE2-NEXT:    paddb 48(%rdx), %xmm17029; SSE2-NEXT:    paddb 32(%rdx), %xmm37030; SSE2-NEXT:    movdqa %xmm3, 32(%rcx)7031; SSE2-NEXT:    movdqa %xmm1, 48(%rcx)7032; SSE2-NEXT:    movdqa %xmm4, (%rcx)7033; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)7034; SSE2-NEXT:    retq7035;7036; SSE42-LABEL: vec512_v16i32_to_v8i64_factor2:7037; SSE42:       # %bb.0:7038; SSE42-NEXT:    movdqa (%rdi), %xmm07039; SSE42-NEXT:    movdqa 16(%rdi), %xmm17040; SSE42-NEXT:    paddb (%rsi), %xmm07041; SSE42-NEXT:    paddb 16(%rsi), %xmm17042; SSE42-NEXT:    pxor %xmm2, %xmm27043; SSE42-NEXT:    pmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero7044; SSE42-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]7045; SSE42-NEXT:    pmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero7046; SSE42-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]7047; SSE42-NEXT:    paddb 16(%rdx), %xmm07048; SSE42-NEXT:    paddb (%rdx), %xmm47049; SSE42-NEXT:    paddb 48(%rdx), %xmm17050; SSE42-NEXT:    paddb 32(%rdx), %xmm37051; SSE42-NEXT:    movdqa %xmm3, 32(%rcx)7052; SSE42-NEXT:    movdqa %xmm1, 48(%rcx)7053; SSE42-NEXT:    movdqa %xmm4, (%rcx)7054; SSE42-NEXT:    movdqa %xmm0, 16(%rcx)7055; SSE42-NEXT:    retq7056;7057; AVX-LABEL: vec512_v16i32_to_v8i64_factor2:7058; AVX:       # %bb.0:7059; AVX-NEXT:    vmovdqa (%rdi), %xmm07060; AVX-NEXT:    vmovdqa 16(%rdi), %xmm17061; AVX-NEXT:    vpaddb 16(%rsi), %xmm1, %xmm17062; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm07063; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero7064; AVX-NEXT:    vpxor %xmm3, %xmm3, %xmm37065; AVX-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3]7066; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero7067; AVX-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]7068; AVX-NEXT:    vpaddb 48(%rdx), %xmm1, %xmm17069; AVX-NEXT:    vpaddb 32(%rdx), %xmm4, %xmm37070; AVX-NEXT:    vpaddb 16(%rdx), %xmm0, %xmm07071; AVX-NEXT:    vpaddb (%rdx), %xmm2, %xmm27072; AVX-NEXT:    vmovdqa %xmm2, (%rcx)7073; AVX-NEXT:    vmovdqa %xmm0, 16(%rcx)7074; AVX-NEXT:    vmovdqa %xmm3, 32(%rcx)7075; AVX-NEXT:    vmovdqa %xmm1, 48(%rcx)7076; AVX-NEXT:    retq7077;7078; AVX2-LABEL: vec512_v16i32_to_v8i64_factor2:7079; AVX2:       # %bb.0:7080; AVX2-NEXT:    vmovdqa (%rdi), %ymm07081; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm07082; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero7083; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm07084; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero7085; AVX2-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm07086; AVX2-NEXT:    vpaddb (%rdx), %ymm1, %ymm17087; AVX2-NEXT:    vmovdqa %ymm1, (%rcx)7088; AVX2-NEXT:    vmovdqa %ymm0, 32(%rcx)7089; AVX2-NEXT:    vzeroupper7090; AVX2-NEXT:    retq7091;7092; AVX512F-LABEL: vec512_v16i32_to_v8i64_factor2:7093; AVX512F:       # %bb.0:7094; AVX512F-NEXT:    vmovdqa (%rdi), %ymm07095; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm07096; AVX512F-NEXT:    vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero7097; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm17098; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm17099; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm07100; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)7101; AVX512F-NEXT:    vmovdqa %ymm1, 32(%rcx)7102; AVX512F-NEXT:    vzeroupper7103; AVX512F-NEXT:    retq7104;7105; AVX512BW-LABEL: vec512_v16i32_to_v8i64_factor2:7106; AVX512BW:       # %bb.0:7107; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm07108; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm07109; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero7110; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm07111; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)7112; AVX512BW-NEXT:    vzeroupper7113; AVX512BW-NEXT:    retq7114  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 647115  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 647116  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias7117  %in.vec.cast = bitcast <64 x i8> %in.vec to <16 x i32>7118  %zextd.vec = shufflevector <16 x i32> %in.vec.cast, <16 x i32> zeroinitializer, <16 x i32> <i32 0, i32 17, i32 1, i32 19, i32 2, i32 21, i32 3, i32 23, i32 4, i32 25, i32 5, i32 27, i32 6, i32 29, i32 7, i32 31>7119  %out.bytevec = bitcast <16 x i32> %zextd.vec to <64 x i8>7120  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 647121  %out.vec = add <64 x i8> %out.bytevec, %out.vec.bias7122  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 647123  ret void7124}7125 7126define void @vec512_v16i32_to_v4i128_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {7127; SSE2-LABEL: vec512_v16i32_to_v4i128_factor4:7128; SSE2:       # %bb.0:7129; SSE2-NEXT:    movdqa (%rdi), %xmm07130; SSE2-NEXT:    paddb (%rsi), %xmm07131; SSE2-NEXT:    xorps %xmm1, %xmm17132; SSE2-NEXT:    movdqa %xmm0, %xmm27133; SSE2-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero7134; SSE2-NEXT:    xorps %xmm3, %xmm37135; SSE2-NEXT:    movss {{.*#+}} xmm3 = xmm0[0],xmm3[1,2,3]7136; SSE2-NEXT:    movdqa %xmm0, %xmm47137; SSE2-NEXT:    psrldq {{.*#+}} xmm4 = xmm4[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero7138; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,2],xmm1[2,3]7139; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,0],xmm1[1,0]7140; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[2,3]7141; SSE2-NEXT:    paddb 16(%rdx), %xmm07142; SSE2-NEXT:    paddb 32(%rdx), %xmm47143; SSE2-NEXT:    paddb (%rdx), %xmm37144; SSE2-NEXT:    paddb 48(%rdx), %xmm27145; SSE2-NEXT:    movdqa %xmm2, 48(%rcx)7146; SSE2-NEXT:    movdqa %xmm3, (%rcx)7147; SSE2-NEXT:    movdqa %xmm4, 32(%rcx)7148; SSE2-NEXT:    movdqa %xmm0, 16(%rcx)7149; SSE2-NEXT:    retq7150;7151; SSE42-LABEL: vec512_v16i32_to_v4i128_factor4:7152; SSE42:       # %bb.0:7153; SSE42-NEXT:    movdqa (%rdi), %xmm07154; SSE42-NEXT:    paddb (%rsi), %xmm07155; SSE42-NEXT:    pxor %xmm1, %xmm17156; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7]7157; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]7158; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]7159; SSE42-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero7160; SSE42-NEXT:    pxor %xmm4, %xmm47161; SSE42-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3,4,5,6,7]7162; SSE42-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3,4,5,6,7]7163; SSE42-NEXT:    paddb 16(%rdx), %xmm37164; SSE42-NEXT:    paddb 32(%rdx), %xmm27165; SSE42-NEXT:    paddb (%rdx), %xmm17166; SSE42-NEXT:    paddb 48(%rdx), %xmm07167; SSE42-NEXT:    movdqa %xmm0, 48(%rcx)7168; SSE42-NEXT:    movdqa %xmm1, (%rcx)7169; SSE42-NEXT:    movdqa %xmm2, 32(%rcx)7170; SSE42-NEXT:    movdqa %xmm3, 16(%rcx)7171; SSE42-NEXT:    retq7172;7173; AVX-LABEL: vec512_v16i32_to_v4i128_factor4:7174; AVX:       # %bb.0:7175; AVX-NEXT:    vmovdqa (%rdi), %xmm07176; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm07177; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm17178; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7]7179; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = xmm0[1],zero,zero,zero7180; AVX-NEXT:    vinsertps {{.*#+}} xmm3 = xmm0[2],zero,zero,zero7181; AVX-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero7182; AVX-NEXT:    vpaddb 48(%rdx), %xmm0, %xmm07183; AVX-NEXT:    vpaddb 32(%rdx), %xmm3, %xmm37184; AVX-NEXT:    vpaddb 16(%rdx), %xmm2, %xmm27185; AVX-NEXT:    vpaddb (%rdx), %xmm1, %xmm17186; AVX-NEXT:    vmovdqa %xmm1, (%rcx)7187; AVX-NEXT:    vmovdqa %xmm2, 16(%rcx)7188; AVX-NEXT:    vmovdqa %xmm3, 32(%rcx)7189; AVX-NEXT:    vmovdqa %xmm0, 48(%rcx)7190; AVX-NEXT:    retq7191;7192; AVX2-SLOW-LABEL: vec512_v16i32_to_v4i128_factor4:7193; AVX2-SLOW:       # %bb.0:7194; AVX2-SLOW-NEXT:    vpxor %xmm0, %xmm0, %xmm07195; AVX2-SLOW-NEXT:    vmovdqa (%rdi), %xmm17196; AVX2-SLOW-NEXT:    vpaddb (%rsi), %xmm1, %xmm17197; AVX2-SLOW-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero7198; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,1,3]7199; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0],ymm0[1,2,3],ymm2[4],ymm0[5,6,7]7200; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]7201; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]7202; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3],ymm1[4],ymm0[5,6,7]7203; AVX2-SLOW-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm07204; AVX2-SLOW-NEXT:    vpaddb (%rdx), %ymm2, %ymm17205; AVX2-SLOW-NEXT:    vmovdqa %ymm1, (%rcx)7206; AVX2-SLOW-NEXT:    vmovdqa %ymm0, 32(%rcx)7207; AVX2-SLOW-NEXT:    vzeroupper7208; AVX2-SLOW-NEXT:    retq7209;7210; AVX2-FAST-PERLANE-LABEL: vec512_v16i32_to_v4i128_factor4:7211; AVX2-FAST-PERLANE:       # %bb.0:7212; AVX2-FAST-PERLANE-NEXT:    vpxor %xmm0, %xmm0, %xmm07213; AVX2-FAST-PERLANE-NEXT:    vmovdqa (%rdi), %xmm17214; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rsi), %xmm1, %xmm17215; AVX2-FAST-PERLANE-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero7216; AVX2-FAST-PERLANE-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,1,3]7217; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0],ymm0[1,2,3],ymm2[4],ymm0[5,6,7]7218; AVX2-FAST-PERLANE-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]7219; AVX2-FAST-PERLANE-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]7220; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3],ymm1[4],ymm0[5,6,7]7221; AVX2-FAST-PERLANE-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm07222; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rdx), %ymm2, %ymm17223; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm1, (%rcx)7224; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm0, 32(%rcx)7225; AVX2-FAST-PERLANE-NEXT:    vzeroupper7226; AVX2-FAST-PERLANE-NEXT:    retq7227;7228; AVX2-FAST-LABEL: vec512_v16i32_to_v4i128_factor4:7229; AVX2-FAST:       # %bb.0:7230; AVX2-FAST-NEXT:    vmovdqa (%rdi), %ymm07231; AVX2-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm07232; AVX2-FAST-NEXT:    vpxor %xmm1, %xmm1, %xmm17233; AVX2-FAST-NEXT:    vpmovsxbq {{.*#+}} ymm2 = [0,0,1,0]7234; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm2, %ymm27235; AVX2-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0],ymm1[1,2,3],ymm2[4],ymm1[5,6,7]7236; AVX2-FAST-NEXT:    vpmovsxbq {{.*#+}} ymm3 = [2,0,3,0]7237; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm3, %ymm07238; AVX2-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]7239; AVX2-FAST-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm07240; AVX2-FAST-NEXT:    vpaddb (%rdx), %ymm2, %ymm17241; AVX2-FAST-NEXT:    vmovdqa %ymm1, (%rcx)7242; AVX2-FAST-NEXT:    vmovdqa %ymm0, 32(%rcx)7243; AVX2-FAST-NEXT:    vzeroupper7244; AVX2-FAST-NEXT:    retq7245;7246; AVX512F-LABEL: vec512_v16i32_to_v4i128_factor4:7247; AVX512F:       # %bb.0:7248; AVX512F-NEXT:    vmovdqa (%rdi), %ymm07249; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm07250; AVX512F-NEXT:    movw $4369, %ax # imm = 0x11117251; AVX512F-NEXT:    kmovw %eax, %k17252; AVX512F-NEXT:    vpexpandd %zmm0, %zmm0 {%k1} {z}7253; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm17254; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm17255; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm07256; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)7257; AVX512F-NEXT:    vmovdqa %ymm1, 32(%rcx)7258; AVX512F-NEXT:    vzeroupper7259; AVX512F-NEXT:    retq7260;7261; AVX512BW-LABEL: vec512_v16i32_to_v4i128_factor4:7262; AVX512BW:       # %bb.0:7263; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm07264; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm07265; AVX512BW-NEXT:    movb $17, %al7266; AVX512BW-NEXT:    kmovd %eax, %k17267; AVX512BW-NEXT:    vpexpandd %ymm0, %ymm1 {%k1} {z}7268; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm27269; AVX512BW-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [2,9,10,11,3,13,14,15]7270; AVX512BW-NEXT:    vpermi2d %ymm2, %ymm0, %ymm37271; AVX512BW-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm07272; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm07273; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)7274; AVX512BW-NEXT:    vzeroupper7275; AVX512BW-NEXT:    retq7276  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 647277  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 647278  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias7279  %in.vec.cast = bitcast <64 x i8> %in.vec to <16 x i32>7280  %zextd.vec = shufflevector <16 x i32> %in.vec.cast, <16 x i32> zeroinitializer, <16 x i32> <i32 0, i32 17, i32 18, i32 19, i32 1, i32 21, i32 22, i32 23, i32 2, i32 25, i32 26, i32 27, i32 3, i32 29, i32 30, i32 31>7281  %out.bytevec = bitcast <16 x i32> %zextd.vec to <64 x i8>7282  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 647283  %out.vec = add <64 x i8> %out.bytevec, %out.vec.bias7284  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 647285  ret void7286}7287 7288define void @vec512_v16i32_to_v2i256_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {7289; SSE2-LABEL: vec512_v16i32_to_v2i256_factor8:7290; SSE2:       # %bb.0:7291; SSE2-NEXT:    movdqa (%rdi), %xmm07292; SSE2-NEXT:    paddb (%rsi), %xmm07293; SSE2-NEXT:    xorps %xmm1, %xmm17294; SSE2-NEXT:    xorps %xmm2, %xmm27295; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3]7296; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,0],xmm1[1,0]7297; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[2,3]7298; SSE2-NEXT:    movaps 16(%rdx), %xmm17299; SSE2-NEXT:    movaps 48(%rdx), %xmm37300; SSE2-NEXT:    paddb 32(%rdx), %xmm07301; SSE2-NEXT:    paddb (%rdx), %xmm27302; SSE2-NEXT:    movaps %xmm3, 48(%rcx)7303; SSE2-NEXT:    movaps %xmm1, 16(%rcx)7304; SSE2-NEXT:    movdqa %xmm2, (%rcx)7305; SSE2-NEXT:    movdqa %xmm0, 32(%rcx)7306; SSE2-NEXT:    retq7307;7308; SSE42-LABEL: vec512_v16i32_to_v2i256_factor8:7309; SSE42:       # %bb.0:7310; SSE42-NEXT:    movdqa (%rdi), %xmm07311; SSE42-NEXT:    paddb (%rsi), %xmm07312; SSE42-NEXT:    pxor %xmm1, %xmm17313; SSE42-NEXT:    pxor %xmm2, %xmm27314; SSE42-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1],xmm2[2,3,4,5,6,7]7315; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]7316; SSE42-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]7317; SSE42-NEXT:    movaps 16(%rdx), %xmm17318; SSE42-NEXT:    movaps 48(%rdx), %xmm37319; SSE42-NEXT:    paddb 32(%rdx), %xmm07320; SSE42-NEXT:    paddb (%rdx), %xmm27321; SSE42-NEXT:    movaps %xmm3, 48(%rcx)7322; SSE42-NEXT:    movaps %xmm1, 16(%rcx)7323; SSE42-NEXT:    movdqa %xmm2, (%rcx)7324; SSE42-NEXT:    movdqa %xmm0, 32(%rcx)7325; SSE42-NEXT:    retq7326;7327; AVX-LABEL: vec512_v16i32_to_v2i256_factor8:7328; AVX:       # %bb.0:7329; AVX-NEXT:    vmovdqa (%rdi), %xmm07330; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm07331; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]7332; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm27333; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7]7334; AVX-NEXT:    vpaddb 32(%rdx), %xmm1, %xmm17335; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3,4,5,6,7]7336; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm07337; AVX-NEXT:    vmovaps 16(%rdx), %xmm27338; AVX-NEXT:    vmovaps 48(%rdx), %xmm37339; AVX-NEXT:    vmovaps %xmm2, 16(%rcx)7340; AVX-NEXT:    vmovaps %xmm3, 48(%rcx)7341; AVX-NEXT:    vmovdqa %xmm0, (%rcx)7342; AVX-NEXT:    vmovdqa %xmm1, 32(%rcx)7343; AVX-NEXT:    retq7344;7345; AVX2-SLOW-LABEL: vec512_v16i32_to_v2i256_factor8:7346; AVX2-SLOW:       # %bb.0:7347; AVX2-SLOW-NEXT:    vmovdqa (%rdi), %ymm07348; AVX2-SLOW-NEXT:    vpaddb (%rsi), %ymm0, %ymm07349; AVX2-SLOW-NEXT:    vpxor %xmm1, %xmm1, %xmm17350; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm0[0],xmm1[1,2,3]7351; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]7352; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]7353; AVX2-SLOW-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm07354; AVX2-SLOW-NEXT:    vpaddb (%rdx), %ymm2, %ymm17355; AVX2-SLOW-NEXT:    vmovdqa %ymm1, (%rcx)7356; AVX2-SLOW-NEXT:    vmovdqa %ymm0, 32(%rcx)7357; AVX2-SLOW-NEXT:    vzeroupper7358; AVX2-SLOW-NEXT:    retq7359;7360; AVX2-FAST-PERLANE-LABEL: vec512_v16i32_to_v2i256_factor8:7361; AVX2-FAST-PERLANE:       # %bb.0:7362; AVX2-FAST-PERLANE-NEXT:    vmovdqa (%rdi), %ymm07363; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rsi), %ymm0, %ymm07364; AVX2-FAST-PERLANE-NEXT:    vpxor %xmm1, %xmm1, %xmm17365; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]7366; AVX2-FAST-PERLANE-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,6,7],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero7367; AVX2-FAST-PERLANE-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm07368; AVX2-FAST-PERLANE-NEXT:    vpaddb (%rdx), %ymm1, %ymm17369; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm1, (%rcx)7370; AVX2-FAST-PERLANE-NEXT:    vmovdqa %ymm0, 32(%rcx)7371; AVX2-FAST-PERLANE-NEXT:    vzeroupper7372; AVX2-FAST-PERLANE-NEXT:    retq7373;7374; AVX2-FAST-LABEL: vec512_v16i32_to_v2i256_factor8:7375; AVX2-FAST:       # %bb.0:7376; AVX2-FAST-NEXT:    vmovdqa (%rdi), %ymm07377; AVX2-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm07378; AVX2-FAST-NEXT:    vpxor %xmm1, %xmm1, %xmm17379; AVX2-FAST-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]7380; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,6,7],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero7381; AVX2-FAST-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm07382; AVX2-FAST-NEXT:    vpaddb (%rdx), %ymm1, %ymm17383; AVX2-FAST-NEXT:    vmovdqa %ymm1, (%rcx)7384; AVX2-FAST-NEXT:    vmovdqa %ymm0, 32(%rcx)7385; AVX2-FAST-NEXT:    vzeroupper7386; AVX2-FAST-NEXT:    retq7387;7388; AVX512F-LABEL: vec512_v16i32_to_v2i256_factor8:7389; AVX512F:       # %bb.0:7390; AVX512F-NEXT:    vmovdqa (%rdi), %ymm07391; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm07392; AVX512F-NEXT:    movw $257, %ax # imm = 0x1017393; AVX512F-NEXT:    kmovw %eax, %k17394; AVX512F-NEXT:    vpexpandd %zmm0, %zmm0 {%k1} {z}7395; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm17396; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm17397; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm07398; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)7399; AVX512F-NEXT:    vmovdqa %ymm1, 32(%rcx)7400; AVX512F-NEXT:    vzeroupper7401; AVX512F-NEXT:    retq7402;7403; AVX512BW-SLOW-LABEL: vec512_v16i32_to_v2i256_factor8:7404; AVX512BW-SLOW:       # %bb.0:7405; AVX512BW-SLOW-NEXT:    vmovdqa (%rdi), %ymm07406; AVX512BW-SLOW-NEXT:    vpaddb (%rsi), %ymm0, %ymm07407; AVX512BW-SLOW-NEXT:    vpxor %xmm1, %xmm1, %xmm17408; AVX512BW-SLOW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm0[0],xmm1[1,2,3]7409; AVX512BW-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]7410; AVX512BW-SLOW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]7411; AVX512BW-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm07412; AVX512BW-SLOW-NEXT:    vpaddb (%rdx), %zmm0, %zmm07413; AVX512BW-SLOW-NEXT:    vmovdqa64 %zmm0, (%rcx)7414; AVX512BW-SLOW-NEXT:    vzeroupper7415; AVX512BW-SLOW-NEXT:    retq7416;7417; AVX512BW-FAST-LABEL: vec512_v16i32_to_v2i256_factor8:7418; AVX512BW-FAST:       # %bb.0:7419; AVX512BW-FAST-NEXT:    vmovdqa (%rdi), %ymm07420; AVX512BW-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm07421; AVX512BW-FAST-NEXT:    vpxor %xmm1, %xmm1, %xmm17422; AVX512BW-FAST-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]7423; AVX512BW-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,6,7],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero7424; AVX512BW-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm07425; AVX512BW-FAST-NEXT:    vpaddb (%rdx), %zmm0, %zmm07426; AVX512BW-FAST-NEXT:    vmovdqa64 %zmm0, (%rcx)7427; AVX512BW-FAST-NEXT:    vzeroupper7428; AVX512BW-FAST-NEXT:    retq7429  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 647430  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 647431  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias7432  %in.vec.cast = bitcast <64 x i8> %in.vec to <16 x i32>7433  %zextd.vec = shufflevector <16 x i32> %in.vec.cast, <16 x i32> zeroinitializer, <16 x i32> <i32 0, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 1, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>7434  %out.bytevec = bitcast <16 x i32> %zextd.vec to <64 x i8>7435  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 647436  %out.vec = add <64 x i8> %out.bytevec, %out.vec.bias7437  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 647438  ret void7439}7440 7441define void @vec512_v16i32_to_v1i512_factor16(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {7442; SSE2-LABEL: vec512_v16i32_to_v1i512_factor16:7443; SSE2:       # %bb.0:7444; SSE2-NEXT:    movdqa (%rdi), %xmm07445; SSE2-NEXT:    paddb (%rsi), %xmm07446; SSE2-NEXT:    xorps %xmm1, %xmm17447; SSE2-NEXT:    movss {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]7448; SSE2-NEXT:    movaps 16(%rdx), %xmm07449; SSE2-NEXT:    movaps 32(%rdx), %xmm27450; SSE2-NEXT:    movaps 48(%rdx), %xmm37451; SSE2-NEXT:    paddb (%rdx), %xmm17452; SSE2-NEXT:    movaps %xmm2, 32(%rcx)7453; SSE2-NEXT:    movaps %xmm3, 48(%rcx)7454; SSE2-NEXT:    movaps %xmm0, 16(%rcx)7455; SSE2-NEXT:    movdqa %xmm1, (%rcx)7456; SSE2-NEXT:    retq7457;7458; SSE42-LABEL: vec512_v16i32_to_v1i512_factor16:7459; SSE42:       # %bb.0:7460; SSE42-NEXT:    movdqa (%rdi), %xmm07461; SSE42-NEXT:    paddb (%rsi), %xmm07462; SSE42-NEXT:    pxor %xmm1, %xmm17463; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7]7464; SSE42-NEXT:    movaps 16(%rdx), %xmm07465; SSE42-NEXT:    movaps 32(%rdx), %xmm27466; SSE42-NEXT:    movaps 48(%rdx), %xmm37467; SSE42-NEXT:    paddb (%rdx), %xmm17468; SSE42-NEXT:    movaps %xmm2, 32(%rcx)7469; SSE42-NEXT:    movaps %xmm3, 48(%rcx)7470; SSE42-NEXT:    movaps %xmm0, 16(%rcx)7471; SSE42-NEXT:    movdqa %xmm1, (%rcx)7472; SSE42-NEXT:    retq7473;7474; AVX-LABEL: vec512_v16i32_to_v1i512_factor16:7475; AVX:       # %bb.0:7476; AVX-NEXT:    vmovdqa (%rdi), %xmm07477; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm07478; AVX-NEXT:    vmovaps 32(%rdx), %ymm17479; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm27480; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3,4,5,6,7]7481; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm07482; AVX-NEXT:    vmovaps 16(%rdx), %xmm27483; AVX-NEXT:    vmovaps %xmm2, 16(%rcx)7484; AVX-NEXT:    vmovaps %ymm1, 32(%rcx)7485; AVX-NEXT:    vmovdqa %xmm0, (%rcx)7486; AVX-NEXT:    vzeroupper7487; AVX-NEXT:    retq7488;7489; AVX2-LABEL: vec512_v16i32_to_v1i512_factor16:7490; AVX2:       # %bb.0:7491; AVX2-NEXT:    vmovdqa (%rdi), %ymm07492; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm07493; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm17494; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]7495; AVX2-NEXT:    vmovaps 32(%rdx), %ymm17496; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm07497; AVX2-NEXT:    vmovaps %ymm1, 32(%rcx)7498; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)7499; AVX2-NEXT:    vzeroupper7500; AVX2-NEXT:    retq7501;7502; AVX512F-LABEL: vec512_v16i32_to_v1i512_factor16:7503; AVX512F:       # %bb.0:7504; AVX512F-NEXT:    vmovdqa (%rdi), %ymm07505; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm07506; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm17507; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]7508; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm07509; AVX512F-NEXT:    vmovaps 32(%rdx), %ymm17510; AVX512F-NEXT:    vmovaps %ymm1, 32(%rcx)7511; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)7512; AVX512F-NEXT:    vzeroupper7513; AVX512F-NEXT:    retq7514;7515; AVX512BW-LABEL: vec512_v16i32_to_v1i512_factor16:7516; AVX512BW:       # %bb.0:7517; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm07518; AVX512BW-NEXT:    vpaddb (%rsi), %zmm0, %zmm07519; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm17520; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]7521; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm07522; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)7523; AVX512BW-NEXT:    vzeroupper7524; AVX512BW-NEXT:    retq7525  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 647526  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 647527  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias7528  %in.vec.cast = bitcast <64 x i8> %in.vec to <16 x i32>7529  %zextd.vec = shufflevector <16 x i32> %in.vec.cast, <16 x i32> zeroinitializer, <16 x i32> <i32 0, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>7530  %out.bytevec = bitcast <16 x i32> %zextd.vec to <64 x i8>7531  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 647532  %out.vec = add <64 x i8> %out.bytevec, %out.vec.bias7533  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 647534  ret void7535}7536 7537define void @vec512_v8i64_to_v4i128_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {7538; SSE-LABEL: vec512_v8i64_to_v4i128_factor2:7539; SSE:       # %bb.0:7540; SSE-NEXT:    movdqa (%rdi), %xmm07541; SSE-NEXT:    movdqa 16(%rdi), %xmm17542; SSE-NEXT:    paddb (%rsi), %xmm07543; SSE-NEXT:    paddb 16(%rsi), %xmm17544; SSE-NEXT:    movq {{.*#+}} xmm2 = xmm1[0],zero7545; SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero7546; SSE-NEXT:    movq {{.*#+}} xmm3 = xmm0[0],zero7547; SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero7548; SSE-NEXT:    paddb 16(%rdx), %xmm07549; SSE-NEXT:    paddb (%rdx), %xmm37550; SSE-NEXT:    paddb 48(%rdx), %xmm17551; SSE-NEXT:    paddb 32(%rdx), %xmm27552; SSE-NEXT:    movdqa %xmm2, 32(%rcx)7553; SSE-NEXT:    movdqa %xmm1, 48(%rcx)7554; SSE-NEXT:    movdqa %xmm3, (%rcx)7555; SSE-NEXT:    movdqa %xmm0, 16(%rcx)7556; SSE-NEXT:    retq7557;7558; AVX-LABEL: vec512_v8i64_to_v4i128_factor2:7559; AVX:       # %bb.0:7560; AVX-NEXT:    vmovdqa (%rdi), %xmm07561; AVX-NEXT:    vmovdqa 16(%rdi), %xmm17562; AVX-NEXT:    vpaddb 16(%rsi), %xmm1, %xmm17563; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm07564; AVX-NEXT:    vxorpd %xmm2, %xmm2, %xmm27565; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm07566; AVX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[3],ymm2[3]7567; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm17568; AVX-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[3],ymm2[3]7569; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm27570; AVX-NEXT:    vpaddb 48(%rdx), %xmm2, %xmm27571; AVX-NEXT:    vpaddb 32(%rdx), %xmm1, %xmm17572; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm37573; AVX-NEXT:    vpaddb 16(%rdx), %xmm3, %xmm37574; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm07575; AVX-NEXT:    vmovdqa %xmm0, (%rcx)7576; AVX-NEXT:    vmovdqa %xmm3, 16(%rcx)7577; AVX-NEXT:    vmovdqa %xmm1, 32(%rcx)7578; AVX-NEXT:    vmovdqa %xmm2, 48(%rcx)7579; AVX-NEXT:    vzeroupper7580; AVX-NEXT:    retq7581;7582; AVX2-LABEL: vec512_v8i64_to_v4i128_factor2:7583; AVX2:       # %bb.0:7584; AVX2-NEXT:    vmovdqa (%rdi), %ymm07585; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm07586; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm17587; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[0,1,1,3]7588; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm1[2,3],ymm2[4,5],ymm1[6,7]7589; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,3]7590; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]7591; AVX2-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm07592; AVX2-NEXT:    vpaddb (%rdx), %ymm2, %ymm17593; AVX2-NEXT:    vmovdqa %ymm1, (%rcx)7594; AVX2-NEXT:    vmovdqa %ymm0, 32(%rcx)7595; AVX2-NEXT:    vzeroupper7596; AVX2-NEXT:    retq7597;7598; AVX512F-LABEL: vec512_v8i64_to_v4i128_factor2:7599; AVX512F:       # %bb.0:7600; AVX512F-NEXT:    vmovdqa (%rdi), %ymm07601; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm07602; AVX512F-NEXT:    movb $85, %al7603; AVX512F-NEXT:    kmovw %eax, %k17604; AVX512F-NEXT:    vpexpandq %zmm0, %zmm0 {%k1} {z}7605; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm17606; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm17607; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm07608; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)7609; AVX512F-NEXT:    vmovdqa %ymm1, 32(%rcx)7610; AVX512F-NEXT:    vzeroupper7611; AVX512F-NEXT:    retq7612;7613; AVX512BW-SLOW-LABEL: vec512_v8i64_to_v4i128_factor2:7614; AVX512BW-SLOW:       # %bb.0:7615; AVX512BW-SLOW-NEXT:    vmovdqa (%rdi), %ymm07616; AVX512BW-SLOW-NEXT:    vpaddb (%rsi), %ymm0, %ymm07617; AVX512BW-SLOW-NEXT:    movb $5, %al7618; AVX512BW-SLOW-NEXT:    kmovd %eax, %k17619; AVX512BW-SLOW-NEXT:    vpexpandq %ymm0, %ymm1 {%k1} {z}7620; AVX512BW-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,3]7621; AVX512BW-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm27622; AVX512BW-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2,3],ymm0[4,5],ymm2[6,7]7623; AVX512BW-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm07624; AVX512BW-SLOW-NEXT:    vpaddb (%rdx), %zmm0, %zmm07625; AVX512BW-SLOW-NEXT:    vmovdqa64 %zmm0, (%rcx)7626; AVX512BW-SLOW-NEXT:    vzeroupper7627; AVX512BW-SLOW-NEXT:    retq7628;7629; AVX512BW-FAST-LABEL: vec512_v8i64_to_v4i128_factor2:7630; AVX512BW-FAST:       # %bb.0:7631; AVX512BW-FAST-NEXT:    vmovdqa (%rdi), %ymm07632; AVX512BW-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm07633; AVX512BW-FAST-NEXT:    movb $5, %al7634; AVX512BW-FAST-NEXT:    kmovd %eax, %k17635; AVX512BW-FAST-NEXT:    vpexpandq %ymm0, %ymm1 {%k1} {z}7636; AVX512BW-FAST-NEXT:    vpxor %xmm2, %xmm2, %xmm27637; AVX512BW-FAST-NEXT:    vpmovsxbq {{.*#+}} ymm3 = [2,5,3,7]7638; AVX512BW-FAST-NEXT:    vpermi2q %ymm2, %ymm0, %ymm37639; AVX512BW-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm07640; AVX512BW-FAST-NEXT:    vpaddb (%rdx), %zmm0, %zmm07641; AVX512BW-FAST-NEXT:    vmovdqa64 %zmm0, (%rcx)7642; AVX512BW-FAST-NEXT:    vzeroupper7643; AVX512BW-FAST-NEXT:    retq7644  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 647645  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 647646  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias7647  %in.vec.cast = bitcast <64 x i8> %in.vec to <8 x i64>7648  %zextd.vec = shufflevector <8 x i64> %in.vec.cast, <8 x i64> zeroinitializer, <8 x i32> <i32 0, i32 9, i32 1, i32 11, i32 2, i32 13, i32 3, i32 15>7649  %out.bytevec = bitcast <8 x i64> %zextd.vec to <64 x i8>7650  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 647651  %out.vec = add <64 x i8> %out.bytevec, %out.vec.bias7652  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 647653  ret void7654}7655 7656define void @vec512_v8i64_to_v2i256_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {7657; SSE-LABEL: vec512_v8i64_to_v2i256_factor4:7658; SSE:       # %bb.0:7659; SSE-NEXT:    movdqa (%rdi), %xmm07660; SSE-NEXT:    paddb (%rsi), %xmm07661; SSE-NEXT:    movq {{.*#+}} xmm1 = xmm0[0],zero7662; SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero7663; SSE-NEXT:    movaps 16(%rdx), %xmm27664; SSE-NEXT:    movaps 48(%rdx), %xmm37665; SSE-NEXT:    paddb (%rdx), %xmm17666; SSE-NEXT:    paddb 32(%rdx), %xmm07667; SSE-NEXT:    movaps %xmm3, 48(%rcx)7668; SSE-NEXT:    movaps %xmm2, 16(%rcx)7669; SSE-NEXT:    movdqa %xmm0, 32(%rcx)7670; SSE-NEXT:    movdqa %xmm1, (%rcx)7671; SSE-NEXT:    retq7672;7673; AVX-LABEL: vec512_v8i64_to_v2i256_factor4:7674; AVX:       # %bb.0:7675; AVX-NEXT:    vmovdqa (%rdi), %xmm07676; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm07677; AVX-NEXT:    vpsrldq {{.*#+}} xmm1 = xmm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero7678; AVX-NEXT:    vpaddb 32(%rdx), %xmm1, %xmm17679; AVX-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero7680; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm07681; AVX-NEXT:    vmovaps 16(%rdx), %xmm27682; AVX-NEXT:    vmovaps 48(%rdx), %xmm37683; AVX-NEXT:    vmovaps %xmm2, 16(%rcx)7684; AVX-NEXT:    vmovaps %xmm3, 48(%rcx)7685; AVX-NEXT:    vmovdqa %xmm0, (%rcx)7686; AVX-NEXT:    vmovdqa %xmm1, 32(%rcx)7687; AVX-NEXT:    retq7688;7689; AVX2-LABEL: vec512_v8i64_to_v2i256_factor4:7690; AVX2:       # %bb.0:7691; AVX2-NEXT:    vmovdqa (%rdi), %ymm07692; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm07693; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = xmm0[0],zero7694; AVX2-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero7695; AVX2-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm07696; AVX2-NEXT:    vpaddb (%rdx), %ymm1, %ymm17697; AVX2-NEXT:    vmovdqa %ymm1, (%rcx)7698; AVX2-NEXT:    vmovdqa %ymm0, 32(%rcx)7699; AVX2-NEXT:    vzeroupper7700; AVX2-NEXT:    retq7701;7702; AVX512F-LABEL: vec512_v8i64_to_v2i256_factor4:7703; AVX512F:       # %bb.0:7704; AVX512F-NEXT:    vmovdqa (%rdi), %ymm07705; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm07706; AVX512F-NEXT:    movb $17, %al7707; AVX512F-NEXT:    kmovw %eax, %k17708; AVX512F-NEXT:    vpexpandq %zmm0, %zmm0 {%k1} {z}7709; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm17710; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm17711; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm07712; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)7713; AVX512F-NEXT:    vmovdqa %ymm1, 32(%rcx)7714; AVX512F-NEXT:    vzeroupper7715; AVX512F-NEXT:    retq7716;7717; AVX512BW-LABEL: vec512_v8i64_to_v2i256_factor4:7718; AVX512BW:       # %bb.0:7719; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm07720; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm07721; AVX512BW-NEXT:    vmovq {{.*#+}} xmm1 = xmm0[0],zero7722; AVX512BW-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero7723; AVX512BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm07724; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm07725; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)7726; AVX512BW-NEXT:    vzeroupper7727; AVX512BW-NEXT:    retq7728  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 647729  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 647730  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias7731  %in.vec.cast = bitcast <64 x i8> %in.vec to <8 x i64>7732  %zextd.vec = shufflevector <8 x i64> %in.vec.cast, <8 x i64> zeroinitializer, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 1, i32 13, i32 14, i32 15>7733  %out.bytevec = bitcast <8 x i64> %zextd.vec to <64 x i8>7734  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 647735  %out.vec = add <64 x i8> %out.bytevec, %out.vec.bias7736  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 647737  ret void7738}7739 7740define void @vec512_v8i64_to_v1i512_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {7741; SSE-LABEL: vec512_v8i64_to_v1i512_factor8:7742; SSE:       # %bb.0:7743; SSE-NEXT:    movdqa (%rdi), %xmm07744; SSE-NEXT:    paddb (%rsi), %xmm07745; SSE-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero7746; SSE-NEXT:    movaps 16(%rdx), %xmm17747; SSE-NEXT:    movaps 32(%rdx), %xmm27748; SSE-NEXT:    movaps 48(%rdx), %xmm37749; SSE-NEXT:    paddb (%rdx), %xmm07750; SSE-NEXT:    movaps %xmm2, 32(%rcx)7751; SSE-NEXT:    movaps %xmm3, 48(%rcx)7752; SSE-NEXT:    movaps %xmm1, 16(%rcx)7753; SSE-NEXT:    movdqa %xmm0, (%rcx)7754; SSE-NEXT:    retq7755;7756; AVX-LABEL: vec512_v8i64_to_v1i512_factor8:7757; AVX:       # %bb.0:7758; AVX-NEXT:    vmovdqa (%rdi), %xmm07759; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm07760; AVX-NEXT:    vmovaps 32(%rdx), %ymm17761; AVX-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero7762; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm07763; AVX-NEXT:    vmovaps 16(%rdx), %xmm27764; AVX-NEXT:    vmovaps %xmm2, 16(%rcx)7765; AVX-NEXT:    vmovaps %ymm1, 32(%rcx)7766; AVX-NEXT:    vmovdqa %xmm0, (%rcx)7767; AVX-NEXT:    vzeroupper7768; AVX-NEXT:    retq7769;7770; AVX2-LABEL: vec512_v8i64_to_v1i512_factor8:7771; AVX2:       # %bb.0:7772; AVX2-NEXT:    vmovdqa (%rdi), %ymm07773; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm07774; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero7775; AVX2-NEXT:    vmovaps 32(%rdx), %ymm17776; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm07777; AVX2-NEXT:    vmovaps %ymm1, 32(%rcx)7778; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)7779; AVX2-NEXT:    vzeroupper7780; AVX2-NEXT:    retq7781;7782; AVX512F-LABEL: vec512_v8i64_to_v1i512_factor8:7783; AVX512F:       # %bb.0:7784; AVX512F-NEXT:    vmovdqa (%rdi), %ymm07785; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm07786; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero7787; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm07788; AVX512F-NEXT:    vmovaps 32(%rdx), %ymm17789; AVX512F-NEXT:    vmovaps %ymm1, 32(%rcx)7790; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)7791; AVX512F-NEXT:    vzeroupper7792; AVX512F-NEXT:    retq7793;7794; AVX512BW-LABEL: vec512_v8i64_to_v1i512_factor8:7795; AVX512BW:       # %bb.0:7796; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm07797; AVX512BW-NEXT:    vpaddb (%rsi), %zmm0, %zmm07798; AVX512BW-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero7799; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm07800; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)7801; AVX512BW-NEXT:    vzeroupper7802; AVX512BW-NEXT:    retq7803  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 647804  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 647805  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias7806  %in.vec.cast = bitcast <64 x i8> %in.vec to <8 x i64>7807  %zextd.vec = shufflevector <8 x i64> %in.vec.cast, <8 x i64> zeroinitializer, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>7808  %out.bytevec = bitcast <8 x i64> %zextd.vec to <64 x i8>7809  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 647810  %out.vec = add <64 x i8> %out.bytevec, %out.vec.bias7811  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 647812  ret void7813}7814 7815define void @vec512_v4i128_to_v2i256_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {7816; SSE-LABEL: vec512_v4i128_to_v2i256_factor2:7817; SSE:       # %bb.0:7818; SSE-NEXT:    movdqa (%rdi), %xmm07819; SSE-NEXT:    movdqa 16(%rdi), %xmm17820; SSE-NEXT:    paddb 16(%rsi), %xmm17821; SSE-NEXT:    paddb (%rsi), %xmm07822; SSE-NEXT:    movaps 16(%rdx), %xmm27823; SSE-NEXT:    movaps 48(%rdx), %xmm37824; SSE-NEXT:    paddb (%rdx), %xmm07825; SSE-NEXT:    paddb 32(%rdx), %xmm17826; SSE-NEXT:    movaps %xmm3, 48(%rcx)7827; SSE-NEXT:    movaps %xmm2, 16(%rcx)7828; SSE-NEXT:    movdqa %xmm1, 32(%rcx)7829; SSE-NEXT:    movdqa %xmm0, (%rcx)7830; SSE-NEXT:    retq7831;7832; AVX-LABEL: vec512_v4i128_to_v2i256_factor2:7833; AVX:       # %bb.0:7834; AVX-NEXT:    vmovdqa (%rdi), %xmm07835; AVX-NEXT:    vmovdqa 16(%rdi), %xmm17836; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm07837; AVX-NEXT:    vpaddb 16(%rsi), %xmm1, %xmm17838; AVX-NEXT:    vpaddb 32(%rdx), %xmm1, %xmm17839; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm07840; AVX-NEXT:    vmovaps 16(%rdx), %xmm27841; AVX-NEXT:    vmovaps 48(%rdx), %xmm37842; AVX-NEXT:    vmovaps %xmm3, 48(%rcx)7843; AVX-NEXT:    vmovaps %xmm2, 16(%rcx)7844; AVX-NEXT:    vmovdqa %xmm0, (%rcx)7845; AVX-NEXT:    vmovdqa %xmm1, 32(%rcx)7846; AVX-NEXT:    retq7847;7848; AVX2-LABEL: vec512_v4i128_to_v2i256_factor2:7849; AVX2:       # %bb.0:7850; AVX2-NEXT:    vmovdqa (%rdi), %ymm07851; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm07852; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm17853; AVX2-NEXT:    vmovdqa %xmm0, %xmm07854; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm07855; AVX2-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm17856; AVX2-NEXT:    vmovdqa %ymm1, 32(%rcx)7857; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)7858; AVX2-NEXT:    vzeroupper7859; AVX2-NEXT:    retq7860;7861; AVX512F-LABEL: vec512_v4i128_to_v2i256_factor2:7862; AVX512F:       # %bb.0:7863; AVX512F-NEXT:    vmovdqa (%rdi), %ymm07864; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm07865; AVX512F-NEXT:    movb $51, %al7866; AVX512F-NEXT:    kmovw %eax, %k17867; AVX512F-NEXT:    vpexpandq %zmm0, %zmm0 {%k1} {z}7868; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm17869; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm1, %ymm17870; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm07871; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)7872; AVX512F-NEXT:    vmovdqa %ymm1, 32(%rcx)7873; AVX512F-NEXT:    vzeroupper7874; AVX512F-NEXT:    retq7875;7876; AVX512BW-LABEL: vec512_v4i128_to_v2i256_factor2:7877; AVX512BW:       # %bb.0:7878; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm07879; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm07880; AVX512BW-NEXT:    vmovdqa %xmm0, %xmm17881; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm07882; AVX512BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm07883; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm07884; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)7885; AVX512BW-NEXT:    vzeroupper7886; AVX512BW-NEXT:    retq7887  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 647888  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 647889  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias7890  %in.vec.cast = bitcast <64 x i8> %in.vec to <4 x i128>7891  %zextd.vec = shufflevector <4 x i128> %in.vec.cast, <4 x i128> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 1, i32 7>7892  %out.bytevec = bitcast <4 x i128> %zextd.vec to <64 x i8>7893  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 647894  %out.vec = add <64 x i8> %out.bytevec, %out.vec.bias7895  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 647896  ret void7897}7898 7899define void @vec512_v4i128_to_v1i512_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {7900; SSE-LABEL: vec512_v4i128_to_v1i512_factor4:7901; SSE:       # %bb.0:7902; SSE-NEXT:    movdqa (%rdi), %xmm07903; SSE-NEXT:    paddb (%rsi), %xmm07904; SSE-NEXT:    movaps 16(%rdx), %xmm17905; SSE-NEXT:    movaps 32(%rdx), %xmm27906; SSE-NEXT:    movaps 48(%rdx), %xmm37907; SSE-NEXT:    paddb (%rdx), %xmm07908; SSE-NEXT:    movaps %xmm2, 32(%rcx)7909; SSE-NEXT:    movaps %xmm3, 48(%rcx)7910; SSE-NEXT:    movaps %xmm1, 16(%rcx)7911; SSE-NEXT:    movdqa %xmm0, (%rcx)7912; SSE-NEXT:    retq7913;7914; AVX-LABEL: vec512_v4i128_to_v1i512_factor4:7915; AVX:       # %bb.0:7916; AVX-NEXT:    vmovdqa (%rdi), %xmm07917; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm07918; AVX-NEXT:    vmovaps 32(%rdx), %ymm17919; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm07920; AVX-NEXT:    vmovaps 16(%rdx), %xmm27921; AVX-NEXT:    vmovaps %xmm2, 16(%rcx)7922; AVX-NEXT:    vmovaps %ymm1, 32(%rcx)7923; AVX-NEXT:    vmovdqa %xmm0, (%rcx)7924; AVX-NEXT:    vzeroupper7925; AVX-NEXT:    retq7926;7927; AVX2-LABEL: vec512_v4i128_to_v1i512_factor4:7928; AVX2:       # %bb.0:7929; AVX2-NEXT:    vmovdqa (%rdi), %xmm07930; AVX2-NEXT:    vpaddb (%rsi), %xmm0, %xmm07931; AVX2-NEXT:    vmovaps 32(%rdx), %ymm17932; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm07933; AVX2-NEXT:    vmovaps %ymm1, 32(%rcx)7934; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)7935; AVX2-NEXT:    vzeroupper7936; AVX2-NEXT:    retq7937;7938; AVX512F-LABEL: vec512_v4i128_to_v1i512_factor4:7939; AVX512F:       # %bb.0:7940; AVX512F-NEXT:    vmovdqa (%rdi), %xmm07941; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm07942; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm07943; AVX512F-NEXT:    vmovaps 32(%rdx), %ymm17944; AVX512F-NEXT:    vmovaps %ymm1, 32(%rcx)7945; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)7946; AVX512F-NEXT:    vzeroupper7947; AVX512F-NEXT:    retq7948;7949; AVX512BW-LABEL: vec512_v4i128_to_v1i512_factor4:7950; AVX512BW:       # %bb.0:7951; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm07952; AVX512BW-NEXT:    vpaddb (%rsi), %xmm0, %xmm07953; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm07954; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)7955; AVX512BW-NEXT:    vzeroupper7956; AVX512BW-NEXT:    retq7957  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 647958  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 647959  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias7960  %in.vec.cast = bitcast <64 x i8> %in.vec to <4 x i128>7961  %zextd.vec = shufflevector <4 x i128> %in.vec.cast, <4 x i128> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 6, i32 7>7962  %out.bytevec = bitcast <4 x i128> %zextd.vec to <64 x i8>7963  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 647964  %out.vec = add <64 x i8> %out.bytevec, %out.vec.bias7965  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 647966  ret void7967}7968 7969define void @vec512_v2i256_to_v1i512_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind {7970; SSE-LABEL: vec512_v2i256_to_v1i512_factor2:7971; SSE:       # %bb.0:7972; SSE-NEXT:    movdqa (%rdi), %xmm07973; SSE-NEXT:    movdqa 16(%rdi), %xmm17974; SSE-NEXT:    paddb (%rsi), %xmm07975; SSE-NEXT:    paddb 16(%rsi), %xmm17976; SSE-NEXT:    movaps 32(%rdx), %xmm27977; SSE-NEXT:    movaps 48(%rdx), %xmm37978; SSE-NEXT:    paddb 16(%rdx), %xmm17979; SSE-NEXT:    paddb (%rdx), %xmm07980; SSE-NEXT:    movaps %xmm2, 32(%rcx)7981; SSE-NEXT:    movaps %xmm3, 48(%rcx)7982; SSE-NEXT:    movdqa %xmm0, (%rcx)7983; SSE-NEXT:    movdqa %xmm1, 16(%rcx)7984; SSE-NEXT:    retq7985;7986; AVX-LABEL: vec512_v2i256_to_v1i512_factor2:7987; AVX:       # %bb.0:7988; AVX-NEXT:    vmovdqa (%rdi), %xmm07989; AVX-NEXT:    vmovdqa 16(%rdi), %xmm17990; AVX-NEXT:    vpaddb (%rsi), %xmm0, %xmm07991; AVX-NEXT:    vpaddb 16(%rsi), %xmm1, %xmm17992; AVX-NEXT:    vmovaps 32(%rdx), %ymm27993; AVX-NEXT:    vpaddb 16(%rdx), %xmm1, %xmm17994; AVX-NEXT:    vpaddb (%rdx), %xmm0, %xmm07995; AVX-NEXT:    vmovaps %ymm2, 32(%rcx)7996; AVX-NEXT:    vmovdqa %xmm0, (%rcx)7997; AVX-NEXT:    vmovdqa %xmm1, 16(%rcx)7998; AVX-NEXT:    vzeroupper7999; AVX-NEXT:    retq8000;8001; AVX2-LABEL: vec512_v2i256_to_v1i512_factor2:8002; AVX2:       # %bb.0:8003; AVX2-NEXT:    vmovdqa (%rdi), %ymm08004; AVX2-NEXT:    vpaddb (%rsi), %ymm0, %ymm08005; AVX2-NEXT:    vmovaps 32(%rdx), %ymm18006; AVX2-NEXT:    vpaddb (%rdx), %ymm0, %ymm08007; AVX2-NEXT:    vmovaps %ymm1, 32(%rcx)8008; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)8009; AVX2-NEXT:    vzeroupper8010; AVX2-NEXT:    retq8011;8012; AVX512F-LABEL: vec512_v2i256_to_v1i512_factor2:8013; AVX512F:       # %bb.0:8014; AVX512F-NEXT:    vmovdqa (%rdi), %ymm08015; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm08016; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm08017; AVX512F-NEXT:    vmovaps 32(%rdx), %ymm18018; AVX512F-NEXT:    vmovaps %ymm1, 32(%rcx)8019; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)8020; AVX512F-NEXT:    vzeroupper8021; AVX512F-NEXT:    retq8022;8023; AVX512BW-LABEL: vec512_v2i256_to_v1i512_factor2:8024; AVX512BW:       # %bb.0:8025; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm08026; AVX512BW-NEXT:    vpaddb (%rsi), %ymm0, %ymm08027; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm08028; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)8029; AVX512BW-NEXT:    vzeroupper8030; AVX512BW-NEXT:    retq8031  %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 648032  %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 648033  %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias8034  %in.vec.cast = bitcast <64 x i8> %in.vec to <2 x i256>8035  %zextd.vec = shufflevector <2 x i256> %in.vec.cast, <2 x i256> zeroinitializer, <2 x i32> <i32 0, i32 3>8036  %out.bytevec = bitcast <2 x i256> %zextd.vec to <64 x i8>8037  %out.vec.bias = load <64 x i8>, ptr %out.vec.bias.ptr, align 648038  %out.vec = add <64 x i8> %out.bytevec, %out.vec.bias8039  store <64 x i8> %out.vec, ptr %out.vec.ptr, align 648040  ret void8041}8042;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:8043; AVX1-ONLY: {{.*}}8044; FALLBACK0: {{.*}}8045; FALLBACK1: {{.*}}8046; FALLBACK2: {{.*}}8047; FALLBACK3: {{.*}}8048; FALLBACK4: {{.*}}8049; FALLBACK5: {{.*}}8050; FALLBACK6: {{.*}}8051; FALLBACK7: {{.*}}8052; FALLBACK8: {{.*}}8053; FALLBACK9: {{.*}}8054