1173 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512F3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512VL4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512BW5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512VBMI26; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512VLBW7; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512VLVBMI28 9declare <8 x i64> @llvm.fshl.v8i64(<8 x i64>, <8 x i64>, <8 x i64>)10declare <16 x i32> @llvm.fshl.v16i32(<16 x i32>, <16 x i32>, <16 x i32>)11declare <32 x i16> @llvm.fshl.v32i16(<32 x i16>, <32 x i16>, <32 x i16>)12declare <64 x i8> @llvm.fshl.v64i8(<64 x i8>, <64 x i8>, <64 x i8>)13 14;15; Variable Shifts16;17 18define <8 x i64> @var_funnnel_v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> %amt) nounwind {19; AVX512F-LABEL: var_funnnel_v8i64:20; AVX512F: # %bb.0:21; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]22; AVX512F-NEXT: vpandnq %zmm3, %zmm2, %zmm423; AVX512F-NEXT: vpsrlq $1, %zmm1, %zmm124; AVX512F-NEXT: vpsrlvq %zmm4, %zmm1, %zmm125; AVX512F-NEXT: vpandq %zmm3, %zmm2, %zmm226; AVX512F-NEXT: vpsllvq %zmm2, %zmm0, %zmm027; AVX512F-NEXT: vporq %zmm1, %zmm0, %zmm028; AVX512F-NEXT: retq29;30; AVX512VL-LABEL: var_funnnel_v8i64:31; AVX512VL: # %bb.0:32; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]33; AVX512VL-NEXT: vpandnq %zmm3, %zmm2, %zmm434; AVX512VL-NEXT: vpsrlq $1, %zmm1, %zmm135; AVX512VL-NEXT: vpsrlvq %zmm4, %zmm1, %zmm136; AVX512VL-NEXT: vpandq %zmm3, %zmm2, %zmm237; AVX512VL-NEXT: vpsllvq %zmm2, %zmm0, %zmm038; AVX512VL-NEXT: vporq %zmm1, %zmm0, %zmm039; AVX512VL-NEXT: retq40;41; AVX512BW-LABEL: var_funnnel_v8i64:42; AVX512BW: # %bb.0:43; AVX512BW-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]44; AVX512BW-NEXT: vpandnq %zmm3, %zmm2, %zmm445; AVX512BW-NEXT: vpsrlq $1, %zmm1, %zmm146; AVX512BW-NEXT: vpsrlvq %zmm4, %zmm1, %zmm147; AVX512BW-NEXT: vpandq %zmm3, %zmm2, %zmm248; AVX512BW-NEXT: vpsllvq %zmm2, %zmm0, %zmm049; AVX512BW-NEXT: vporq %zmm1, %zmm0, %zmm050; AVX512BW-NEXT: retq51;52; AVX512VBMI2-LABEL: var_funnnel_v8i64:53; AVX512VBMI2: # %bb.0:54; AVX512VBMI2-NEXT: vpshldvq %zmm2, %zmm1, %zmm055; AVX512VBMI2-NEXT: retq56;57; AVX512VLBW-LABEL: var_funnnel_v8i64:58; AVX512VLBW: # %bb.0:59; AVX512VLBW-NEXT: vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]60; AVX512VLBW-NEXT: vpandnq %zmm3, %zmm2, %zmm461; AVX512VLBW-NEXT: vpsrlq $1, %zmm1, %zmm162; AVX512VLBW-NEXT: vpsrlvq %zmm4, %zmm1, %zmm163; AVX512VLBW-NEXT: vpandq %zmm3, %zmm2, %zmm264; AVX512VLBW-NEXT: vpsllvq %zmm2, %zmm0, %zmm065; AVX512VLBW-NEXT: vporq %zmm1, %zmm0, %zmm066; AVX512VLBW-NEXT: retq67;68; AVX512VLVBMI2-LABEL: var_funnnel_v8i64:69; AVX512VLVBMI2: # %bb.0:70; AVX512VLVBMI2-NEXT: vpshldvq %zmm2, %zmm1, %zmm071; AVX512VLVBMI2-NEXT: retq72 %res = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> %amt)73 ret <8 x i64> %res74}75 76define <16 x i32> @var_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> %amt) nounwind {77; AVX512F-LABEL: var_funnnel_v16i32:78; AVX512F: # %bb.0:79; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]80; AVX512F-NEXT: vpandnd %zmm3, %zmm2, %zmm481; AVX512F-NEXT: vpsrld $1, %zmm1, %zmm182; AVX512F-NEXT: vpsrlvd %zmm4, %zmm1, %zmm183; AVX512F-NEXT: vpandd %zmm3, %zmm2, %zmm284; AVX512F-NEXT: vpsllvd %zmm2, %zmm0, %zmm085; AVX512F-NEXT: vpord %zmm1, %zmm0, %zmm086; AVX512F-NEXT: retq87;88; AVX512VL-LABEL: var_funnnel_v16i32:89; AVX512VL: # %bb.0:90; AVX512VL-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]91; AVX512VL-NEXT: vpandnd %zmm3, %zmm2, %zmm492; AVX512VL-NEXT: vpsrld $1, %zmm1, %zmm193; AVX512VL-NEXT: vpsrlvd %zmm4, %zmm1, %zmm194; AVX512VL-NEXT: vpandd %zmm3, %zmm2, %zmm295; AVX512VL-NEXT: vpsllvd %zmm2, %zmm0, %zmm096; AVX512VL-NEXT: vpord %zmm1, %zmm0, %zmm097; AVX512VL-NEXT: retq98;99; AVX512BW-LABEL: var_funnnel_v16i32:100; AVX512BW: # %bb.0:101; AVX512BW-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]102; AVX512BW-NEXT: vpandnd %zmm3, %zmm2, %zmm4103; AVX512BW-NEXT: vpsrld $1, %zmm1, %zmm1104; AVX512BW-NEXT: vpsrlvd %zmm4, %zmm1, %zmm1105; AVX512BW-NEXT: vpandd %zmm3, %zmm2, %zmm2106; AVX512BW-NEXT: vpsllvd %zmm2, %zmm0, %zmm0107; AVX512BW-NEXT: vpord %zmm1, %zmm0, %zmm0108; AVX512BW-NEXT: retq109;110; AVX512VBMI2-LABEL: var_funnnel_v16i32:111; AVX512VBMI2: # %bb.0:112; AVX512VBMI2-NEXT: vpshldvd %zmm2, %zmm1, %zmm0113; AVX512VBMI2-NEXT: retq114;115; AVX512VLBW-LABEL: var_funnnel_v16i32:116; AVX512VLBW: # %bb.0:117; AVX512VLBW-NEXT: vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]118; AVX512VLBW-NEXT: vpandnd %zmm3, %zmm2, %zmm4119; AVX512VLBW-NEXT: vpsrld $1, %zmm1, %zmm1120; AVX512VLBW-NEXT: vpsrlvd %zmm4, %zmm1, %zmm1121; AVX512VLBW-NEXT: vpandd %zmm3, %zmm2, %zmm2122; AVX512VLBW-NEXT: vpsllvd %zmm2, %zmm0, %zmm0123; AVX512VLBW-NEXT: vpord %zmm1, %zmm0, %zmm0124; AVX512VLBW-NEXT: retq125;126; AVX512VLVBMI2-LABEL: var_funnnel_v16i32:127; AVX512VLVBMI2: # %bb.0:128; AVX512VLVBMI2-NEXT: vpshldvd %zmm2, %zmm1, %zmm0129; AVX512VLVBMI2-NEXT: retq130 %res = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> %amt)131 ret <16 x i32> %res132}133 134define <32 x i16> @var_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> %amt) nounwind {135; AVX512F-LABEL: var_funnnel_v32i16:136; AVX512F: # %bb.0:137; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm3 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero138; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm4 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero139; AVX512F-NEXT: vpslld $16, %zmm4, %zmm4140; AVX512F-NEXT: vpord %zmm3, %zmm4, %zmm3141; AVX512F-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2142; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm4 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero143; AVX512F-NEXT: vpsllvd %zmm4, %zmm3, %zmm3144; AVX512F-NEXT: vpsrld $16, %zmm3, %zmm3145; AVX512F-NEXT: vpmovdw %zmm3, %ymm3146; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm1147; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero148; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm0149; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero150; AVX512F-NEXT: vpslld $16, %zmm0, %zmm0151; AVX512F-NEXT: vpord %zmm1, %zmm0, %zmm0152; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm1153; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero154; AVX512F-NEXT: vpsllvd %zmm1, %zmm0, %zmm0155; AVX512F-NEXT: vpsrld $16, %zmm0, %zmm0156; AVX512F-NEXT: vpmovdw %zmm0, %ymm0157; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm0158; AVX512F-NEXT: retq159;160; AVX512VL-LABEL: var_funnnel_v32i16:161; AVX512VL: # %bb.0:162; AVX512VL-NEXT: vpmovzxwd {{.*#+}} zmm3 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero163; AVX512VL-NEXT: vpmovzxwd {{.*#+}} zmm4 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero164; AVX512VL-NEXT: vpslld $16, %zmm4, %zmm4165; AVX512VL-NEXT: vpord %zmm3, %zmm4, %zmm3166; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2167; AVX512VL-NEXT: vpmovzxwd {{.*#+}} zmm4 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero168; AVX512VL-NEXT: vpsllvd %zmm4, %zmm3, %zmm3169; AVX512VL-NEXT: vpsrld $16, %zmm3, %zmm3170; AVX512VL-NEXT: vpmovdw %zmm3, %ymm3171; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm1172; AVX512VL-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero173; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm0174; AVX512VL-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero175; AVX512VL-NEXT: vpslld $16, %zmm0, %zmm0176; AVX512VL-NEXT: vpord %zmm1, %zmm0, %zmm0177; AVX512VL-NEXT: vextracti64x4 $1, %zmm2, %ymm1178; AVX512VL-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero179; AVX512VL-NEXT: vpsllvd %zmm1, %zmm0, %zmm0180; AVX512VL-NEXT: vpsrld $16, %zmm0, %zmm0181; AVX512VL-NEXT: vpmovdw %zmm0, %ymm0182; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm0183; AVX512VL-NEXT: retq184;185; AVX512BW-LABEL: var_funnnel_v32i16:186; AVX512BW: # %bb.0:187; AVX512BW-NEXT: vpbroadcastw {{.*#+}} zmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]188; AVX512BW-NEXT: vpandnq %zmm3, %zmm2, %zmm4189; AVX512BW-NEXT: vpsrlw $1, %zmm1, %zmm1190; AVX512BW-NEXT: vpsrlvw %zmm4, %zmm1, %zmm1191; AVX512BW-NEXT: vpandq %zmm3, %zmm2, %zmm2192; AVX512BW-NEXT: vpsllvw %zmm2, %zmm0, %zmm0193; AVX512BW-NEXT: vporq %zmm1, %zmm0, %zmm0194; AVX512BW-NEXT: retq195;196; AVX512VBMI2-LABEL: var_funnnel_v32i16:197; AVX512VBMI2: # %bb.0:198; AVX512VBMI2-NEXT: vpshldvw %zmm2, %zmm1, %zmm0199; AVX512VBMI2-NEXT: retq200;201; AVX512VLBW-LABEL: var_funnnel_v32i16:202; AVX512VLBW: # %bb.0:203; AVX512VLBW-NEXT: vpbroadcastw {{.*#+}} zmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]204; AVX512VLBW-NEXT: vpandnq %zmm3, %zmm2, %zmm4205; AVX512VLBW-NEXT: vpsrlw $1, %zmm1, %zmm1206; AVX512VLBW-NEXT: vpsrlvw %zmm4, %zmm1, %zmm1207; AVX512VLBW-NEXT: vpandq %zmm3, %zmm2, %zmm2208; AVX512VLBW-NEXT: vpsllvw %zmm2, %zmm0, %zmm0209; AVX512VLBW-NEXT: vporq %zmm1, %zmm0, %zmm0210; AVX512VLBW-NEXT: retq211;212; AVX512VLVBMI2-LABEL: var_funnnel_v32i16:213; AVX512VLVBMI2: # %bb.0:214; AVX512VLVBMI2-NEXT: vpshldvw %zmm2, %zmm1, %zmm0215; AVX512VLVBMI2-NEXT: retq216 %res = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> %amt)217 ret <32 x i16> %res218}219 220define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt) nounwind {221; AVX512F-LABEL: var_funnnel_v64i8:222; AVX512F: # %bb.0:223; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm3224; AVX512F-NEXT: vpsrlw $1, %ymm3, %ymm3225; AVX512F-NEXT: vpbroadcastb {{.*#+}} ymm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]226; AVX512F-NEXT: vpand %ymm4, %ymm3, %ymm5227; AVX512F-NEXT: vpsrlw $4, %ymm5, %ymm3228; AVX512F-NEXT: vpbroadcastb {{.*#+}} ymm6 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]229; AVX512F-NEXT: vpand %ymm6, %ymm3, %ymm7230; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm8 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]231; AVX512F-NEXT: vpandq %zmm8, %zmm2, %zmm2232; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm3233; AVX512F-NEXT: vpxor %ymm3, %ymm8, %ymm9234; AVX512F-NEXT: vpsllw $5, %ymm9, %ymm9235; AVX512F-NEXT: vpblendvb %ymm9, %ymm7, %ymm5, %ymm5236; AVX512F-NEXT: vpsrlw $2, %ymm5, %ymm7237; AVX512F-NEXT: vpbroadcastb {{.*#+}} ymm10 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]238; AVX512F-NEXT: vpand %ymm7, %ymm10, %ymm7239; AVX512F-NEXT: vpaddb %ymm9, %ymm9, %ymm9240; AVX512F-NEXT: vpblendvb %ymm9, %ymm7, %ymm5, %ymm5241; AVX512F-NEXT: vpsrlw $1, %ymm5, %ymm7242; AVX512F-NEXT: vpand %ymm4, %ymm7, %ymm7243; AVX512F-NEXT: vpaddb %ymm9, %ymm9, %ymm9244; AVX512F-NEXT: vpblendvb %ymm9, %ymm7, %ymm5, %ymm5245; AVX512F-NEXT: vpsrlw $1, %ymm1, %ymm1246; AVX512F-NEXT: vpand %ymm4, %ymm1, %ymm1247; AVX512F-NEXT: vpsrlw $4, %ymm1, %ymm7248; AVX512F-NEXT: vpand %ymm6, %ymm7, %ymm6249; AVX512F-NEXT: vpxor %ymm2, %ymm8, %ymm7250; AVX512F-NEXT: vpsllw $5, %ymm7, %ymm7251; AVX512F-NEXT: vpblendvb %ymm7, %ymm6, %ymm1, %ymm1252; AVX512F-NEXT: vpsrlw $2, %ymm1, %ymm6253; AVX512F-NEXT: vpand %ymm6, %ymm10, %ymm6254; AVX512F-NEXT: vpaddb %ymm7, %ymm7, %ymm7255; AVX512F-NEXT: vpblendvb %ymm7, %ymm6, %ymm1, %ymm1256; AVX512F-NEXT: vpsrlw $1, %ymm1, %ymm6257; AVX512F-NEXT: vpand %ymm4, %ymm6, %ymm4258; AVX512F-NEXT: vpaddb %ymm7, %ymm7, %ymm6259; AVX512F-NEXT: vpblendvb %ymm6, %ymm4, %ymm1, %ymm1260; AVX512F-NEXT: vinserti64x4 $1, %ymm5, %zmm1, %zmm1261; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm4262; AVX512F-NEXT: vpsllw $4, %ymm4, %ymm5263; AVX512F-NEXT: vpbroadcastb {{.*#+}} ymm6 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]264; AVX512F-NEXT: vpand %ymm6, %ymm5, %ymm5265; AVX512F-NEXT: vpsllw $5, %ymm3, %ymm3266; AVX512F-NEXT: vpblendvb %ymm3, %ymm5, %ymm4, %ymm4267; AVX512F-NEXT: vpsllw $2, %ymm4, %ymm5268; AVX512F-NEXT: vpbroadcastb {{.*#+}} ymm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]269; AVX512F-NEXT: vpand %ymm7, %ymm5, %ymm5270; AVX512F-NEXT: vpaddb %ymm3, %ymm3, %ymm3271; AVX512F-NEXT: vpblendvb %ymm3, %ymm5, %ymm4, %ymm4272; AVX512F-NEXT: vpaddb %ymm4, %ymm4, %ymm5273; AVX512F-NEXT: vpaddb %ymm3, %ymm3, %ymm3274; AVX512F-NEXT: vpblendvb %ymm3, %ymm5, %ymm4, %ymm3275; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm4276; AVX512F-NEXT: vpand %ymm6, %ymm4, %ymm4277; AVX512F-NEXT: vpsllw $5, %ymm2, %ymm2278; AVX512F-NEXT: vpblendvb %ymm2, %ymm4, %ymm0, %ymm0279; AVX512F-NEXT: vpsllw $2, %ymm0, %ymm4280; AVX512F-NEXT: vpand %ymm7, %ymm4, %ymm4281; AVX512F-NEXT: vpaddb %ymm2, %ymm2, %ymm2282; AVX512F-NEXT: vpblendvb %ymm2, %ymm4, %ymm0, %ymm0283; AVX512F-NEXT: vpaddb %ymm0, %ymm0, %ymm4284; AVX512F-NEXT: vpaddb %ymm2, %ymm2, %ymm2285; AVX512F-NEXT: vpblendvb %ymm2, %ymm4, %ymm0, %ymm0286; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0287; AVX512F-NEXT: vporq %zmm1, %zmm0, %zmm0288; AVX512F-NEXT: retq289;290; AVX512VL-LABEL: var_funnnel_v64i8:291; AVX512VL: # %bb.0:292; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm3293; AVX512VL-NEXT: vpsrlw $1, %ymm3, %ymm3294; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]295; AVX512VL-NEXT: vpand %ymm4, %ymm3, %ymm5296; AVX512VL-NEXT: vpsrlw $4, %ymm5, %ymm3297; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm6 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]298; AVX512VL-NEXT: vpand %ymm6, %ymm3, %ymm7299; AVX512VL-NEXT: vpbroadcastd {{.*#+}} zmm8 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]300; AVX512VL-NEXT: vpandq %zmm8, %zmm2, %zmm2301; AVX512VL-NEXT: vextracti64x4 $1, %zmm2, %ymm3302; AVX512VL-NEXT: vpxor %ymm3, %ymm8, %ymm9303; AVX512VL-NEXT: vpsllw $5, %ymm9, %ymm9304; AVX512VL-NEXT: vpblendvb %ymm9, %ymm7, %ymm5, %ymm5305; AVX512VL-NEXT: vpsrlw $2, %ymm5, %ymm7306; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm10 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]307; AVX512VL-NEXT: vpand %ymm7, %ymm10, %ymm7308; AVX512VL-NEXT: vpaddb %ymm9, %ymm9, %ymm9309; AVX512VL-NEXT: vpblendvb %ymm9, %ymm7, %ymm5, %ymm5310; AVX512VL-NEXT: vpsrlw $1, %ymm5, %ymm7311; AVX512VL-NEXT: vpand %ymm4, %ymm7, %ymm7312; AVX512VL-NEXT: vpaddb %ymm9, %ymm9, %ymm9313; AVX512VL-NEXT: vpblendvb %ymm9, %ymm7, %ymm5, %ymm5314; AVX512VL-NEXT: vpsrlw $1, %ymm1, %ymm1315; AVX512VL-NEXT: vpand %ymm4, %ymm1, %ymm1316; AVX512VL-NEXT: vpsrlw $4, %ymm1, %ymm7317; AVX512VL-NEXT: vpand %ymm6, %ymm7, %ymm6318; AVX512VL-NEXT: vpxor %ymm2, %ymm8, %ymm7319; AVX512VL-NEXT: vpsllw $5, %ymm7, %ymm7320; AVX512VL-NEXT: vpblendvb %ymm7, %ymm6, %ymm1, %ymm1321; AVX512VL-NEXT: vpsrlw $2, %ymm1, %ymm6322; AVX512VL-NEXT: vpand %ymm6, %ymm10, %ymm6323; AVX512VL-NEXT: vpaddb %ymm7, %ymm7, %ymm7324; AVX512VL-NEXT: vpblendvb %ymm7, %ymm6, %ymm1, %ymm1325; AVX512VL-NEXT: vpsrlw $1, %ymm1, %ymm6326; AVX512VL-NEXT: vpand %ymm4, %ymm6, %ymm4327; AVX512VL-NEXT: vpaddb %ymm7, %ymm7, %ymm6328; AVX512VL-NEXT: vpblendvb %ymm6, %ymm4, %ymm1, %ymm1329; AVX512VL-NEXT: vinserti64x4 $1, %ymm5, %zmm1, %zmm1330; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm4331; AVX512VL-NEXT: vpsllw $4, %ymm4, %ymm5332; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm6 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]333; AVX512VL-NEXT: vpand %ymm6, %ymm5, %ymm5334; AVX512VL-NEXT: vpsllw $5, %ymm3, %ymm3335; AVX512VL-NEXT: vpblendvb %ymm3, %ymm5, %ymm4, %ymm4336; AVX512VL-NEXT: vpsllw $2, %ymm4, %ymm5337; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]338; AVX512VL-NEXT: vpand %ymm7, %ymm5, %ymm5339; AVX512VL-NEXT: vpaddb %ymm3, %ymm3, %ymm3340; AVX512VL-NEXT: vpblendvb %ymm3, %ymm5, %ymm4, %ymm4341; AVX512VL-NEXT: vpaddb %ymm4, %ymm4, %ymm5342; AVX512VL-NEXT: vpaddb %ymm3, %ymm3, %ymm3343; AVX512VL-NEXT: vpblendvb %ymm3, %ymm5, %ymm4, %ymm3344; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm4345; AVX512VL-NEXT: vpand %ymm6, %ymm4, %ymm4346; AVX512VL-NEXT: vpsllw $5, %ymm2, %ymm2347; AVX512VL-NEXT: vpblendvb %ymm2, %ymm4, %ymm0, %ymm0348; AVX512VL-NEXT: vpsllw $2, %ymm0, %ymm4349; AVX512VL-NEXT: vpand %ymm7, %ymm4, %ymm4350; AVX512VL-NEXT: vpaddb %ymm2, %ymm2, %ymm2351; AVX512VL-NEXT: vpblendvb %ymm2, %ymm4, %ymm0, %ymm0352; AVX512VL-NEXT: vpaddb %ymm0, %ymm0, %ymm4353; AVX512VL-NEXT: vpaddb %ymm2, %ymm2, %ymm2354; AVX512VL-NEXT: vpblendvb %ymm2, %ymm4, %ymm0, %ymm0355; AVX512VL-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0356; AVX512VL-NEXT: vporq %zmm1, %zmm0, %zmm0357; AVX512VL-NEXT: retq358;359; AVX512BW-LABEL: var_funnnel_v64i8:360; AVX512BW: # %bb.0:361; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]362; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2363; AVX512BW-NEXT: vpxor %xmm4, %xmm4, %xmm4364; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm5 = zmm2[8],zmm4[8],zmm2[9],zmm4[9],zmm2[10],zmm4[10],zmm2[11],zmm4[11],zmm2[12],zmm4[12],zmm2[13],zmm4[13],zmm2[14],zmm4[14],zmm2[15],zmm4[15],zmm2[24],zmm4[24],zmm2[25],zmm4[25],zmm2[26],zmm4[26],zmm2[27],zmm4[27],zmm2[28],zmm4[28],zmm2[29],zmm4[29],zmm2[30],zmm4[30],zmm2[31],zmm4[31],zmm2[40],zmm4[40],zmm2[41],zmm4[41],zmm2[42],zmm4[42],zmm2[43],zmm4[43],zmm2[44],zmm4[44],zmm2[45],zmm4[45],zmm2[46],zmm4[46],zmm2[47],zmm4[47],zmm2[56],zmm4[56],zmm2[57],zmm4[57],zmm2[58],zmm4[58],zmm2[59],zmm4[59],zmm2[60],zmm4[60],zmm2[61],zmm4[61],zmm2[62],zmm4[62],zmm2[63],zmm4[63]365; AVX512BW-NEXT: vpsllvw %zmm5, %zmm3, %zmm3366; AVX512BW-NEXT: vpsrlw $8, %zmm3, %zmm3367; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]368; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm2[0],zmm4[0],zmm2[1],zmm4[1],zmm2[2],zmm4[2],zmm2[3],zmm4[3],zmm2[4],zmm4[4],zmm2[5],zmm4[5],zmm2[6],zmm4[6],zmm2[7],zmm4[7],zmm2[16],zmm4[16],zmm2[17],zmm4[17],zmm2[18],zmm4[18],zmm2[19],zmm4[19],zmm2[20],zmm4[20],zmm2[21],zmm4[21],zmm2[22],zmm4[22],zmm2[23],zmm4[23],zmm2[32],zmm4[32],zmm2[33],zmm4[33],zmm2[34],zmm4[34],zmm2[35],zmm4[35],zmm2[36],zmm4[36],zmm2[37],zmm4[37],zmm2[38],zmm4[38],zmm2[39],zmm4[39],zmm2[48],zmm4[48],zmm2[49],zmm4[49],zmm2[50],zmm4[50],zmm2[51],zmm4[51],zmm2[52],zmm4[52],zmm2[53],zmm4[53],zmm2[54],zmm4[54],zmm2[55],zmm4[55]369; AVX512BW-NEXT: vpsllvw %zmm1, %zmm0, %zmm0370; AVX512BW-NEXT: vpsrlw $8, %zmm0, %zmm0371; AVX512BW-NEXT: vpackuswb %zmm3, %zmm0, %zmm0372; AVX512BW-NEXT: retq373;374; AVX512VBMI2-LABEL: var_funnnel_v64i8:375; AVX512VBMI2: # %bb.0:376; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]377; AVX512VBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2378; AVX512VBMI2-NEXT: vpxor %xmm4, %xmm4, %xmm4379; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} zmm5 = zmm2[8],zmm4[8],zmm2[9],zmm4[9],zmm2[10],zmm4[10],zmm2[11],zmm4[11],zmm2[12],zmm4[12],zmm2[13],zmm4[13],zmm2[14],zmm4[14],zmm2[15],zmm4[15],zmm2[24],zmm4[24],zmm2[25],zmm4[25],zmm2[26],zmm4[26],zmm2[27],zmm4[27],zmm2[28],zmm4[28],zmm2[29],zmm4[29],zmm2[30],zmm4[30],zmm2[31],zmm4[31],zmm2[40],zmm4[40],zmm2[41],zmm4[41],zmm2[42],zmm4[42],zmm2[43],zmm4[43],zmm2[44],zmm4[44],zmm2[45],zmm4[45],zmm2[46],zmm4[46],zmm2[47],zmm4[47],zmm2[56],zmm4[56],zmm2[57],zmm4[57],zmm2[58],zmm4[58],zmm2[59],zmm4[59],zmm2[60],zmm4[60],zmm2[61],zmm4[61],zmm2[62],zmm4[62],zmm2[63],zmm4[63]380; AVX512VBMI2-NEXT: vpsllvw %zmm5, %zmm3, %zmm3381; AVX512VBMI2-NEXT: vpsrlw $8, %zmm3, %zmm3382; AVX512VBMI2-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]383; AVX512VBMI2-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm2[0],zmm4[0],zmm2[1],zmm4[1],zmm2[2],zmm4[2],zmm2[3],zmm4[3],zmm2[4],zmm4[4],zmm2[5],zmm4[5],zmm2[6],zmm4[6],zmm2[7],zmm4[7],zmm2[16],zmm4[16],zmm2[17],zmm4[17],zmm2[18],zmm4[18],zmm2[19],zmm4[19],zmm2[20],zmm4[20],zmm2[21],zmm4[21],zmm2[22],zmm4[22],zmm2[23],zmm4[23],zmm2[32],zmm4[32],zmm2[33],zmm4[33],zmm2[34],zmm4[34],zmm2[35],zmm4[35],zmm2[36],zmm4[36],zmm2[37],zmm4[37],zmm2[38],zmm4[38],zmm2[39],zmm4[39],zmm2[48],zmm4[48],zmm2[49],zmm4[49],zmm2[50],zmm4[50],zmm2[51],zmm4[51],zmm2[52],zmm4[52],zmm2[53],zmm4[53],zmm2[54],zmm4[54],zmm2[55],zmm4[55]384; AVX512VBMI2-NEXT: vpsllvw %zmm1, %zmm0, %zmm0385; AVX512VBMI2-NEXT: vpsrlw $8, %zmm0, %zmm0386; AVX512VBMI2-NEXT: vpackuswb %zmm3, %zmm0, %zmm0387; AVX512VBMI2-NEXT: retq388;389; AVX512VLBW-LABEL: var_funnnel_v64i8:390; AVX512VLBW: # %bb.0:391; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]392; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2393; AVX512VLBW-NEXT: vpxor %xmm4, %xmm4, %xmm4394; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} zmm5 = zmm2[8],zmm4[8],zmm2[9],zmm4[9],zmm2[10],zmm4[10],zmm2[11],zmm4[11],zmm2[12],zmm4[12],zmm2[13],zmm4[13],zmm2[14],zmm4[14],zmm2[15],zmm4[15],zmm2[24],zmm4[24],zmm2[25],zmm4[25],zmm2[26],zmm4[26],zmm2[27],zmm4[27],zmm2[28],zmm4[28],zmm2[29],zmm4[29],zmm2[30],zmm4[30],zmm2[31],zmm4[31],zmm2[40],zmm4[40],zmm2[41],zmm4[41],zmm2[42],zmm4[42],zmm2[43],zmm4[43],zmm2[44],zmm4[44],zmm2[45],zmm4[45],zmm2[46],zmm4[46],zmm2[47],zmm4[47],zmm2[56],zmm4[56],zmm2[57],zmm4[57],zmm2[58],zmm4[58],zmm2[59],zmm4[59],zmm2[60],zmm4[60],zmm2[61],zmm4[61],zmm2[62],zmm4[62],zmm2[63],zmm4[63]395; AVX512VLBW-NEXT: vpsllvw %zmm5, %zmm3, %zmm3396; AVX512VLBW-NEXT: vpsrlw $8, %zmm3, %zmm3397; AVX512VLBW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]398; AVX512VLBW-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm2[0],zmm4[0],zmm2[1],zmm4[1],zmm2[2],zmm4[2],zmm2[3],zmm4[3],zmm2[4],zmm4[4],zmm2[5],zmm4[5],zmm2[6],zmm4[6],zmm2[7],zmm4[7],zmm2[16],zmm4[16],zmm2[17],zmm4[17],zmm2[18],zmm4[18],zmm2[19],zmm4[19],zmm2[20],zmm4[20],zmm2[21],zmm4[21],zmm2[22],zmm4[22],zmm2[23],zmm4[23],zmm2[32],zmm4[32],zmm2[33],zmm4[33],zmm2[34],zmm4[34],zmm2[35],zmm4[35],zmm2[36],zmm4[36],zmm2[37],zmm4[37],zmm2[38],zmm4[38],zmm2[39],zmm4[39],zmm2[48],zmm4[48],zmm2[49],zmm4[49],zmm2[50],zmm4[50],zmm2[51],zmm4[51],zmm2[52],zmm4[52],zmm2[53],zmm4[53],zmm2[54],zmm4[54],zmm2[55],zmm4[55]399; AVX512VLBW-NEXT: vpsllvw %zmm1, %zmm0, %zmm0400; AVX512VLBW-NEXT: vpsrlw $8, %zmm0, %zmm0401; AVX512VLBW-NEXT: vpackuswb %zmm3, %zmm0, %zmm0402; AVX512VLBW-NEXT: retq403;404; AVX512VLVBMI2-LABEL: var_funnnel_v64i8:405; AVX512VLVBMI2: # %bb.0:406; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]407; AVX512VLVBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2408; AVX512VLVBMI2-NEXT: vpxor %xmm4, %xmm4, %xmm4409; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} zmm5 = zmm2[8],zmm4[8],zmm2[9],zmm4[9],zmm2[10],zmm4[10],zmm2[11],zmm4[11],zmm2[12],zmm4[12],zmm2[13],zmm4[13],zmm2[14],zmm4[14],zmm2[15],zmm4[15],zmm2[24],zmm4[24],zmm2[25],zmm4[25],zmm2[26],zmm4[26],zmm2[27],zmm4[27],zmm2[28],zmm4[28],zmm2[29],zmm4[29],zmm2[30],zmm4[30],zmm2[31],zmm4[31],zmm2[40],zmm4[40],zmm2[41],zmm4[41],zmm2[42],zmm4[42],zmm2[43],zmm4[43],zmm2[44],zmm4[44],zmm2[45],zmm4[45],zmm2[46],zmm4[46],zmm2[47],zmm4[47],zmm2[56],zmm4[56],zmm2[57],zmm4[57],zmm2[58],zmm4[58],zmm2[59],zmm4[59],zmm2[60],zmm4[60],zmm2[61],zmm4[61],zmm2[62],zmm4[62],zmm2[63],zmm4[63]410; AVX512VLVBMI2-NEXT: vpsllvw %zmm5, %zmm3, %zmm3411; AVX512VLVBMI2-NEXT: vpsrlw $8, %zmm3, %zmm3412; AVX512VLVBMI2-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]413; AVX512VLVBMI2-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm2[0],zmm4[0],zmm2[1],zmm4[1],zmm2[2],zmm4[2],zmm2[3],zmm4[3],zmm2[4],zmm4[4],zmm2[5],zmm4[5],zmm2[6],zmm4[6],zmm2[7],zmm4[7],zmm2[16],zmm4[16],zmm2[17],zmm4[17],zmm2[18],zmm4[18],zmm2[19],zmm4[19],zmm2[20],zmm4[20],zmm2[21],zmm4[21],zmm2[22],zmm4[22],zmm2[23],zmm4[23],zmm2[32],zmm4[32],zmm2[33],zmm4[33],zmm2[34],zmm4[34],zmm2[35],zmm4[35],zmm2[36],zmm4[36],zmm2[37],zmm4[37],zmm2[38],zmm4[38],zmm2[39],zmm4[39],zmm2[48],zmm4[48],zmm2[49],zmm4[49],zmm2[50],zmm4[50],zmm2[51],zmm4[51],zmm2[52],zmm4[52],zmm2[53],zmm4[53],zmm2[54],zmm4[54],zmm2[55],zmm4[55]414; AVX512VLVBMI2-NEXT: vpsllvw %zmm1, %zmm0, %zmm0415; AVX512VLVBMI2-NEXT: vpsrlw $8, %zmm0, %zmm0416; AVX512VLVBMI2-NEXT: vpackuswb %zmm3, %zmm0, %zmm0417; AVX512VLVBMI2-NEXT: retq418 %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt)419 ret <64 x i8> %res420}421 422;423; Uniform Variable Shifts424;425 426define <8 x i64> @splatvar_funnnel_v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> %amt) nounwind {427; AVX512F-LABEL: splatvar_funnnel_v8i64:428; AVX512F: # %bb.0:429; AVX512F-NEXT: vpmovsxbq {{.*#+}} xmm3 = [63,63]430; AVX512F-NEXT: vpandn %xmm3, %xmm2, %xmm4431; AVX512F-NEXT: vpsrlq $1, %zmm1, %zmm1432; AVX512F-NEXT: vpsrlq %xmm4, %zmm1, %zmm1433; AVX512F-NEXT: vpand %xmm3, %xmm2, %xmm2434; AVX512F-NEXT: vpsllq %xmm2, %zmm0, %zmm0435; AVX512F-NEXT: vporq %zmm1, %zmm0, %zmm0436; AVX512F-NEXT: retq437;438; AVX512VL-LABEL: splatvar_funnnel_v8i64:439; AVX512VL: # %bb.0:440; AVX512VL-NEXT: vpbroadcastq {{.*#+}} xmm3 = [63,63]441; AVX512VL-NEXT: vpandn %xmm3, %xmm2, %xmm4442; AVX512VL-NEXT: vpsrlq $1, %zmm1, %zmm1443; AVX512VL-NEXT: vpsrlq %xmm4, %zmm1, %zmm1444; AVX512VL-NEXT: vpand %xmm3, %xmm2, %xmm2445; AVX512VL-NEXT: vpsllq %xmm2, %zmm0, %zmm0446; AVX512VL-NEXT: vporq %zmm1, %zmm0, %zmm0447; AVX512VL-NEXT: retq448;449; AVX512BW-LABEL: splatvar_funnnel_v8i64:450; AVX512BW: # %bb.0:451; AVX512BW-NEXT: vpmovsxbq {{.*#+}} xmm3 = [63,63]452; AVX512BW-NEXT: vpandn %xmm3, %xmm2, %xmm4453; AVX512BW-NEXT: vpsrlq $1, %zmm1, %zmm1454; AVX512BW-NEXT: vpsrlq %xmm4, %zmm1, %zmm1455; AVX512BW-NEXT: vpand %xmm3, %xmm2, %xmm2456; AVX512BW-NEXT: vpsllq %xmm2, %zmm0, %zmm0457; AVX512BW-NEXT: vporq %zmm1, %zmm0, %zmm0458; AVX512BW-NEXT: retq459;460; AVX512VBMI2-LABEL: splatvar_funnnel_v8i64:461; AVX512VBMI2: # %bb.0:462; AVX512VBMI2-NEXT: vpbroadcastq %xmm2, %zmm2463; AVX512VBMI2-NEXT: vpshldvq %zmm2, %zmm1, %zmm0464; AVX512VBMI2-NEXT: retq465;466; AVX512VLBW-LABEL: splatvar_funnnel_v8i64:467; AVX512VLBW: # %bb.0:468; AVX512VLBW-NEXT: vpbroadcastq {{.*#+}} xmm3 = [63,63]469; AVX512VLBW-NEXT: vpandn %xmm3, %xmm2, %xmm4470; AVX512VLBW-NEXT: vpsrlq $1, %zmm1, %zmm1471; AVX512VLBW-NEXT: vpsrlq %xmm4, %zmm1, %zmm1472; AVX512VLBW-NEXT: vpand %xmm3, %xmm2, %xmm2473; AVX512VLBW-NEXT: vpsllq %xmm2, %zmm0, %zmm0474; AVX512VLBW-NEXT: vporq %zmm1, %zmm0, %zmm0475; AVX512VLBW-NEXT: retq476;477; AVX512VLVBMI2-LABEL: splatvar_funnnel_v8i64:478; AVX512VLVBMI2: # %bb.0:479; AVX512VLVBMI2-NEXT: vpbroadcastq %xmm2, %zmm2480; AVX512VLVBMI2-NEXT: vpshldvq %zmm2, %zmm1, %zmm0481; AVX512VLVBMI2-NEXT: retq482 %splat = shufflevector <8 x i64> %amt, <8 x i64> undef, <8 x i32> zeroinitializer483 %res = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> %splat)484 ret <8 x i64> %res485}486 487define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> %amt) nounwind {488; AVX512F-LABEL: splatvar_funnnel_v16i32:489; AVX512F: # %bb.0:490; AVX512F-NEXT: vpunpckhdq {{.*#+}} zmm3 = zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[14],zmm0[14],zmm1[15],zmm0[15]491; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2492; AVX512F-NEXT: vpsllq %xmm2, %zmm3, %zmm3493; AVX512F-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[12],zmm0[12],zmm1[13],zmm0[13]494; AVX512F-NEXT: vpsllq %xmm2, %zmm0, %zmm0495; AVX512F-NEXT: vshufps {{.*#+}} zmm0 = zmm0[1,3],zmm3[1,3],zmm0[5,7],zmm3[5,7],zmm0[9,11],zmm3[9,11],zmm0[13,15],zmm3[13,15]496; AVX512F-NEXT: retq497;498; AVX512VL-LABEL: splatvar_funnnel_v16i32:499; AVX512VL: # %bb.0:500; AVX512VL-NEXT: vpunpckhdq {{.*#+}} zmm3 = zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[14],zmm0[14],zmm1[15],zmm0[15]501; AVX512VL-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2502; AVX512VL-NEXT: vpsllq %xmm2, %zmm3, %zmm3503; AVX512VL-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[12],zmm0[12],zmm1[13],zmm0[13]504; AVX512VL-NEXT: vpsllq %xmm2, %zmm0, %zmm0505; AVX512VL-NEXT: vshufps {{.*#+}} zmm0 = zmm0[1,3],zmm3[1,3],zmm0[5,7],zmm3[5,7],zmm0[9,11],zmm3[9,11],zmm0[13,15],zmm3[13,15]506; AVX512VL-NEXT: retq507;508; AVX512BW-LABEL: splatvar_funnnel_v16i32:509; AVX512BW: # %bb.0:510; AVX512BW-NEXT: vpunpckhdq {{.*#+}} zmm3 = zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[14],zmm0[14],zmm1[15],zmm0[15]511; AVX512BW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2512; AVX512BW-NEXT: vpsllq %xmm2, %zmm3, %zmm3513; AVX512BW-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[12],zmm0[12],zmm1[13],zmm0[13]514; AVX512BW-NEXT: vpsllq %xmm2, %zmm0, %zmm0515; AVX512BW-NEXT: vshufps {{.*#+}} zmm0 = zmm0[1,3],zmm3[1,3],zmm0[5,7],zmm3[5,7],zmm0[9,11],zmm3[9,11],zmm0[13,15],zmm3[13,15]516; AVX512BW-NEXT: retq517;518; AVX512VBMI2-LABEL: splatvar_funnnel_v16i32:519; AVX512VBMI2: # %bb.0:520; AVX512VBMI2-NEXT: vpbroadcastd %xmm2, %zmm2521; AVX512VBMI2-NEXT: vpshldvd %zmm2, %zmm1, %zmm0522; AVX512VBMI2-NEXT: retq523;524; AVX512VLBW-LABEL: splatvar_funnnel_v16i32:525; AVX512VLBW: # %bb.0:526; AVX512VLBW-NEXT: vpunpckhdq {{.*#+}} zmm3 = zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[14],zmm0[14],zmm1[15],zmm0[15]527; AVX512VLBW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2528; AVX512VLBW-NEXT: vpsllq %xmm2, %zmm3, %zmm3529; AVX512VLBW-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[12],zmm0[12],zmm1[13],zmm0[13]530; AVX512VLBW-NEXT: vpsllq %xmm2, %zmm0, %zmm0531; AVX512VLBW-NEXT: vshufps {{.*#+}} zmm0 = zmm0[1,3],zmm3[1,3],zmm0[5,7],zmm3[5,7],zmm0[9,11],zmm3[9,11],zmm0[13,15],zmm3[13,15]532; AVX512VLBW-NEXT: retq533;534; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i32:535; AVX512VLVBMI2: # %bb.0:536; AVX512VLVBMI2-NEXT: vpbroadcastd %xmm2, %zmm2537; AVX512VLVBMI2-NEXT: vpshldvd %zmm2, %zmm1, %zmm0538; AVX512VLVBMI2-NEXT: retq539 %splat = shufflevector <16 x i32> %amt, <16 x i32> undef, <16 x i32> zeroinitializer540 %res = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> %splat)541 ret <16 x i32> %res542}543 544define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> %amt) nounwind {545; AVX512F-LABEL: splatvar_funnnel_v32i16:546; AVX512F: # %bb.0:547; AVX512F-NEXT: vpbroadcastq {{.*#+}} xmm3 = [15,0,0,0,15,0,0,0]548; AVX512F-NEXT: vpandn %xmm3, %xmm2, %xmm4549; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm5550; AVX512F-NEXT: vpsrlw $1, %ymm5, %ymm5551; AVX512F-NEXT: vpsrlw %xmm4, %ymm5, %ymm5552; AVX512F-NEXT: vpsrlw $1, %ymm1, %ymm1553; AVX512F-NEXT: vpsrlw %xmm4, %ymm1, %ymm1554; AVX512F-NEXT: vinserti64x4 $1, %ymm5, %zmm1, %zmm1555; AVX512F-NEXT: vpand %xmm3, %xmm2, %xmm2556; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm3557; AVX512F-NEXT: vpsllw %xmm2, %ymm3, %ymm3558; AVX512F-NEXT: vpsllw %xmm2, %ymm0, %ymm0559; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0560; AVX512F-NEXT: vporq %zmm1, %zmm0, %zmm0561; AVX512F-NEXT: retq562;563; AVX512VL-LABEL: splatvar_funnnel_v32i16:564; AVX512VL: # %bb.0:565; AVX512VL-NEXT: vpbroadcastq {{.*#+}} xmm3 = [15,0,0,0,15,0,0,0]566; AVX512VL-NEXT: vpandn %xmm3, %xmm2, %xmm4567; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm5568; AVX512VL-NEXT: vpsrlw $1, %ymm5, %ymm5569; AVX512VL-NEXT: vpsrlw %xmm4, %ymm5, %ymm5570; AVX512VL-NEXT: vpsrlw $1, %ymm1, %ymm1571; AVX512VL-NEXT: vpsrlw %xmm4, %ymm1, %ymm1572; AVX512VL-NEXT: vinserti64x4 $1, %ymm5, %zmm1, %zmm1573; AVX512VL-NEXT: vpand %xmm3, %xmm2, %xmm2574; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm3575; AVX512VL-NEXT: vpsllw %xmm2, %ymm3, %ymm3576; AVX512VL-NEXT: vpsllw %xmm2, %ymm0, %ymm0577; AVX512VL-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0578; AVX512VL-NEXT: vporq %zmm1, %zmm0, %zmm0579; AVX512VL-NEXT: retq580;581; AVX512BW-LABEL: splatvar_funnnel_v32i16:582; AVX512BW: # %bb.0:583; AVX512BW-NEXT: vpbroadcastq {{.*#+}} xmm3 = [15,0,0,0,15,0,0,0]584; AVX512BW-NEXT: vpandn %xmm3, %xmm2, %xmm4585; AVX512BW-NEXT: vpsrlw $1, %zmm1, %zmm1586; AVX512BW-NEXT: vpsrlw %xmm4, %zmm1, %zmm1587; AVX512BW-NEXT: vpand %xmm3, %xmm2, %xmm2588; AVX512BW-NEXT: vpsllw %xmm2, %zmm0, %zmm0589; AVX512BW-NEXT: vporq %zmm1, %zmm0, %zmm0590; AVX512BW-NEXT: retq591;592; AVX512VBMI2-LABEL: splatvar_funnnel_v32i16:593; AVX512VBMI2: # %bb.0:594; AVX512VBMI2-NEXT: vpbroadcastw %xmm2, %zmm2595; AVX512VBMI2-NEXT: vpshldvw %zmm2, %zmm1, %zmm0596; AVX512VBMI2-NEXT: retq597;598; AVX512VLBW-LABEL: splatvar_funnnel_v32i16:599; AVX512VLBW: # %bb.0:600; AVX512VLBW-NEXT: vpbroadcastq {{.*#+}} xmm3 = [15,0,0,0,15,0,0,0]601; AVX512VLBW-NEXT: vpandn %xmm3, %xmm2, %xmm4602; AVX512VLBW-NEXT: vpsrlw $1, %zmm1, %zmm1603; AVX512VLBW-NEXT: vpsrlw %xmm4, %zmm1, %zmm1604; AVX512VLBW-NEXT: vpand %xmm3, %xmm2, %xmm2605; AVX512VLBW-NEXT: vpsllw %xmm2, %zmm0, %zmm0606; AVX512VLBW-NEXT: vporq %zmm1, %zmm0, %zmm0607; AVX512VLBW-NEXT: retq608;609; AVX512VLVBMI2-LABEL: splatvar_funnnel_v32i16:610; AVX512VLVBMI2: # %bb.0:611; AVX512VLVBMI2-NEXT: vpbroadcastw %xmm2, %zmm2612; AVX512VLVBMI2-NEXT: vpshldvw %zmm2, %zmm1, %zmm0613; AVX512VLVBMI2-NEXT: retq614 %splat = shufflevector <32 x i16> %amt, <32 x i16> undef, <32 x i32> zeroinitializer615 %res = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> %splat)616 ret <32 x i16> %res617}618 619define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt) nounwind {620; AVX512F-LABEL: splatvar_funnnel_v64i8:621; AVX512F: # %bb.0:622; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm3623; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm4624; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm5 = ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15],ymm4[24],ymm3[24],ymm4[25],ymm3[25],ymm4[26],ymm3[26],ymm4[27],ymm3[27],ymm4[28],ymm3[28],ymm4[29],ymm3[29],ymm4[30],ymm3[30],ymm4[31],ymm3[31]625; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2626; AVX512F-NEXT: vpsllw %xmm2, %ymm5, %ymm5627; AVX512F-NEXT: vpsrlw $8, %ymm5, %ymm5628; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[16],ymm3[16],ymm4[17],ymm3[17],ymm4[18],ymm3[18],ymm4[19],ymm3[19],ymm4[20],ymm3[20],ymm4[21],ymm3[21],ymm4[22],ymm3[22],ymm4[23],ymm3[23]629; AVX512F-NEXT: vpsllw %xmm2, %ymm3, %ymm3630; AVX512F-NEXT: vpsrlw $8, %ymm3, %ymm3631; AVX512F-NEXT: vpackuswb %ymm5, %ymm3, %ymm3632; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]633; AVX512F-NEXT: vpsllw %xmm2, %ymm4, %ymm4634; AVX512F-NEXT: vpsrlw $8, %ymm4, %ymm4635; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]636; AVX512F-NEXT: vpsllw %xmm2, %ymm0, %ymm0637; AVX512F-NEXT: vpsrlw $8, %ymm0, %ymm0638; AVX512F-NEXT: vpackuswb %ymm4, %ymm0, %ymm0639; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0640; AVX512F-NEXT: retq641;642; AVX512VL-LABEL: splatvar_funnnel_v64i8:643; AVX512VL: # %bb.0:644; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm3645; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm4646; AVX512VL-NEXT: vpunpckhbw {{.*#+}} ymm5 = ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15],ymm4[24],ymm3[24],ymm4[25],ymm3[25],ymm4[26],ymm3[26],ymm4[27],ymm3[27],ymm4[28],ymm3[28],ymm4[29],ymm3[29],ymm4[30],ymm3[30],ymm4[31],ymm3[31]647; AVX512VL-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2648; AVX512VL-NEXT: vpsllw %xmm2, %ymm5, %ymm5649; AVX512VL-NEXT: vpsrlw $8, %ymm5, %ymm5650; AVX512VL-NEXT: vpunpcklbw {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[16],ymm3[16],ymm4[17],ymm3[17],ymm4[18],ymm3[18],ymm4[19],ymm3[19],ymm4[20],ymm3[20],ymm4[21],ymm3[21],ymm4[22],ymm3[22],ymm4[23],ymm3[23]651; AVX512VL-NEXT: vpsllw %xmm2, %ymm3, %ymm3652; AVX512VL-NEXT: vpsrlw $8, %ymm3, %ymm3653; AVX512VL-NEXT: vpackuswb %ymm5, %ymm3, %ymm3654; AVX512VL-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]655; AVX512VL-NEXT: vpsllw %xmm2, %ymm4, %ymm4656; AVX512VL-NEXT: vpsrlw $8, %ymm4, %ymm4657; AVX512VL-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]658; AVX512VL-NEXT: vpsllw %xmm2, %ymm0, %ymm0659; AVX512VL-NEXT: vpsrlw $8, %ymm0, %ymm0660; AVX512VL-NEXT: vpackuswb %ymm4, %ymm0, %ymm0661; AVX512VL-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0662; AVX512VL-NEXT: retq663;664; AVX512BW-LABEL: splatvar_funnnel_v64i8:665; AVX512BW: # %bb.0:666; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]667; AVX512BW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2668; AVX512BW-NEXT: vpsllw %xmm2, %zmm3, %zmm3669; AVX512BW-NEXT: vpsrlw $8, %zmm3, %zmm3670; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]671; AVX512BW-NEXT: vpsllw %xmm2, %zmm0, %zmm0672; AVX512BW-NEXT: vpsrlw $8, %zmm0, %zmm0673; AVX512BW-NEXT: vpackuswb %zmm3, %zmm0, %zmm0674; AVX512BW-NEXT: retq675;676; AVX512VBMI2-LABEL: splatvar_funnnel_v64i8:677; AVX512VBMI2: # %bb.0:678; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]679; AVX512VBMI2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2680; AVX512VBMI2-NEXT: vpsllw %xmm2, %zmm3, %zmm3681; AVX512VBMI2-NEXT: vpsrlw $8, %zmm3, %zmm3682; AVX512VBMI2-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]683; AVX512VBMI2-NEXT: vpsllw %xmm2, %zmm0, %zmm0684; AVX512VBMI2-NEXT: vpsrlw $8, %zmm0, %zmm0685; AVX512VBMI2-NEXT: vpackuswb %zmm3, %zmm0, %zmm0686; AVX512VBMI2-NEXT: retq687;688; AVX512VLBW-LABEL: splatvar_funnnel_v64i8:689; AVX512VLBW: # %bb.0:690; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]691; AVX512VLBW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2692; AVX512VLBW-NEXT: vpsllw %xmm2, %zmm3, %zmm3693; AVX512VLBW-NEXT: vpsrlw $8, %zmm3, %zmm3694; AVX512VLBW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]695; AVX512VLBW-NEXT: vpsllw %xmm2, %zmm0, %zmm0696; AVX512VLBW-NEXT: vpsrlw $8, %zmm0, %zmm0697; AVX512VLBW-NEXT: vpackuswb %zmm3, %zmm0, %zmm0698; AVX512VLBW-NEXT: retq699;700; AVX512VLVBMI2-LABEL: splatvar_funnnel_v64i8:701; AVX512VLVBMI2: # %bb.0:702; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]703; AVX512VLVBMI2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2704; AVX512VLVBMI2-NEXT: vpsllw %xmm2, %zmm3, %zmm3705; AVX512VLVBMI2-NEXT: vpsrlw $8, %zmm3, %zmm3706; AVX512VLVBMI2-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]707; AVX512VLVBMI2-NEXT: vpsllw %xmm2, %zmm0, %zmm0708; AVX512VLVBMI2-NEXT: vpsrlw $8, %zmm0, %zmm0709; AVX512VLVBMI2-NEXT: vpackuswb %zmm3, %zmm0, %zmm0710; AVX512VLVBMI2-NEXT: retq711 %splat = shufflevector <64 x i8> %amt, <64 x i8> undef, <64 x i32> zeroinitializer712 %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %splat)713 ret <64 x i8> %res714}715 716;717; Constant Shifts718;719 720define <8 x i64> @constant_funnnel_v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {721; AVX512F-LABEL: constant_funnnel_v8i64:722; AVX512F: # %bb.0:723; AVX512F-NEXT: vpsrlvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1724; AVX512F-NEXT: vpsllvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0725; AVX512F-NEXT: vporq %zmm1, %zmm0, %zmm0726; AVX512F-NEXT: retq727;728; AVX512VL-LABEL: constant_funnnel_v8i64:729; AVX512VL: # %bb.0:730; AVX512VL-NEXT: vpsrlvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1731; AVX512VL-NEXT: vpsllvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0732; AVX512VL-NEXT: vporq %zmm1, %zmm0, %zmm0733; AVX512VL-NEXT: retq734;735; AVX512BW-LABEL: constant_funnnel_v8i64:736; AVX512BW: # %bb.0:737; AVX512BW-NEXT: vpsrlvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1738; AVX512BW-NEXT: vpsllvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0739; AVX512BW-NEXT: vporq %zmm1, %zmm0, %zmm0740; AVX512BW-NEXT: retq741;742; AVX512VBMI2-LABEL: constant_funnnel_v8i64:743; AVX512VBMI2: # %bb.0:744; AVX512VBMI2-NEXT: vpshldvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0745; AVX512VBMI2-NEXT: retq746;747; AVX512VLBW-LABEL: constant_funnnel_v8i64:748; AVX512VLBW: # %bb.0:749; AVX512VLBW-NEXT: vpsrlvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1750; AVX512VLBW-NEXT: vpsllvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0751; AVX512VLBW-NEXT: vporq %zmm1, %zmm0, %zmm0752; AVX512VLBW-NEXT: retq753;754; AVX512VLVBMI2-LABEL: constant_funnnel_v8i64:755; AVX512VLVBMI2: # %bb.0:756; AVX512VLVBMI2-NEXT: vpshldvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0757; AVX512VLVBMI2-NEXT: retq758 %res = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> <i64 4, i64 14, i64 50, i64 60, i64 4, i64 14, i64 50, i64 60>)759 ret <8 x i64> %res760}761 762define <16 x i32> @constant_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {763; AVX512F-LABEL: constant_funnnel_v16i32:764; AVX512F: # %bb.0:765; AVX512F-NEXT: vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1766; AVX512F-NEXT: vpsllvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0767; AVX512F-NEXT: vpord %zmm1, %zmm0, %zmm0768; AVX512F-NEXT: retq769;770; AVX512VL-LABEL: constant_funnnel_v16i32:771; AVX512VL: # %bb.0:772; AVX512VL-NEXT: vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1773; AVX512VL-NEXT: vpsllvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0774; AVX512VL-NEXT: vpord %zmm1, %zmm0, %zmm0775; AVX512VL-NEXT: retq776;777; AVX512BW-LABEL: constant_funnnel_v16i32:778; AVX512BW: # %bb.0:779; AVX512BW-NEXT: vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1780; AVX512BW-NEXT: vpsllvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0781; AVX512BW-NEXT: vpord %zmm1, %zmm0, %zmm0782; AVX512BW-NEXT: retq783;784; AVX512VBMI2-LABEL: constant_funnnel_v16i32:785; AVX512VBMI2: # %bb.0:786; AVX512VBMI2-NEXT: vpshldvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0787; AVX512VBMI2-NEXT: retq788;789; AVX512VLBW-LABEL: constant_funnnel_v16i32:790; AVX512VLBW: # %bb.0:791; AVX512VLBW-NEXT: vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1792; AVX512VLBW-NEXT: vpsllvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0793; AVX512VLBW-NEXT: vpord %zmm1, %zmm0, %zmm0794; AVX512VLBW-NEXT: retq795;796; AVX512VLVBMI2-LABEL: constant_funnnel_v16i32:797; AVX512VLVBMI2: # %bb.0:798; AVX512VLVBMI2-NEXT: vpshldvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0799; AVX512VLVBMI2-NEXT: retq800 %res = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>)801 ret <16 x i32> %res802}803 804define <32 x i16> @constant_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y) nounwind {805; AVX512F-LABEL: constant_funnnel_v32i16:806; AVX512F: # %bb.0:807; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm2808; AVX512F-NEXT: vpsrlw $1, %ymm2, %ymm2809; AVX512F-NEXT: vmovdqa {{.*#+}} ymm3 = [2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768,u]810; AVX512F-NEXT: vpmulhuw %ymm3, %ymm2, %ymm4811; AVX512F-NEXT: vpblendw {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7],ymm4[8,9,10,11,12,13,14],ymm2[15]812; AVX512F-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]813; AVX512F-NEXT: vpsrlw $1, %ymm1, %ymm1814; AVX512F-NEXT: vpmulhuw %ymm3, %ymm1, %ymm3815; AVX512F-NEXT: vpblendw {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7],ymm3[8,9,10,11,12,13,14],ymm1[15]816; AVX512F-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]817; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1818; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2819; AVX512F-NEXT: vmovdqa {{.*#+}} ymm3 = [1,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768]820; AVX512F-NEXT: vpmullw %ymm3, %ymm2, %ymm2821; AVX512F-NEXT: vpmullw %ymm3, %ymm0, %ymm0822; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0823; AVX512F-NEXT: vporq %zmm1, %zmm0, %zmm0824; AVX512F-NEXT: retq825;826; AVX512VL-LABEL: constant_funnnel_v32i16:827; AVX512VL: # %bb.0:828; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm2829; AVX512VL-NEXT: vpsrlw $1, %ymm2, %ymm2830; AVX512VL-NEXT: vmovdqa {{.*#+}} ymm3 = [2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768,u]831; AVX512VL-NEXT: vpmulhuw %ymm3, %ymm2, %ymm4832; AVX512VL-NEXT: vpblendw {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7],ymm4[8,9,10,11,12,13,14],ymm2[15]833; AVX512VL-NEXT: vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]834; AVX512VL-NEXT: vpsrlw $1, %ymm1, %ymm1835; AVX512VL-NEXT: vpmulhuw %ymm3, %ymm1, %ymm3836; AVX512VL-NEXT: vpblendw {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7],ymm3[8,9,10,11,12,13,14],ymm1[15]837; AVX512VL-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]838; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1839; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm2840; AVX512VL-NEXT: vmovdqa {{.*#+}} ymm3 = [1,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768]841; AVX512VL-NEXT: vpmullw %ymm3, %ymm2, %ymm2842; AVX512VL-NEXT: vpmullw %ymm3, %ymm0, %ymm0843; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0844; AVX512VL-NEXT: vporq %zmm1, %zmm0, %zmm0845; AVX512VL-NEXT: retq846;847; AVX512BW-LABEL: constant_funnnel_v32i16:848; AVX512BW: # %bb.0:849; AVX512BW-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0850; AVX512BW-NEXT: vpsrlw $1, %zmm1, %zmm1851; AVX512BW-NEXT: vpsrlvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1852; AVX512BW-NEXT: vporq %zmm1, %zmm0, %zmm0853; AVX512BW-NEXT: retq854;855; AVX512VBMI2-LABEL: constant_funnnel_v32i16:856; AVX512VBMI2: # %bb.0:857; AVX512VBMI2-NEXT: vpshldvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0858; AVX512VBMI2-NEXT: retq859;860; AVX512VLBW-LABEL: constant_funnnel_v32i16:861; AVX512VLBW: # %bb.0:862; AVX512VLBW-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0863; AVX512VLBW-NEXT: vpsrlw $1, %zmm1, %zmm1864; AVX512VLBW-NEXT: vpsrlvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1865; AVX512VLBW-NEXT: vporq %zmm1, %zmm0, %zmm0866; AVX512VLBW-NEXT: retq867;868; AVX512VLVBMI2-LABEL: constant_funnnel_v32i16:869; AVX512VLVBMI2: # %bb.0:870; AVX512VLVBMI2-NEXT: vpshldvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0871; AVX512VLVBMI2-NEXT: retq872 %res = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>)873 ret <32 x i16> %res874}875 876define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {877; AVX512F-LABEL: constant_funnnel_v64i8:878; AVX512F: # %bb.0:879; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2880; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm3881; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11],ymm3[12],ymm2[12],ymm3[13],ymm2[13],ymm3[14],ymm2[14],ymm3[15],ymm2[15],ymm3[24],ymm2[24],ymm3[25],ymm2[25],ymm3[26],ymm2[26],ymm3[27],ymm2[27],ymm3[28],ymm2[28],ymm3[29],ymm2[29],ymm3[30],ymm2[30],ymm3[31],ymm2[31]882; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2]883; AVX512F-NEXT: # ymm5 = mem[0,1,0,1]884; AVX512F-NEXT: vpmullw %ymm5, %ymm4, %ymm4885; AVX512F-NEXT: vpsrlw $8, %ymm4, %ymm4886; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[4],ymm2[4],ymm3[5],ymm2[5],ymm3[6],ymm2[6],ymm3[7],ymm2[7],ymm3[16],ymm2[16],ymm3[17],ymm2[17],ymm3[18],ymm2[18],ymm3[19],ymm2[19],ymm3[20],ymm2[20],ymm3[21],ymm2[21],ymm3[22],ymm2[22],ymm3[23],ymm2[23]887; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]888; AVX512F-NEXT: # ymm3 = mem[0,1,0,1]889; AVX512F-NEXT: vpmullw %ymm3, %ymm2, %ymm2890; AVX512F-NEXT: vpsrlw $8, %ymm2, %ymm2891; AVX512F-NEXT: vpackuswb %ymm4, %ymm2, %ymm2892; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]893; AVX512F-NEXT: vpmullw %ymm5, %ymm4, %ymm4894; AVX512F-NEXT: vpsrlw $8, %ymm4, %ymm4895; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]896; AVX512F-NEXT: vpmullw %ymm3, %ymm0, %ymm0897; AVX512F-NEXT: vpsrlw $8, %ymm0, %ymm0898; AVX512F-NEXT: vpackuswb %ymm4, %ymm0, %ymm0899; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0900; AVX512F-NEXT: retq901;902; AVX512VL-LABEL: constant_funnnel_v64i8:903; AVX512VL: # %bb.0:904; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm2905; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm3906; AVX512VL-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11],ymm3[12],ymm2[12],ymm3[13],ymm2[13],ymm3[14],ymm2[14],ymm3[15],ymm2[15],ymm3[24],ymm2[24],ymm3[25],ymm2[25],ymm3[26],ymm2[26],ymm3[27],ymm2[27],ymm3[28],ymm2[28],ymm3[29],ymm2[29],ymm3[30],ymm2[30],ymm3[31],ymm2[31]907; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2]908; AVX512VL-NEXT: # ymm5 = mem[0,1,0,1]909; AVX512VL-NEXT: vpmullw %ymm5, %ymm4, %ymm4910; AVX512VL-NEXT: vpsrlw $8, %ymm4, %ymm4911; AVX512VL-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[4],ymm2[4],ymm3[5],ymm2[5],ymm3[6],ymm2[6],ymm3[7],ymm2[7],ymm3[16],ymm2[16],ymm3[17],ymm2[17],ymm3[18],ymm2[18],ymm3[19],ymm2[19],ymm3[20],ymm2[20],ymm3[21],ymm2[21],ymm3[22],ymm2[22],ymm3[23],ymm2[23]912; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]913; AVX512VL-NEXT: # ymm3 = mem[0,1,0,1]914; AVX512VL-NEXT: vpmullw %ymm3, %ymm2, %ymm2915; AVX512VL-NEXT: vpsrlw $8, %ymm2, %ymm2916; AVX512VL-NEXT: vpackuswb %ymm4, %ymm2, %ymm2917; AVX512VL-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]918; AVX512VL-NEXT: vpmullw %ymm5, %ymm4, %ymm4919; AVX512VL-NEXT: vpsrlw $8, %ymm4, %ymm4920; AVX512VL-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]921; AVX512VL-NEXT: vpmullw %ymm3, %ymm0, %ymm0922; AVX512VL-NEXT: vpsrlw $8, %ymm0, %ymm0923; AVX512VL-NEXT: vpackuswb %ymm4, %ymm0, %ymm0924; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0925; AVX512VL-NEXT: retq926;927; AVX512BW-LABEL: constant_funnnel_v64i8:928; AVX512BW: # %bb.0:929; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]930; AVX512BW-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2931; AVX512BW-NEXT: vpsrlw $8, %zmm2, %zmm2932; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]933; AVX512BW-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0934; AVX512BW-NEXT: vpsrlw $8, %zmm0, %zmm0935; AVX512BW-NEXT: vpackuswb %zmm2, %zmm0, %zmm0936; AVX512BW-NEXT: retq937;938; AVX512VBMI2-LABEL: constant_funnnel_v64i8:939; AVX512VBMI2: # %bb.0:940; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]941; AVX512VBMI2-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2942; AVX512VBMI2-NEXT: vpsrlw $8, %zmm2, %zmm2943; AVX512VBMI2-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]944; AVX512VBMI2-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0945; AVX512VBMI2-NEXT: vpsrlw $8, %zmm0, %zmm0946; AVX512VBMI2-NEXT: vpackuswb %zmm2, %zmm0, %zmm0947; AVX512VBMI2-NEXT: retq948;949; AVX512VLBW-LABEL: constant_funnnel_v64i8:950; AVX512VLBW: # %bb.0:951; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]952; AVX512VLBW-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2953; AVX512VLBW-NEXT: vpsrlw $8, %zmm2, %zmm2954; AVX512VLBW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]955; AVX512VLBW-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0956; AVX512VLBW-NEXT: vpsrlw $8, %zmm0, %zmm0957; AVX512VLBW-NEXT: vpackuswb %zmm2, %zmm0, %zmm0958; AVX512VLBW-NEXT: retq959;960; AVX512VLVBMI2-LABEL: constant_funnnel_v64i8:961; AVX512VLVBMI2: # %bb.0:962; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]963; AVX512VLVBMI2-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm2964; AVX512VLVBMI2-NEXT: vpsrlw $8, %zmm2, %zmm2965; AVX512VLVBMI2-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]966; AVX512VLVBMI2-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0967; AVX512VLVBMI2-NEXT: vpsrlw $8, %zmm0, %zmm0968; AVX512VLVBMI2-NEXT: vpackuswb %zmm2, %zmm0, %zmm0969; AVX512VLVBMI2-NEXT: retq970 %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1>)971 ret <64 x i8> %res972}973 974;975; Uniform Constant Shifts976;977 978define <8 x i64> @splatconstant_funnnel_v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {979; AVX512F-LABEL: splatconstant_funnnel_v8i64:980; AVX512F: # %bb.0:981; AVX512F-NEXT: vpsrlq $50, %zmm1, %zmm1982; AVX512F-NEXT: vpsllq $14, %zmm0, %zmm0983; AVX512F-NEXT: vporq %zmm1, %zmm0, %zmm0984; AVX512F-NEXT: retq985;986; AVX512VL-LABEL: splatconstant_funnnel_v8i64:987; AVX512VL: # %bb.0:988; AVX512VL-NEXT: vpsrlq $50, %zmm1, %zmm1989; AVX512VL-NEXT: vpsllq $14, %zmm0, %zmm0990; AVX512VL-NEXT: vporq %zmm1, %zmm0, %zmm0991; AVX512VL-NEXT: retq992;993; AVX512BW-LABEL: splatconstant_funnnel_v8i64:994; AVX512BW: # %bb.0:995; AVX512BW-NEXT: vpsrlq $50, %zmm1, %zmm1996; AVX512BW-NEXT: vpsllq $14, %zmm0, %zmm0997; AVX512BW-NEXT: vporq %zmm1, %zmm0, %zmm0998; AVX512BW-NEXT: retq999;1000; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i64:1001; AVX512VBMI2: # %bb.0:1002; AVX512VBMI2-NEXT: vpshldq $14, %zmm1, %zmm0, %zmm01003; AVX512VBMI2-NEXT: retq1004;1005; AVX512VLBW-LABEL: splatconstant_funnnel_v8i64:1006; AVX512VLBW: # %bb.0:1007; AVX512VLBW-NEXT: vpsrlq $50, %zmm1, %zmm11008; AVX512VLBW-NEXT: vpsllq $14, %zmm0, %zmm01009; AVX512VLBW-NEXT: vporq %zmm1, %zmm0, %zmm01010; AVX512VLBW-NEXT: retq1011;1012; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v8i64:1013; AVX512VLVBMI2: # %bb.0:1014; AVX512VLVBMI2-NEXT: vpshldq $14, %zmm1, %zmm0, %zmm01015; AVX512VLVBMI2-NEXT: retq1016 %res = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> <i64 14, i64 14, i64 14, i64 14, i64 14, i64 14, i64 14, i64 14>)1017 ret <8 x i64> %res1018}1019 1020define <16 x i32> @splatconstant_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {1021; AVX512F-LABEL: splatconstant_funnnel_v16i32:1022; AVX512F: # %bb.0:1023; AVX512F-NEXT: vpsrld $28, %zmm1, %zmm11024; AVX512F-NEXT: vpslld $4, %zmm0, %zmm01025; AVX512F-NEXT: vpord %zmm1, %zmm0, %zmm01026; AVX512F-NEXT: retq1027;1028; AVX512VL-LABEL: splatconstant_funnnel_v16i32:1029; AVX512VL: # %bb.0:1030; AVX512VL-NEXT: vpsrld $28, %zmm1, %zmm11031; AVX512VL-NEXT: vpslld $4, %zmm0, %zmm01032; AVX512VL-NEXT: vpord %zmm1, %zmm0, %zmm01033; AVX512VL-NEXT: retq1034;1035; AVX512BW-LABEL: splatconstant_funnnel_v16i32:1036; AVX512BW: # %bb.0:1037; AVX512BW-NEXT: vpsrld $28, %zmm1, %zmm11038; AVX512BW-NEXT: vpslld $4, %zmm0, %zmm01039; AVX512BW-NEXT: vpord %zmm1, %zmm0, %zmm01040; AVX512BW-NEXT: retq1041;1042; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i32:1043; AVX512VBMI2: # %bb.0:1044; AVX512VBMI2-NEXT: vpshldd $4, %zmm1, %zmm0, %zmm01045; AVX512VBMI2-NEXT: retq1046;1047; AVX512VLBW-LABEL: splatconstant_funnnel_v16i32:1048; AVX512VLBW: # %bb.0:1049; AVX512VLBW-NEXT: vpsrld $28, %zmm1, %zmm11050; AVX512VLBW-NEXT: vpslld $4, %zmm0, %zmm01051; AVX512VLBW-NEXT: vpord %zmm1, %zmm0, %zmm01052; AVX512VLBW-NEXT: retq1053;1054; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i32:1055; AVX512VLVBMI2: # %bb.0:1056; AVX512VLVBMI2-NEXT: vpshldd $4, %zmm1, %zmm0, %zmm01057; AVX512VLVBMI2-NEXT: retq1058 %res = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>)1059 ret <16 x i32> %res1060}1061 1062define <32 x i16> @splatconstant_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y) nounwind {1063; AVX512F-LABEL: splatconstant_funnnel_v32i16:1064; AVX512F: # %bb.0:1065; AVX512F-NEXT: vpsrlw $9, %ymm1, %ymm21066; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm11067; AVX512F-NEXT: vpsrlw $9, %ymm1, %ymm11068; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm11069; AVX512F-NEXT: vpsllw $7, %ymm0, %ymm21070; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm01071; AVX512F-NEXT: vpsllw $7, %ymm0, %ymm01072; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm01073; AVX512F-NEXT: vporq %zmm1, %zmm0, %zmm01074; AVX512F-NEXT: retq1075;1076; AVX512VL-LABEL: splatconstant_funnnel_v32i16:1077; AVX512VL: # %bb.0:1078; AVX512VL-NEXT: vpsrlw $9, %ymm1, %ymm21079; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm11080; AVX512VL-NEXT: vpsrlw $9, %ymm1, %ymm11081; AVX512VL-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm11082; AVX512VL-NEXT: vpsllw $7, %ymm0, %ymm21083; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm01084; AVX512VL-NEXT: vpsllw $7, %ymm0, %ymm01085; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm01086; AVX512VL-NEXT: vporq %zmm1, %zmm0, %zmm01087; AVX512VL-NEXT: retq1088;1089; AVX512BW-LABEL: splatconstant_funnnel_v32i16:1090; AVX512BW: # %bb.0:1091; AVX512BW-NEXT: vpsrlw $9, %zmm1, %zmm11092; AVX512BW-NEXT: vpsllw $7, %zmm0, %zmm01093; AVX512BW-NEXT: vporq %zmm1, %zmm0, %zmm01094; AVX512BW-NEXT: retq1095;1096; AVX512VBMI2-LABEL: splatconstant_funnnel_v32i16:1097; AVX512VBMI2: # %bb.0:1098; AVX512VBMI2-NEXT: vpshldw $7, %zmm1, %zmm0, %zmm01099; AVX512VBMI2-NEXT: retq1100;1101; AVX512VLBW-LABEL: splatconstant_funnnel_v32i16:1102; AVX512VLBW: # %bb.0:1103; AVX512VLBW-NEXT: vpsrlw $9, %zmm1, %zmm11104; AVX512VLBW-NEXT: vpsllw $7, %zmm0, %zmm01105; AVX512VLBW-NEXT: vporq %zmm1, %zmm0, %zmm01106; AVX512VLBW-NEXT: retq1107;1108; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v32i16:1109; AVX512VLVBMI2: # %bb.0:1110; AVX512VLVBMI2-NEXT: vpshldw $7, %zmm1, %zmm0, %zmm01111; AVX512VLVBMI2-NEXT: retq1112 %res = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>)1113 ret <32 x i16> %res1114}1115 1116define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {1117; AVX512F-LABEL: splatconstant_funnnel_v64i8:1118; AVX512F: # %bb.0:1119; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm21120; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm01121; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm01122; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm21123; AVX512F-NEXT: vpsrlw $4, %ymm1, %ymm01124; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm11125; AVX512F-NEXT: vpsrlw $4, %ymm1, %ymm11126; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm01127; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))1128; AVX512F-NEXT: retq1129;1130; AVX512VL-LABEL: splatconstant_funnnel_v64i8:1131; AVX512VL: # %bb.0:1132; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm21133; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm01134; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm01135; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm21136; AVX512VL-NEXT: vpsrlw $4, %ymm1, %ymm01137; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm11138; AVX512VL-NEXT: vpsrlw $4, %ymm1, %ymm11139; AVX512VL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm01140; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))1141; AVX512VL-NEXT: retq1142;1143; AVX512BW-LABEL: splatconstant_funnnel_v64i8:1144; AVX512BW: # %bb.0:1145; AVX512BW-NEXT: vpsllw $4, %zmm0, %zmm21146; AVX512BW-NEXT: vpsrlw $4, %zmm1, %zmm01147; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))1148; AVX512BW-NEXT: retq1149;1150; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8:1151; AVX512VBMI2: # %bb.0:1152; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm21153; AVX512VBMI2-NEXT: vpsrlw $4, %zmm1, %zmm01154; AVX512VBMI2-NEXT: vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))1155; AVX512VBMI2-NEXT: retq1156;1157; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8:1158; AVX512VLBW: # %bb.0:1159; AVX512VLBW-NEXT: vpsllw $4, %zmm0, %zmm21160; AVX512VLBW-NEXT: vpsrlw $4, %zmm1, %zmm01161; AVX512VLBW-NEXT: vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))1162; AVX512VLBW-NEXT: retq1163;1164; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8:1165; AVX512VLVBMI2: # %bb.0:1166; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm21167; AVX512VLVBMI2-NEXT: vpsrlw $4, %zmm1, %zmm01168; AVX512VLVBMI2-NEXT: vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm2))1169; AVX512VLVBMI2-NEXT: retq1170 %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)1171 ret <64 x i8> %res1172}1173