769 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI27; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI28 9declare <8 x i64> @llvm.fshr.v8i64(<8 x i64>, <8 x i64>, <8 x i64>)10declare <16 x i32> @llvm.fshr.v16i32(<16 x i32>, <16 x i32>, <16 x i32>)11declare <32 x i16> @llvm.fshr.v32i16(<32 x i16>, <32 x i16>, <32 x i16>)12declare <64 x i8> @llvm.fshr.v64i8(<64 x i8>, <64 x i8>, <64 x i8>)13 14;15; Variable Shifts16;17 18define <8 x i64> @var_funnnel_v8i64(<8 x i64> %x, <8 x i64> %amt) nounwind {19; AVX512-LABEL: var_funnnel_v8i64:20; AVX512: # %bb.0:21; AVX512-NEXT: vprorvq %zmm1, %zmm0, %zmm022; AVX512-NEXT: retq23 %res = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x, <8 x i64> %x, <8 x i64> %amt)24 ret <8 x i64> %res25}26 27define <16 x i32> @var_funnnel_v16i32(<16 x i32> %x, <16 x i32> %amt) nounwind {28; AVX512-LABEL: var_funnnel_v16i32:29; AVX512: # %bb.0:30; AVX512-NEXT: vprorvd %zmm1, %zmm0, %zmm031; AVX512-NEXT: retq32 %res = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x, <16 x i32> %x, <16 x i32> %amt)33 ret <16 x i32> %res34}35 36define <32 x i16> @var_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounwind {37; AVX512F-LABEL: var_funnnel_v32i16:38; AVX512F: # %bb.0:39; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm240; AVX512F-NEXT: vpbroadcastw {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]41; AVX512F-NEXT: vpand %ymm3, %ymm2, %ymm242; AVX512F-NEXT: vpxor %xmm4, %xmm4, %xmm443; AVX512F-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm2[4],ymm4[4],ymm2[5],ymm4[5],ymm2[6],ymm4[6],ymm2[7],ymm4[7],ymm2[12],ymm4[12],ymm2[13],ymm4[13],ymm2[14],ymm4[14],ymm2[15],ymm4[15]44; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm645; AVX512F-NEXT: vpunpckhwd {{.*#+}} ymm7 = ymm6[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]46; AVX512F-NEXT: vpsrlvd %ymm5, %ymm7, %ymm547; AVX512F-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7],ymm5[8],ymm4[9],ymm5[10],ymm4[11],ymm5[12],ymm4[13],ymm5[14],ymm4[15]48; AVX512F-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[8],ymm4[8],ymm2[9],ymm4[9],ymm2[10],ymm4[10],ymm2[11],ymm4[11]49; AVX512F-NEXT: vpunpcklwd {{.*#+}} ymm6 = ymm6[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]50; AVX512F-NEXT: vpsrlvd %ymm2, %ymm6, %ymm251; AVX512F-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0],ymm4[1],ymm2[2],ymm4[3],ymm2[4],ymm4[5],ymm2[6],ymm4[7],ymm2[8],ymm4[9],ymm2[10],ymm4[11],ymm2[12],ymm4[13],ymm2[14],ymm4[15]52; AVX512F-NEXT: vpackusdw %ymm5, %ymm2, %ymm253; AVX512F-NEXT: vpand %ymm3, %ymm1, %ymm154; AVX512F-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm4[4],ymm1[5],ymm4[5],ymm1[6],ymm4[6],ymm1[7],ymm4[7],ymm1[12],ymm4[12],ymm1[13],ymm4[13],ymm1[14],ymm4[14],ymm1[15],ymm4[15]55; AVX512F-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]56; AVX512F-NEXT: vpsrlvd %ymm3, %ymm5, %ymm357; AVX512F-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7],ymm3[8],ymm4[9],ymm3[10],ymm4[11],ymm3[12],ymm4[13],ymm3[14],ymm4[15]58; AVX512F-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm4[0],ymm1[1],ymm4[1],ymm1[2],ymm4[2],ymm1[3],ymm4[3],ymm1[8],ymm4[8],ymm1[9],ymm4[9],ymm1[10],ymm4[10],ymm1[11],ymm4[11]59; AVX512F-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]60; AVX512F-NEXT: vpsrlvd %ymm1, %ymm0, %ymm061; AVX512F-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm4[1],ymm0[2],ymm4[3],ymm0[4],ymm4[5],ymm0[6],ymm4[7],ymm0[8],ymm4[9],ymm0[10],ymm4[11],ymm0[12],ymm4[13],ymm0[14],ymm4[15]62; AVX512F-NEXT: vpackusdw %ymm3, %ymm0, %ymm063; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm064; AVX512F-NEXT: retq65;66; AVX512VL-LABEL: var_funnnel_v32i16:67; AVX512VL: # %bb.0:68; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm269; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]70; AVX512VL-NEXT: vpand %ymm3, %ymm2, %ymm271; AVX512VL-NEXT: vpxor %xmm4, %xmm4, %xmm472; AVX512VL-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm2[4],ymm4[4],ymm2[5],ymm4[5],ymm2[6],ymm4[6],ymm2[7],ymm4[7],ymm2[12],ymm4[12],ymm2[13],ymm4[13],ymm2[14],ymm4[14],ymm2[15],ymm4[15]73; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm674; AVX512VL-NEXT: vpunpckhwd {{.*#+}} ymm7 = ymm6[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]75; AVX512VL-NEXT: vpsrlvd %ymm5, %ymm7, %ymm576; AVX512VL-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7],ymm5[8],ymm4[9],ymm5[10],ymm4[11],ymm5[12],ymm4[13],ymm5[14],ymm4[15]77; AVX512VL-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[8],ymm4[8],ymm2[9],ymm4[9],ymm2[10],ymm4[10],ymm2[11],ymm4[11]78; AVX512VL-NEXT: vpunpcklwd {{.*#+}} ymm6 = ymm6[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]79; AVX512VL-NEXT: vpsrlvd %ymm2, %ymm6, %ymm280; AVX512VL-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0],ymm4[1],ymm2[2],ymm4[3],ymm2[4],ymm4[5],ymm2[6],ymm4[7],ymm2[8],ymm4[9],ymm2[10],ymm4[11],ymm2[12],ymm4[13],ymm2[14],ymm4[15]81; AVX512VL-NEXT: vpackusdw %ymm5, %ymm2, %ymm282; AVX512VL-NEXT: vpand %ymm3, %ymm1, %ymm183; AVX512VL-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm4[4],ymm1[5],ymm4[5],ymm1[6],ymm4[6],ymm1[7],ymm4[7],ymm1[12],ymm4[12],ymm1[13],ymm4[13],ymm1[14],ymm4[14],ymm1[15],ymm4[15]84; AVX512VL-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]85; AVX512VL-NEXT: vpsrlvd %ymm3, %ymm5, %ymm386; AVX512VL-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7],ymm3[8],ymm4[9],ymm3[10],ymm4[11],ymm3[12],ymm4[13],ymm3[14],ymm4[15]87; AVX512VL-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm4[0],ymm1[1],ymm4[1],ymm1[2],ymm4[2],ymm1[3],ymm4[3],ymm1[8],ymm4[8],ymm1[9],ymm4[9],ymm1[10],ymm4[10],ymm1[11],ymm4[11]88; AVX512VL-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]89; AVX512VL-NEXT: vpsrlvd %ymm1, %ymm0, %ymm090; AVX512VL-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm4[1],ymm0[2],ymm4[3],ymm0[4],ymm4[5],ymm0[6],ymm4[7],ymm0[8],ymm4[9],ymm0[10],ymm4[11],ymm0[12],ymm4[13],ymm0[14],ymm4[15]91; AVX512VL-NEXT: vpackusdw %ymm3, %ymm0, %ymm092; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm093; AVX512VL-NEXT: retq94;95; AVX512BW-LABEL: var_funnnel_v32i16:96; AVX512BW: # %bb.0:97; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm198; AVX512BW-NEXT: vpsrlvw %zmm1, %zmm0, %zmm299; AVX512BW-NEXT: vpbroadcastw {{.*#+}} zmm3 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]100; AVX512BW-NEXT: vpsubw %zmm1, %zmm3, %zmm1101; AVX512BW-NEXT: vpsllvw %zmm1, %zmm0, %zmm0102; AVX512BW-NEXT: vporq %zmm0, %zmm2, %zmm0103; AVX512BW-NEXT: retq104;105; AVX512VLBW-LABEL: var_funnnel_v32i16:106; AVX512VLBW: # %bb.0:107; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1108; AVX512VLBW-NEXT: vpsrlvw %zmm1, %zmm0, %zmm2109; AVX512VLBW-NEXT: vpbroadcastw {{.*#+}} zmm3 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]110; AVX512VLBW-NEXT: vpsubw %zmm1, %zmm3, %zmm1111; AVX512VLBW-NEXT: vpsllvw %zmm1, %zmm0, %zmm0112; AVX512VLBW-NEXT: vporq %zmm0, %zmm2, %zmm0113; AVX512VLBW-NEXT: retq114;115; AVX512VBMI2-LABEL: var_funnnel_v32i16:116; AVX512VBMI2: # %bb.0:117; AVX512VBMI2-NEXT: vpshrdvw %zmm1, %zmm0, %zmm0118; AVX512VBMI2-NEXT: retq119;120; AVX512VLVBMI2-LABEL: var_funnnel_v32i16:121; AVX512VLVBMI2: # %bb.0:122; AVX512VLVBMI2-NEXT: vpshrdvw %zmm1, %zmm0, %zmm0123; AVX512VLVBMI2-NEXT: retq124 %res = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x, <32 x i16> %x, <32 x i16> %amt)125 ret <32 x i16> %res126}127 128define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {129; AVX512F-LABEL: var_funnnel_v64i8:130; AVX512F: # %bb.0:131; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2132; AVX512F-NEXT: vpsllw $4, %ymm2, %ymm3133; AVX512F-NEXT: vpsrlw $4, %ymm2, %ymm4134; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm5 = [252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135]135; AVX512F-NEXT: vpternlogd {{.*#+}} zmm4 = zmm3 ^ (zmm5 & (zmm4 ^ zmm3))136; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm3137; AVX512F-NEXT: vpsllw $5, %ymm3, %ymm3138; AVX512F-NEXT: vpblendvb %ymm3, %ymm4, %ymm2, %ymm2139; AVX512F-NEXT: vpsllw $6, %ymm2, %ymm4140; AVX512F-NEXT: vpsrlw $2, %ymm2, %ymm6141; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm7 = [1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567]142; AVX512F-NEXT: vpternlogd {{.*#+}} zmm6 = zmm4 ^ (zmm7 & (zmm6 ^ zmm4))143; AVX512F-NEXT: vpaddb %ymm3, %ymm3, %ymm3144; AVX512F-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2145; AVX512F-NEXT: vpsllw $7, %ymm2, %ymm4146; AVX512F-NEXT: vpsrlw $1, %ymm2, %ymm6147; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm8 = [2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143]148; AVX512F-NEXT: vpternlogd {{.*#+}} zmm6 = zmm4 ^ (zmm8 & (zmm6 ^ zmm4))149; AVX512F-NEXT: vpaddb %ymm3, %ymm3, %ymm3150; AVX512F-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2151; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm3152; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm4153; AVX512F-NEXT: vpternlogd {{.*#+}} zmm4 = zmm3 ^ (zmm5 & (zmm4 ^ zmm3))154; AVX512F-NEXT: vpsllw $5, %ymm1, %ymm1155; AVX512F-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0156; AVX512F-NEXT: vpsllw $6, %ymm0, %ymm3157; AVX512F-NEXT: vpsrlw $2, %ymm0, %ymm4158; AVX512F-NEXT: vpternlogd {{.*#+}} zmm4 = zmm3 ^ (zmm7 & (zmm4 ^ zmm3))159; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1160; AVX512F-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0161; AVX512F-NEXT: vpsllw $7, %ymm0, %ymm3162; AVX512F-NEXT: vpsrlw $1, %ymm0, %ymm4163; AVX512F-NEXT: vpternlogd {{.*#+}} zmm4 = zmm3 ^ (zmm8 & (zmm4 ^ zmm3))164; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1165; AVX512F-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0166; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0167; AVX512F-NEXT: retq168;169; AVX512VL-LABEL: var_funnnel_v64i8:170; AVX512VL: # %bb.0:171; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm2172; AVX512VL-NEXT: vpsllw $4, %ymm2, %ymm3173; AVX512VL-NEXT: vpsrlw $4, %ymm2, %ymm4174; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm5 = [252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135]175; AVX512VL-NEXT: vpternlogd {{.*#+}} ymm4 = ymm3 ^ (ymm5 & (ymm4 ^ ymm3))176; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm3177; AVX512VL-NEXT: vpsllw $5, %ymm3, %ymm3178; AVX512VL-NEXT: vpblendvb %ymm3, %ymm4, %ymm2, %ymm2179; AVX512VL-NEXT: vpsllw $6, %ymm2, %ymm4180; AVX512VL-NEXT: vpsrlw $2, %ymm2, %ymm6181; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm7 = [1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567]182; AVX512VL-NEXT: vpternlogd {{.*#+}} ymm6 = ymm4 ^ (ymm7 & (ymm6 ^ ymm4))183; AVX512VL-NEXT: vpaddb %ymm3, %ymm3, %ymm3184; AVX512VL-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2185; AVX512VL-NEXT: vpsllw $7, %ymm2, %ymm4186; AVX512VL-NEXT: vpsrlw $1, %ymm2, %ymm6187; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm8 = [2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143]188; AVX512VL-NEXT: vpternlogd {{.*#+}} ymm6 = ymm4 ^ (ymm8 & (ymm6 ^ ymm4))189; AVX512VL-NEXT: vpaddb %ymm3, %ymm3, %ymm3190; AVX512VL-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2191; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm3192; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm4193; AVX512VL-NEXT: vpternlogd {{.*#+}} ymm4 = ymm3 ^ (ymm5 & (ymm4 ^ ymm3))194; AVX512VL-NEXT: vpsllw $5, %ymm1, %ymm1195; AVX512VL-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0196; AVX512VL-NEXT: vpsllw $6, %ymm0, %ymm3197; AVX512VL-NEXT: vpsrlw $2, %ymm0, %ymm4198; AVX512VL-NEXT: vpternlogd {{.*#+}} ymm4 = ymm3 ^ (ymm7 & (ymm4 ^ ymm3))199; AVX512VL-NEXT: vpaddb %ymm1, %ymm1, %ymm1200; AVX512VL-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0201; AVX512VL-NEXT: vpsllw $7, %ymm0, %ymm3202; AVX512VL-NEXT: vpsrlw $1, %ymm0, %ymm4203; AVX512VL-NEXT: vpternlogd {{.*#+}} ymm4 = ymm3 ^ (ymm8 & (ymm4 ^ ymm3))204; AVX512VL-NEXT: vpaddb %ymm1, %ymm1, %ymm1205; AVX512VL-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0206; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0207; AVX512VL-NEXT: retq208;209; AVX512BW-LABEL: var_funnnel_v64i8:210; AVX512BW: # %bb.0:211; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1212; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2213; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[12],zmm2[12],zmm1[13],zmm2[13],zmm1[14],zmm2[14],zmm1[15],zmm2[15],zmm1[24],zmm2[24],zmm1[25],zmm2[25],zmm1[26],zmm2[26],zmm1[27],zmm2[27],zmm1[28],zmm2[28],zmm1[29],zmm2[29],zmm1[30],zmm2[30],zmm1[31],zmm2[31],zmm1[40],zmm2[40],zmm1[41],zmm2[41],zmm1[42],zmm2[42],zmm1[43],zmm2[43],zmm1[44],zmm2[44],zmm1[45],zmm2[45],zmm1[46],zmm2[46],zmm1[47],zmm2[47],zmm1[56],zmm2[56],zmm1[57],zmm2[57],zmm1[58],zmm2[58],zmm1[59],zmm2[59],zmm1[60],zmm2[60],zmm1[61],zmm2[61],zmm1[62],zmm2[62],zmm1[63],zmm2[63]214; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]215; AVX512BW-NEXT: vpsrlvw %zmm3, %zmm4, %zmm3216; AVX512BW-NEXT: vpbroadcastw {{.*#+}} zmm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]217; AVX512BW-NEXT: vpandq %zmm4, %zmm3, %zmm3218; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm2[0],zmm1[1],zmm2[1],zmm1[2],zmm2[2],zmm1[3],zmm2[3],zmm1[4],zmm2[4],zmm1[5],zmm2[5],zmm1[6],zmm2[6],zmm1[7],zmm2[7],zmm1[16],zmm2[16],zmm1[17],zmm2[17],zmm1[18],zmm2[18],zmm1[19],zmm2[19],zmm1[20],zmm2[20],zmm1[21],zmm2[21],zmm1[22],zmm2[22],zmm1[23],zmm2[23],zmm1[32],zmm2[32],zmm1[33],zmm2[33],zmm1[34],zmm2[34],zmm1[35],zmm2[35],zmm1[36],zmm2[36],zmm1[37],zmm2[37],zmm1[38],zmm2[38],zmm1[39],zmm2[39],zmm1[48],zmm2[48],zmm1[49],zmm2[49],zmm1[50],zmm2[50],zmm1[51],zmm2[51],zmm1[52],zmm2[52],zmm1[53],zmm2[53],zmm1[54],zmm2[54],zmm1[55],zmm2[55]219; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]220; AVX512BW-NEXT: vpsrlvw %zmm1, %zmm0, %zmm0221; AVX512BW-NEXT: vpandq %zmm4, %zmm0, %zmm0222; AVX512BW-NEXT: vpackuswb %zmm3, %zmm0, %zmm0223; AVX512BW-NEXT: retq224;225; AVX512VLBW-LABEL: var_funnnel_v64i8:226; AVX512VLBW: # %bb.0:227; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1228; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2229; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[12],zmm2[12],zmm1[13],zmm2[13],zmm1[14],zmm2[14],zmm1[15],zmm2[15],zmm1[24],zmm2[24],zmm1[25],zmm2[25],zmm1[26],zmm2[26],zmm1[27],zmm2[27],zmm1[28],zmm2[28],zmm1[29],zmm2[29],zmm1[30],zmm2[30],zmm1[31],zmm2[31],zmm1[40],zmm2[40],zmm1[41],zmm2[41],zmm1[42],zmm2[42],zmm1[43],zmm2[43],zmm1[44],zmm2[44],zmm1[45],zmm2[45],zmm1[46],zmm2[46],zmm1[47],zmm2[47],zmm1[56],zmm2[56],zmm1[57],zmm2[57],zmm1[58],zmm2[58],zmm1[59],zmm2[59],zmm1[60],zmm2[60],zmm1[61],zmm2[61],zmm1[62],zmm2[62],zmm1[63],zmm2[63]230; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]231; AVX512VLBW-NEXT: vpsrlvw %zmm3, %zmm4, %zmm3232; AVX512VLBW-NEXT: vpbroadcastw {{.*#+}} zmm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]233; AVX512VLBW-NEXT: vpandq %zmm4, %zmm3, %zmm3234; AVX512VLBW-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm2[0],zmm1[1],zmm2[1],zmm1[2],zmm2[2],zmm1[3],zmm2[3],zmm1[4],zmm2[4],zmm1[5],zmm2[5],zmm1[6],zmm2[6],zmm1[7],zmm2[7],zmm1[16],zmm2[16],zmm1[17],zmm2[17],zmm1[18],zmm2[18],zmm1[19],zmm2[19],zmm1[20],zmm2[20],zmm1[21],zmm2[21],zmm1[22],zmm2[22],zmm1[23],zmm2[23],zmm1[32],zmm2[32],zmm1[33],zmm2[33],zmm1[34],zmm2[34],zmm1[35],zmm2[35],zmm1[36],zmm2[36],zmm1[37],zmm2[37],zmm1[38],zmm2[38],zmm1[39],zmm2[39],zmm1[48],zmm2[48],zmm1[49],zmm2[49],zmm1[50],zmm2[50],zmm1[51],zmm2[51],zmm1[52],zmm2[52],zmm1[53],zmm2[53],zmm1[54],zmm2[54],zmm1[55],zmm2[55]235; AVX512VLBW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]236; AVX512VLBW-NEXT: vpsrlvw %zmm1, %zmm0, %zmm0237; AVX512VLBW-NEXT: vpandq %zmm4, %zmm0, %zmm0238; AVX512VLBW-NEXT: vpackuswb %zmm3, %zmm0, %zmm0239; AVX512VLBW-NEXT: retq240;241; AVX512VBMI2-LABEL: var_funnnel_v64i8:242; AVX512VBMI2: # %bb.0:243; AVX512VBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1244; AVX512VBMI2-NEXT: vpxor %xmm2, %xmm2, %xmm2245; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[12],zmm2[12],zmm1[13],zmm2[13],zmm1[14],zmm2[14],zmm1[15],zmm2[15],zmm1[24],zmm2[24],zmm1[25],zmm2[25],zmm1[26],zmm2[26],zmm1[27],zmm2[27],zmm1[28],zmm2[28],zmm1[29],zmm2[29],zmm1[30],zmm2[30],zmm1[31],zmm2[31],zmm1[40],zmm2[40],zmm1[41],zmm2[41],zmm1[42],zmm2[42],zmm1[43],zmm2[43],zmm1[44],zmm2[44],zmm1[45],zmm2[45],zmm1[46],zmm2[46],zmm1[47],zmm2[47],zmm1[56],zmm2[56],zmm1[57],zmm2[57],zmm1[58],zmm2[58],zmm1[59],zmm2[59],zmm1[60],zmm2[60],zmm1[61],zmm2[61],zmm1[62],zmm2[62],zmm1[63],zmm2[63]246; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]247; AVX512VBMI2-NEXT: vpsrlvw %zmm3, %zmm4, %zmm3248; AVX512VBMI2-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm2[0],zmm1[1],zmm2[1],zmm1[2],zmm2[2],zmm1[3],zmm2[3],zmm1[4],zmm2[4],zmm1[5],zmm2[5],zmm1[6],zmm2[6],zmm1[7],zmm2[7],zmm1[16],zmm2[16],zmm1[17],zmm2[17],zmm1[18],zmm2[18],zmm1[19],zmm2[19],zmm1[20],zmm2[20],zmm1[21],zmm2[21],zmm1[22],zmm2[22],zmm1[23],zmm2[23],zmm1[32],zmm2[32],zmm1[33],zmm2[33],zmm1[34],zmm2[34],zmm1[35],zmm2[35],zmm1[36],zmm2[36],zmm1[37],zmm2[37],zmm1[38],zmm2[38],zmm1[39],zmm2[39],zmm1[48],zmm2[48],zmm1[49],zmm2[49],zmm1[50],zmm2[50],zmm1[51],zmm2[51],zmm1[52],zmm2[52],zmm1[53],zmm2[53],zmm1[54],zmm2[54],zmm1[55],zmm2[55]249; AVX512VBMI2-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]250; AVX512VBMI2-NEXT: vpsrlvw %zmm1, %zmm0, %zmm1251; AVX512VBMI2-NEXT: vmovdqa64 {{.*#+}} zmm0 = [0,2,4,6,8,10,12,14,64,66,68,70,72,74,76,78,16,18,20,22,24,26,28,30,80,82,84,86,88,90,92,94,32,34,36,38,40,42,44,46,96,98,100,102,104,106,108,110,48,50,52,54,56,58,60,62,112,114,116,118,120,122,124,126]252; AVX512VBMI2-NEXT: vpermi2b %zmm3, %zmm1, %zmm0253; AVX512VBMI2-NEXT: retq254;255; AVX512VLVBMI2-LABEL: var_funnnel_v64i8:256; AVX512VLVBMI2: # %bb.0:257; AVX512VLVBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1258; AVX512VLVBMI2-NEXT: vpxor %xmm2, %xmm2, %xmm2259; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[12],zmm2[12],zmm1[13],zmm2[13],zmm1[14],zmm2[14],zmm1[15],zmm2[15],zmm1[24],zmm2[24],zmm1[25],zmm2[25],zmm1[26],zmm2[26],zmm1[27],zmm2[27],zmm1[28],zmm2[28],zmm1[29],zmm2[29],zmm1[30],zmm2[30],zmm1[31],zmm2[31],zmm1[40],zmm2[40],zmm1[41],zmm2[41],zmm1[42],zmm2[42],zmm1[43],zmm2[43],zmm1[44],zmm2[44],zmm1[45],zmm2[45],zmm1[46],zmm2[46],zmm1[47],zmm2[47],zmm1[56],zmm2[56],zmm1[57],zmm2[57],zmm1[58],zmm2[58],zmm1[59],zmm2[59],zmm1[60],zmm2[60],zmm1[61],zmm2[61],zmm1[62],zmm2[62],zmm1[63],zmm2[63]260; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]261; AVX512VLVBMI2-NEXT: vpsrlvw %zmm3, %zmm4, %zmm3262; AVX512VLVBMI2-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm2[0],zmm1[1],zmm2[1],zmm1[2],zmm2[2],zmm1[3],zmm2[3],zmm1[4],zmm2[4],zmm1[5],zmm2[5],zmm1[6],zmm2[6],zmm1[7],zmm2[7],zmm1[16],zmm2[16],zmm1[17],zmm2[17],zmm1[18],zmm2[18],zmm1[19],zmm2[19],zmm1[20],zmm2[20],zmm1[21],zmm2[21],zmm1[22],zmm2[22],zmm1[23],zmm2[23],zmm1[32],zmm2[32],zmm1[33],zmm2[33],zmm1[34],zmm2[34],zmm1[35],zmm2[35],zmm1[36],zmm2[36],zmm1[37],zmm2[37],zmm1[38],zmm2[38],zmm1[39],zmm2[39],zmm1[48],zmm2[48],zmm1[49],zmm2[49],zmm1[50],zmm2[50],zmm1[51],zmm2[51],zmm1[52],zmm2[52],zmm1[53],zmm2[53],zmm1[54],zmm2[54],zmm1[55],zmm2[55]263; AVX512VLVBMI2-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]264; AVX512VLVBMI2-NEXT: vpsrlvw %zmm1, %zmm0, %zmm1265; AVX512VLVBMI2-NEXT: vmovdqa64 {{.*#+}} zmm0 = [0,2,4,6,8,10,12,14,64,66,68,70,72,74,76,78,16,18,20,22,24,26,28,30,80,82,84,86,88,90,92,94,32,34,36,38,40,42,44,46,96,98,100,102,104,106,108,110,48,50,52,54,56,58,60,62,112,114,116,118,120,122,124,126]266; AVX512VLVBMI2-NEXT: vpermi2b %zmm3, %zmm1, %zmm0267; AVX512VLVBMI2-NEXT: retq268 %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> %amt)269 ret <64 x i8> %res270}271 272;273; Uniform Variable Shifts274;275 276define <8 x i64> @splatvar_funnnel_v8i64(<8 x i64> %x, <8 x i64> %amt) nounwind {277; AVX512-LABEL: splatvar_funnnel_v8i64:278; AVX512: # %bb.0:279; AVX512-NEXT: vpbroadcastq %xmm1, %zmm1280; AVX512-NEXT: vprorvq %zmm1, %zmm0, %zmm0281; AVX512-NEXT: retq282 %splat = shufflevector <8 x i64> %amt, <8 x i64> undef, <8 x i32> zeroinitializer283 %res = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x, <8 x i64> %x, <8 x i64> %splat)284 ret <8 x i64> %res285}286 287define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %amt) nounwind {288; AVX512-LABEL: splatvar_funnnel_v16i32:289; AVX512: # %bb.0:290; AVX512-NEXT: vpbroadcastd %xmm1, %zmm1291; AVX512-NEXT: vprorvd %zmm1, %zmm0, %zmm0292; AVX512-NEXT: retq293 %splat = shufflevector <16 x i32> %amt, <16 x i32> undef, <16 x i32> zeroinitializer294 %res = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x, <16 x i32> %x, <16 x i32> %splat)295 ret <16 x i32> %res296}297 298define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounwind {299; AVX512F-LABEL: splatvar_funnnel_v32i16:300; AVX512F: # %bb.0:301; AVX512F-NEXT: vpbroadcastq {{.*#+}} xmm2 = [15,0,0,0,15,0,0,0]302; AVX512F-NEXT: vpand %xmm2, %xmm1, %xmm3303; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm4304; AVX512F-NEXT: vpsrlw %xmm3, %ymm4, %ymm5305; AVX512F-NEXT: vpsrlw %xmm3, %ymm0, %ymm3306; AVX512F-NEXT: vinserti64x4 $1, %ymm5, %zmm3, %zmm3307; AVX512F-NEXT: vpandn %xmm2, %xmm1, %xmm1308; AVX512F-NEXT: vpaddw %ymm4, %ymm4, %ymm2309; AVX512F-NEXT: vpsllw %xmm1, %ymm2, %ymm2310; AVX512F-NEXT: vpaddw %ymm0, %ymm0, %ymm0311; AVX512F-NEXT: vpsllw %xmm1, %ymm0, %ymm0312; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0313; AVX512F-NEXT: vporq %zmm3, %zmm0, %zmm0314; AVX512F-NEXT: retq315;316; AVX512VL-LABEL: splatvar_funnnel_v32i16:317; AVX512VL: # %bb.0:318; AVX512VL-NEXT: vpbroadcastq {{.*#+}} xmm2 = [15,0,0,0,15,0,0,0]319; AVX512VL-NEXT: vpand %xmm2, %xmm1, %xmm3320; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm4321; AVX512VL-NEXT: vpsrlw %xmm3, %ymm4, %ymm5322; AVX512VL-NEXT: vpsrlw %xmm3, %ymm0, %ymm3323; AVX512VL-NEXT: vinserti64x4 $1, %ymm5, %zmm3, %zmm3324; AVX512VL-NEXT: vpandn %xmm2, %xmm1, %xmm1325; AVX512VL-NEXT: vpaddw %ymm4, %ymm4, %ymm2326; AVX512VL-NEXT: vpsllw %xmm1, %ymm2, %ymm2327; AVX512VL-NEXT: vpaddw %ymm0, %ymm0, %ymm0328; AVX512VL-NEXT: vpsllw %xmm1, %ymm0, %ymm0329; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0330; AVX512VL-NEXT: vporq %zmm3, %zmm0, %zmm0331; AVX512VL-NEXT: retq332;333; AVX512BW-LABEL: splatvar_funnnel_v32i16:334; AVX512BW: # %bb.0:335; AVX512BW-NEXT: vpbroadcastq {{.*#+}} xmm2 = [15,0,0,0,15,0,0,0]336; AVX512BW-NEXT: vpand %xmm2, %xmm1, %xmm3337; AVX512BW-NEXT: vpsrlw %xmm3, %zmm0, %zmm3338; AVX512BW-NEXT: vpandn %xmm2, %xmm1, %xmm1339; AVX512BW-NEXT: vpaddw %zmm0, %zmm0, %zmm0340; AVX512BW-NEXT: vpsllw %xmm1, %zmm0, %zmm0341; AVX512BW-NEXT: vporq %zmm3, %zmm0, %zmm0342; AVX512BW-NEXT: retq343;344; AVX512VLBW-LABEL: splatvar_funnnel_v32i16:345; AVX512VLBW: # %bb.0:346; AVX512VLBW-NEXT: vpbroadcastq {{.*#+}} xmm2 = [15,0,0,0,15,0,0,0]347; AVX512VLBW-NEXT: vpand %xmm2, %xmm1, %xmm3348; AVX512VLBW-NEXT: vpsrlw %xmm3, %zmm0, %zmm3349; AVX512VLBW-NEXT: vpandn %xmm2, %xmm1, %xmm1350; AVX512VLBW-NEXT: vpaddw %zmm0, %zmm0, %zmm0351; AVX512VLBW-NEXT: vpsllw %xmm1, %zmm0, %zmm0352; AVX512VLBW-NEXT: vporq %zmm3, %zmm0, %zmm0353; AVX512VLBW-NEXT: retq354;355; AVX512VBMI2-LABEL: splatvar_funnnel_v32i16:356; AVX512VBMI2: # %bb.0:357; AVX512VBMI2-NEXT: vpbroadcastw %xmm1, %zmm1358; AVX512VBMI2-NEXT: vpshrdvw %zmm1, %zmm0, %zmm0359; AVX512VBMI2-NEXT: retq360;361; AVX512VLVBMI2-LABEL: splatvar_funnnel_v32i16:362; AVX512VLVBMI2: # %bb.0:363; AVX512VLVBMI2-NEXT: vpbroadcastw %xmm1, %zmm1364; AVX512VLVBMI2-NEXT: vpshrdvw %zmm1, %zmm0, %zmm0365; AVX512VLVBMI2-NEXT: retq366 %splat = shufflevector <32 x i16> %amt, <32 x i16> undef, <32 x i32> zeroinitializer367 %res = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x, <32 x i16> %x, <32 x i16> %splat)368 ret <32 x i16> %res369}370 371define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {372; AVX512F-LABEL: splatvar_funnnel_v64i8:373; AVX512F: # %bb.0:374; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1375; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2376; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]377; AVX512F-NEXT: vpsrlw %xmm1, %ymm3, %ymm3378; AVX512F-NEXT: vpbroadcastw {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]379; AVX512F-NEXT: vpand %ymm4, %ymm3, %ymm3380; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]381; AVX512F-NEXT: vpsrlw %xmm1, %ymm2, %ymm2382; AVX512F-NEXT: vpand %ymm4, %ymm2, %ymm2383; AVX512F-NEXT: vpackuswb %ymm3, %ymm2, %ymm2384; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]385; AVX512F-NEXT: vpsrlw %xmm1, %ymm3, %ymm3386; AVX512F-NEXT: vpand %ymm4, %ymm3, %ymm3387; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]388; AVX512F-NEXT: vpsrlw %xmm1, %ymm0, %ymm0389; AVX512F-NEXT: vpand %ymm4, %ymm0, %ymm0390; AVX512F-NEXT: vpackuswb %ymm3, %ymm0, %ymm0391; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0392; AVX512F-NEXT: retq393;394; AVX512VL-LABEL: splatvar_funnnel_v64i8:395; AVX512VL: # %bb.0:396; AVX512VL-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1397; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm2398; AVX512VL-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]399; AVX512VL-NEXT: vpsrlw %xmm1, %ymm3, %ymm3400; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]401; AVX512VL-NEXT: vpand %ymm4, %ymm3, %ymm3402; AVX512VL-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]403; AVX512VL-NEXT: vpsrlw %xmm1, %ymm2, %ymm2404; AVX512VL-NEXT: vpand %ymm4, %ymm2, %ymm2405; AVX512VL-NEXT: vpackuswb %ymm3, %ymm2, %ymm2406; AVX512VL-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]407; AVX512VL-NEXT: vpsrlw %xmm1, %ymm3, %ymm3408; AVX512VL-NEXT: vpand %ymm4, %ymm3, %ymm3409; AVX512VL-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]410; AVX512VL-NEXT: vpsrlw %xmm1, %ymm0, %ymm0411; AVX512VL-NEXT: vpand %ymm4, %ymm0, %ymm0412; AVX512VL-NEXT: vpackuswb %ymm3, %ymm0, %ymm0413; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0414; AVX512VL-NEXT: retq415;416; AVX512BW-LABEL: splatvar_funnnel_v64i8:417; AVX512BW: # %bb.0:418; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]419; AVX512BW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1420; AVX512BW-NEXT: vpsrlw %xmm1, %zmm2, %zmm2421; AVX512BW-NEXT: vpbroadcastw {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]422; AVX512BW-NEXT: vpandq %zmm3, %zmm2, %zmm2423; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]424; AVX512BW-NEXT: vpsrlw %xmm1, %zmm0, %zmm0425; AVX512BW-NEXT: vpandq %zmm3, %zmm0, %zmm0426; AVX512BW-NEXT: vpackuswb %zmm2, %zmm0, %zmm0427; AVX512BW-NEXT: retq428;429; AVX512VLBW-LABEL: splatvar_funnnel_v64i8:430; AVX512VLBW: # %bb.0:431; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]432; AVX512VLBW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1433; AVX512VLBW-NEXT: vpsrlw %xmm1, %zmm2, %zmm2434; AVX512VLBW-NEXT: vpbroadcastw {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]435; AVX512VLBW-NEXT: vpandq %zmm3, %zmm2, %zmm2436; AVX512VLBW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]437; AVX512VLBW-NEXT: vpsrlw %xmm1, %zmm0, %zmm0438; AVX512VLBW-NEXT: vpandq %zmm3, %zmm0, %zmm0439; AVX512VLBW-NEXT: vpackuswb %zmm2, %zmm0, %zmm0440; AVX512VLBW-NEXT: retq441;442; AVX512VBMI2-LABEL: splatvar_funnnel_v64i8:443; AVX512VBMI2: # %bb.0:444; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]445; AVX512VBMI2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1446; AVX512VBMI2-NEXT: vpsrlw %xmm1, %zmm2, %zmm2447; AVX512VBMI2-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]448; AVX512VBMI2-NEXT: vpsrlw %xmm1, %zmm0, %zmm1449; AVX512VBMI2-NEXT: vmovdqa64 {{.*#+}} zmm0 = [0,2,4,6,8,10,12,14,64,66,68,70,72,74,76,78,16,18,20,22,24,26,28,30,80,82,84,86,88,90,92,94,32,34,36,38,40,42,44,46,96,98,100,102,104,106,108,110,48,50,52,54,56,58,60,62,112,114,116,118,120,122,124,126]450; AVX512VBMI2-NEXT: vpermi2b %zmm2, %zmm1, %zmm0451; AVX512VBMI2-NEXT: retq452;453; AVX512VLVBMI2-LABEL: splatvar_funnnel_v64i8:454; AVX512VLVBMI2: # %bb.0:455; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]456; AVX512VLVBMI2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1457; AVX512VLVBMI2-NEXT: vpsrlw %xmm1, %zmm2, %zmm2458; AVX512VLVBMI2-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]459; AVX512VLVBMI2-NEXT: vpsrlw %xmm1, %zmm0, %zmm1460; AVX512VLVBMI2-NEXT: vmovdqa64 {{.*#+}} zmm0 = [0,2,4,6,8,10,12,14,64,66,68,70,72,74,76,78,16,18,20,22,24,26,28,30,80,82,84,86,88,90,92,94,32,34,36,38,40,42,44,46,96,98,100,102,104,106,108,110,48,50,52,54,56,58,60,62,112,114,116,118,120,122,124,126]461; AVX512VLVBMI2-NEXT: vpermi2b %zmm2, %zmm1, %zmm0462; AVX512VLVBMI2-NEXT: retq463 %splat = shufflevector <64 x i8> %amt, <64 x i8> undef, <64 x i32> zeroinitializer464 %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> %splat)465 ret <64 x i8> %res466}467 468;469; Constant Shifts470;471 472define <8 x i64> @constant_funnnel_v8i64(<8 x i64> %x) nounwind {473; AVX512-LABEL: constant_funnnel_v8i64:474; AVX512: # %bb.0:475; AVX512-NEXT: vprorvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0476; AVX512-NEXT: retq477 %res = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x, <8 x i64> %x, <8 x i64> <i64 4, i64 14, i64 50, i64 60, i64 4, i64 14, i64 50, i64 60>)478 ret <8 x i64> %res479}480 481define <16 x i32> @constant_funnnel_v16i32(<16 x i32> %x) nounwind {482; AVX512-LABEL: constant_funnnel_v16i32:483; AVX512: # %bb.0:484; AVX512-NEXT: vprorvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0485; AVX512-NEXT: retq486 %res = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x, <16 x i32> %x, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>)487 ret <16 x i32> %res488}489 490define <32 x i16> @constant_funnnel_v32i16(<32 x i16> %x) nounwind {491; AVX512F-LABEL: constant_funnnel_v32i16:492; AVX512F: # %bb.0:493; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1494; AVX512F-NEXT: vmovdqa {{.*#+}} ymm2 = [1,32768,16384,8192,4096,2048,1024,512,256,128,64,32,16,8,4,2]495; AVX512F-NEXT: vpmulhuw %ymm2, %ymm1, %ymm3496; AVX512F-NEXT: vpmulhuw %ymm2, %ymm0, %ymm4497; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm3498; AVX512F-NEXT: vpmullw %ymm2, %ymm1, %ymm1499; AVX512F-NEXT: vpmullw %ymm2, %ymm0, %ymm0500; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0501; AVX512F-NEXT: vporq %zmm3, %zmm0, %zmm0502; AVX512F-NEXT: retq503;504; AVX512VL-LABEL: constant_funnnel_v32i16:505; AVX512VL: # %bb.0:506; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm1507; AVX512VL-NEXT: vmovdqa {{.*#+}} ymm2 = [1,32768,16384,8192,4096,2048,1024,512,256,128,64,32,16,8,4,2]508; AVX512VL-NEXT: vpmulhuw %ymm2, %ymm1, %ymm3509; AVX512VL-NEXT: vpmulhuw %ymm2, %ymm0, %ymm4510; AVX512VL-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm3511; AVX512VL-NEXT: vpmullw %ymm2, %ymm1, %ymm1512; AVX512VL-NEXT: vpmullw %ymm2, %ymm0, %ymm0513; AVX512VL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0514; AVX512VL-NEXT: vporq %zmm3, %zmm0, %zmm0515; AVX512VL-NEXT: retq516;517; AVX512BW-LABEL: constant_funnnel_v32i16:518; AVX512BW: # %bb.0:519; AVX512BW-NEXT: vpsrlvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1520; AVX512BW-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0521; AVX512BW-NEXT: vporq %zmm1, %zmm0, %zmm0522; AVX512BW-NEXT: retq523;524; AVX512VLBW-LABEL: constant_funnnel_v32i16:525; AVX512VLBW: # %bb.0:526; AVX512VLBW-NEXT: vpsrlvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1527; AVX512VLBW-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0528; AVX512VLBW-NEXT: vporq %zmm1, %zmm0, %zmm0529; AVX512VLBW-NEXT: retq530;531; AVX512VBMI2-LABEL: constant_funnnel_v32i16:532; AVX512VBMI2: # %bb.0:533; AVX512VBMI2-NEXT: vpshrdvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0534; AVX512VBMI2-NEXT: retq535;536; AVX512VLVBMI2-LABEL: constant_funnnel_v32i16:537; AVX512VLVBMI2: # %bb.0:538; AVX512VLVBMI2-NEXT: vpshrdvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0539; AVX512VLVBMI2-NEXT: retq540 %res = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x, <32 x i16> %x, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>)541 ret <32 x i16> %res542}543 544define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x) nounwind {545; AVX512F-LABEL: constant_funnnel_v64i8:546; AVX512F: # %bb.0:547; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1548; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]549; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]550; AVX512F-NEXT: # ymm3 = mem[0,1,0,1]551; AVX512F-NEXT: vpmullw %ymm3, %ymm2, %ymm2552; AVX512F-NEXT: vpsrlw $8, %ymm2, %ymm2553; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]554; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2]555; AVX512F-NEXT: # ymm4 = mem[0,1,0,1]556; AVX512F-NEXT: vpmullw %ymm4, %ymm1, %ymm1557; AVX512F-NEXT: vpsrlw $8, %ymm1, %ymm1558; AVX512F-NEXT: vpackuswb %ymm2, %ymm1, %ymm1559; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]560; AVX512F-NEXT: vpmullw %ymm3, %ymm2, %ymm2561; AVX512F-NEXT: vpsrlw $8, %ymm2, %ymm2562; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]563; AVX512F-NEXT: vpmullw %ymm4, %ymm0, %ymm0564; AVX512F-NEXT: vpsrlw $8, %ymm0, %ymm0565; AVX512F-NEXT: vpackuswb %ymm2, %ymm0, %ymm0566; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0567; AVX512F-NEXT: retq568;569; AVX512VL-LABEL: constant_funnnel_v64i8:570; AVX512VL: # %bb.0:571; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm1572; AVX512VL-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]573; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]574; AVX512VL-NEXT: # ymm3 = mem[0,1,0,1]575; AVX512VL-NEXT: vpmullw %ymm3, %ymm2, %ymm2576; AVX512VL-NEXT: vpsrlw $8, %ymm2, %ymm2577; AVX512VL-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]578; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2]579; AVX512VL-NEXT: # ymm4 = mem[0,1,0,1]580; AVX512VL-NEXT: vpmullw %ymm4, %ymm1, %ymm1581; AVX512VL-NEXT: vpsrlw $8, %ymm1, %ymm1582; AVX512VL-NEXT: vpackuswb %ymm2, %ymm1, %ymm1583; AVX512VL-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]584; AVX512VL-NEXT: vpmullw %ymm3, %ymm2, %ymm2585; AVX512VL-NEXT: vpsrlw $8, %ymm2, %ymm2586; AVX512VL-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]587; AVX512VL-NEXT: vpmullw %ymm4, %ymm0, %ymm0588; AVX512VL-NEXT: vpsrlw $8, %ymm0, %ymm0589; AVX512VL-NEXT: vpackuswb %ymm2, %ymm0, %ymm0590; AVX512VL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0591; AVX512VL-NEXT: retq592;593; AVX512BW-LABEL: constant_funnnel_v64i8:594; AVX512BW: # %bb.0:595; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]596; AVX512BW-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1597; AVX512BW-NEXT: vpsrlw $8, %zmm1, %zmm1598; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]599; AVX512BW-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0600; AVX512BW-NEXT: vpsrlw $8, %zmm0, %zmm0601; AVX512BW-NEXT: vpackuswb %zmm1, %zmm0, %zmm0602; AVX512BW-NEXT: retq603;604; AVX512VLBW-LABEL: constant_funnnel_v64i8:605; AVX512VLBW: # %bb.0:606; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]607; AVX512VLBW-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1608; AVX512VLBW-NEXT: vpsrlw $8, %zmm1, %zmm1609; AVX512VLBW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]610; AVX512VLBW-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0611; AVX512VLBW-NEXT: vpsrlw $8, %zmm0, %zmm0612; AVX512VLBW-NEXT: vpackuswb %zmm1, %zmm0, %zmm0613; AVX512VLBW-NEXT: retq614;615; AVX512VBMI2-LABEL: constant_funnnel_v64i8:616; AVX512VBMI2: # %bb.0:617; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]618; AVX512VBMI2-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1619; AVX512VBMI2-NEXT: vpsrlw $8, %zmm1, %zmm1620; AVX512VBMI2-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]621; AVX512VBMI2-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0622; AVX512VBMI2-NEXT: vpsrlw $8, %zmm0, %zmm0623; AVX512VBMI2-NEXT: vpackuswb %zmm1, %zmm0, %zmm0624; AVX512VBMI2-NEXT: retq625;626; AVX512VLVBMI2-LABEL: constant_funnnel_v64i8:627; AVX512VLVBMI2: # %bb.0:628; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]629; AVX512VLVBMI2-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1630; AVX512VLVBMI2-NEXT: vpsrlw $8, %zmm1, %zmm1631; AVX512VLVBMI2-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]632; AVX512VLVBMI2-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0633; AVX512VLVBMI2-NEXT: vpsrlw $8, %zmm0, %zmm0634; AVX512VLVBMI2-NEXT: vpackuswb %zmm1, %zmm0, %zmm0635; AVX512VLVBMI2-NEXT: retq636 %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1>)637 ret <64 x i8> %res638}639 640;641; Uniform Constant Shifts642;643 644define <8 x i64> @splatconstant_funnnel_v8i64(<8 x i64> %x) nounwind {645; AVX512-LABEL: splatconstant_funnnel_v8i64:646; AVX512: # %bb.0:647; AVX512-NEXT: vprorq $14, %zmm0, %zmm0648; AVX512-NEXT: retq649 %res = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x, <8 x i64> %x, <8 x i64> <i64 14, i64 14, i64 14, i64 14, i64 14, i64 14, i64 14, i64 14>)650 ret <8 x i64> %res651}652 653define <16 x i32> @splatconstant_funnnel_v16i32(<16 x i32> %x) nounwind {654; AVX512-LABEL: splatconstant_funnnel_v16i32:655; AVX512: # %bb.0:656; AVX512-NEXT: vprord $4, %zmm0, %zmm0657; AVX512-NEXT: retq658 %res = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x, <16 x i32> %x, <16 x i32> <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>)659 ret <16 x i32> %res660}661 662define <32 x i16> @splatconstant_funnnel_v32i16(<32 x i16> %x) nounwind {663; AVX512F-LABEL: splatconstant_funnnel_v32i16:664; AVX512F: # %bb.0:665; AVX512F-NEXT: vpsrlw $7, %ymm0, %ymm1666; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2667; AVX512F-NEXT: vpsrlw $7, %ymm2, %ymm3668; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1669; AVX512F-NEXT: vpsllw $9, %ymm0, %ymm0670; AVX512F-NEXT: vpsllw $9, %ymm2, %ymm2671; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0672; AVX512F-NEXT: vporq %zmm1, %zmm0, %zmm0673; AVX512F-NEXT: retq674;675; AVX512VL-LABEL: splatconstant_funnnel_v32i16:676; AVX512VL: # %bb.0:677; AVX512VL-NEXT: vpsrlw $7, %ymm0, %ymm1678; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm2679; AVX512VL-NEXT: vpsrlw $7, %ymm2, %ymm3680; AVX512VL-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1681; AVX512VL-NEXT: vpsllw $9, %ymm0, %ymm0682; AVX512VL-NEXT: vpsllw $9, %ymm2, %ymm2683; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0684; AVX512VL-NEXT: vporq %zmm1, %zmm0, %zmm0685; AVX512VL-NEXT: retq686;687; AVX512BW-LABEL: splatconstant_funnnel_v32i16:688; AVX512BW: # %bb.0:689; AVX512BW-NEXT: vpsrlw $7, %zmm0, %zmm1690; AVX512BW-NEXT: vpsllw $9, %zmm0, %zmm0691; AVX512BW-NEXT: vporq %zmm1, %zmm0, %zmm0692; AVX512BW-NEXT: retq693;694; AVX512VLBW-LABEL: splatconstant_funnnel_v32i16:695; AVX512VLBW: # %bb.0:696; AVX512VLBW-NEXT: vpsrlw $7, %zmm0, %zmm1697; AVX512VLBW-NEXT: vpsllw $9, %zmm0, %zmm0698; AVX512VLBW-NEXT: vporq %zmm1, %zmm0, %zmm0699; AVX512VLBW-NEXT: retq700;701; AVX512VBMI2-LABEL: splatconstant_funnnel_v32i16:702; AVX512VBMI2: # %bb.0:703; AVX512VBMI2-NEXT: vpshrdw $7, %zmm0, %zmm0, %zmm0704; AVX512VBMI2-NEXT: retq705;706; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v32i16:707; AVX512VLVBMI2: # %bb.0:708; AVX512VLVBMI2-NEXT: vpshrdw $7, %zmm0, %zmm0, %zmm0709; AVX512VLVBMI2-NEXT: retq710 %res = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x, <32 x i16> %x, <32 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>)711 ret <32 x i16> %res712}713 714define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind {715; AVX512F-LABEL: splatconstant_funnnel_v64i8:716; AVX512F: # %bb.0:717; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm1718; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2719; AVX512F-NEXT: vpsllw $4, %ymm2, %ymm3720; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1721; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm0722; AVX512F-NEXT: vpsrlw $4, %ymm2, %ymm2723; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0724; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))725; AVX512F-NEXT: retq726;727; AVX512VL-LABEL: splatconstant_funnnel_v64i8:728; AVX512VL: # %bb.0:729; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm1730; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm2731; AVX512VL-NEXT: vpsllw $4, %ymm2, %ymm3732; AVX512VL-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1733; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0734; AVX512VL-NEXT: vpsrlw $4, %ymm2, %ymm2735; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0736; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))737; AVX512VL-NEXT: retq738;739; AVX512BW-LABEL: splatconstant_funnnel_v64i8:740; AVX512BW: # %bb.0:741; AVX512BW-NEXT: vpsllw $4, %zmm0, %zmm1742; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0743; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))744; AVX512BW-NEXT: retq745;746; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8:747; AVX512VLBW: # %bb.0:748; AVX512VLBW-NEXT: vpsllw $4, %zmm0, %zmm1749; AVX512VLBW-NEXT: vpsrlw $4, %zmm0, %zmm0750; AVX512VLBW-NEXT: vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))751; AVX512VLBW-NEXT: retq752;753; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8:754; AVX512VBMI2: # %bb.0:755; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm1756; AVX512VBMI2-NEXT: vpsrlw $4, %zmm0, %zmm0757; AVX512VBMI2-NEXT: vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))758; AVX512VBMI2-NEXT: retq759;760; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8:761; AVX512VLVBMI2: # %bb.0:762; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm1763; AVX512VLVBMI2-NEXT: vpsrlw $4, %zmm0, %zmm0764; AVX512VLVBMI2-NEXT: vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))765; AVX512VLVBMI2-NEXT: retq766 %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)767 ret <64 x i8> %res768}769