1755 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX13; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX24; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX512F5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefix=AVX512VL6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=AVX512BW7; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefix=AVX512VLBW8; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi,+avx512vbmi2 | FileCheck %s --check-prefix=AVX512VBMI29; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefix=AVX512VLVBMI210; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefix=XOPAVX111; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefix=XOPAVX212 13declare <4 x i64> @llvm.fshr.v4i64(<4 x i64>, <4 x i64>, <4 x i64>)14declare <8 x i32> @llvm.fshr.v8i32(<8 x i32>, <8 x i32>, <8 x i32>)15declare <16 x i16> @llvm.fshr.v16i16(<16 x i16>, <16 x i16>, <16 x i16>)16declare <32 x i8> @llvm.fshr.v32i8(<32 x i8>, <32 x i8>, <32 x i8>)17 18;19; Variable Shifts20;21 22define <4 x i64> @var_funnnel_v4i64(<4 x i64> %x, <4 x i64> %amt) nounwind {23; AVX1-LABEL: var_funnnel_v4i64:24; AVX1: # %bb.0:25; AVX1-NEXT: vbroadcastsd {{.*#+}} ymm2 = [63,63,63,63]26; AVX1-NEXT: vandps %ymm2, %ymm1, %ymm327; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm428; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm529; AVX1-NEXT: vpsrlq %xmm4, %xmm5, %xmm630; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]31; AVX1-NEXT: vpsrlq %xmm4, %xmm5, %xmm432; AVX1-NEXT: vpblendw {{.*#+}} xmm4 = xmm6[0,1,2,3],xmm4[4,5,6,7]33; AVX1-NEXT: vpsrlq %xmm3, %xmm0, %xmm634; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]35; AVX1-NEXT: vpsrlq %xmm3, %xmm0, %xmm336; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm6[0,1,2,3],xmm3[4,5,6,7]37; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm338; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm439; AVX1-NEXT: vpxor %xmm6, %xmm6, %xmm640; AVX1-NEXT: vpsubq %xmm4, %xmm6, %xmm441; AVX1-NEXT: vpand %xmm2, %xmm4, %xmm442; AVX1-NEXT: vpsllq %xmm4, %xmm5, %xmm743; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]44; AVX1-NEXT: vpsllq %xmm4, %xmm5, %xmm445; AVX1-NEXT: vpblendw {{.*#+}} xmm4 = xmm7[0,1,2,3],xmm4[4,5,6,7]46; AVX1-NEXT: vpsubq %xmm1, %xmm6, %xmm147; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm148; AVX1-NEXT: vpsllq %xmm1, %xmm0, %xmm249; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]50; AVX1-NEXT: vpsllq %xmm1, %xmm0, %xmm051; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7]52; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm053; AVX1-NEXT: vorps %ymm0, %ymm3, %ymm054; AVX1-NEXT: retq55;56; AVX2-LABEL: var_funnnel_v4i64:57; AVX2: # %bb.0:58; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm2 = [63,63,63,63]59; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm360; AVX2-NEXT: vpsrlvq %ymm3, %ymm0, %ymm361; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm462; AVX2-NEXT: vpsubq %ymm1, %ymm4, %ymm163; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm164; AVX2-NEXT: vpsllvq %ymm1, %ymm0, %ymm065; AVX2-NEXT: vpor %ymm0, %ymm3, %ymm066; AVX2-NEXT: retq67;68; AVX512F-LABEL: var_funnnel_v4i64:69; AVX512F: # %bb.0:70; AVX512F-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm171; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm072; AVX512F-NEXT: vprorvq %zmm1, %zmm0, %zmm073; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm074; AVX512F-NEXT: retq75;76; AVX512VL-LABEL: var_funnnel_v4i64:77; AVX512VL: # %bb.0:78; AVX512VL-NEXT: vprorvq %ymm1, %ymm0, %ymm079; AVX512VL-NEXT: retq80;81; AVX512BW-LABEL: var_funnnel_v4i64:82; AVX512BW: # %bb.0:83; AVX512BW-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm184; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm085; AVX512BW-NEXT: vprorvq %zmm1, %zmm0, %zmm086; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm087; AVX512BW-NEXT: retq88;89; AVX512VLBW-LABEL: var_funnnel_v4i64:90; AVX512VLBW: # %bb.0:91; AVX512VLBW-NEXT: vprorvq %ymm1, %ymm0, %ymm092; AVX512VLBW-NEXT: retq93;94; AVX512VBMI2-LABEL: var_funnnel_v4i64:95; AVX512VBMI2: # %bb.0:96; AVX512VBMI2-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm197; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm098; AVX512VBMI2-NEXT: vprorvq %zmm1, %zmm0, %zmm099; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0100; AVX512VBMI2-NEXT: retq101;102; AVX512VLVBMI2-LABEL: var_funnnel_v4i64:103; AVX512VLVBMI2: # %bb.0:104; AVX512VLVBMI2-NEXT: vprorvq %ymm1, %ymm0, %ymm0105; AVX512VLVBMI2-NEXT: retq106;107; XOPAVX1-LABEL: var_funnnel_v4i64:108; XOPAVX1: # %bb.0:109; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2110; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3111; XOPAVX1-NEXT: vpsubq %xmm2, %xmm3, %xmm2112; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm4113; XOPAVX1-NEXT: vprotq %xmm2, %xmm4, %xmm2114; XOPAVX1-NEXT: vpsubq %xmm1, %xmm3, %xmm1115; XOPAVX1-NEXT: vprotq %xmm1, %xmm0, %xmm0116; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0117; XOPAVX1-NEXT: retq118;119; XOPAVX2-LABEL: var_funnnel_v4i64:120; XOPAVX2: # %bb.0:121; XOPAVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2122; XOPAVX2-NEXT: vpsubq %ymm1, %ymm2, %ymm1123; XOPAVX2-NEXT: vextracti128 $1, %ymm1, %xmm2124; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm3125; XOPAVX2-NEXT: vprotq %xmm2, %xmm3, %xmm2126; XOPAVX2-NEXT: vprotq %xmm1, %xmm0, %xmm0127; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0128; XOPAVX2-NEXT: retq129 %res = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %x, <4 x i64> %x, <4 x i64> %amt)130 ret <4 x i64> %res131}132 133define <8 x i32> @var_funnnel_v8i32(<8 x i32> %x, <8 x i32> %amt) nounwind {134; AVX1-LABEL: var_funnnel_v8i32:135; AVX1: # %bb.0:136; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2137; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3138; AVX1-NEXT: vpsubd %xmm2, %xmm3, %xmm2139; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [31,31,31,31]140; AVX1-NEXT: vpand %xmm4, %xmm2, %xmm2141; AVX1-NEXT: vpslld $23, %xmm2, %xmm2142; AVX1-NEXT: vbroadcastss {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216]143; AVX1-NEXT: vpaddd %xmm5, %xmm2, %xmm2144; AVX1-NEXT: vcvttps2dq %xmm2, %xmm2145; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]146; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm7147; AVX1-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[1,1,3,3]148; AVX1-NEXT: vpmuludq %xmm6, %xmm8, %xmm6149; AVX1-NEXT: vpsubd %xmm1, %xmm3, %xmm1150; AVX1-NEXT: vpand %xmm4, %xmm1, %xmm1151; AVX1-NEXT: vpslld $23, %xmm1, %xmm1152; AVX1-NEXT: vpaddd %xmm5, %xmm1, %xmm1153; AVX1-NEXT: vcvttps2dq %xmm1, %xmm1154; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]155; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]156; AVX1-NEXT: vpmuludq %xmm3, %xmm4, %xmm3157; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm3, %ymm3158; AVX1-NEXT: vpmuludq %xmm2, %xmm7, %xmm2159; AVX1-NEXT: vpmuludq %xmm1, %xmm0, %xmm0160; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0161; AVX1-NEXT: vmovshdup {{.*#+}} ymm1 = ymm0[1,1,3,3,5,5,7,7]162; AVX1-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2],ymm3[3],ymm1[4],ymm3[5],ymm1[6],ymm3[7]163; AVX1-NEXT: vmovsldup {{.*#+}} ymm2 = ymm3[0,0,2,2,4,4,6,6]164; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7]165; AVX1-NEXT: vorps %ymm1, %ymm0, %ymm0166; AVX1-NEXT: retq167;168; AVX2-LABEL: var_funnnel_v8i32:169; AVX2: # %bb.0:170; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [31,31,31,31,31,31,31,31]171; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm1172; AVX2-NEXT: vpsrlvd %ymm1, %ymm0, %ymm2173; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm3 = [32,32,32,32,32,32,32,32]174; AVX2-NEXT: vpsubd %ymm1, %ymm3, %ymm1175; AVX2-NEXT: vpsllvd %ymm1, %ymm0, %ymm0176; AVX2-NEXT: vpor %ymm0, %ymm2, %ymm0177; AVX2-NEXT: retq178;179; AVX512F-LABEL: var_funnnel_v8i32:180; AVX512F: # %bb.0:181; AVX512F-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1182; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0183; AVX512F-NEXT: vprorvd %zmm1, %zmm0, %zmm0184; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0185; AVX512F-NEXT: retq186;187; AVX512VL-LABEL: var_funnnel_v8i32:188; AVX512VL: # %bb.0:189; AVX512VL-NEXT: vprorvd %ymm1, %ymm0, %ymm0190; AVX512VL-NEXT: retq191;192; AVX512BW-LABEL: var_funnnel_v8i32:193; AVX512BW: # %bb.0:194; AVX512BW-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1195; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0196; AVX512BW-NEXT: vprorvd %zmm1, %zmm0, %zmm0197; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0198; AVX512BW-NEXT: retq199;200; AVX512VLBW-LABEL: var_funnnel_v8i32:201; AVX512VLBW: # %bb.0:202; AVX512VLBW-NEXT: vprorvd %ymm1, %ymm0, %ymm0203; AVX512VLBW-NEXT: retq204;205; AVX512VBMI2-LABEL: var_funnnel_v8i32:206; AVX512VBMI2: # %bb.0:207; AVX512VBMI2-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1208; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0209; AVX512VBMI2-NEXT: vprorvd %zmm1, %zmm0, %zmm0210; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0211; AVX512VBMI2-NEXT: retq212;213; AVX512VLVBMI2-LABEL: var_funnnel_v8i32:214; AVX512VLVBMI2: # %bb.0:215; AVX512VLVBMI2-NEXT: vprorvd %ymm1, %ymm0, %ymm0216; AVX512VLVBMI2-NEXT: retq217;218; XOPAVX1-LABEL: var_funnnel_v8i32:219; XOPAVX1: # %bb.0:220; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2221; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3222; XOPAVX1-NEXT: vpsubd %xmm2, %xmm3, %xmm2223; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm4224; XOPAVX1-NEXT: vprotd %xmm2, %xmm4, %xmm2225; XOPAVX1-NEXT: vpsubd %xmm1, %xmm3, %xmm1226; XOPAVX1-NEXT: vprotd %xmm1, %xmm0, %xmm0227; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0228; XOPAVX1-NEXT: retq229;230; XOPAVX2-LABEL: var_funnnel_v8i32:231; XOPAVX2: # %bb.0:232; XOPAVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2233; XOPAVX2-NEXT: vpsubd %ymm1, %ymm2, %ymm1234; XOPAVX2-NEXT: vextracti128 $1, %ymm1, %xmm2235; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm3236; XOPAVX2-NEXT: vprotd %xmm2, %xmm3, %xmm2237; XOPAVX2-NEXT: vprotd %xmm1, %xmm0, %xmm0238; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0239; XOPAVX2-NEXT: retq240 %res = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %x, <8 x i32> %x, <8 x i32> %amt)241 ret <8 x i32> %res242}243 244define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounwind {245; AVX1-LABEL: var_funnnel_v16i16:246; AVX1: # %bb.0:247; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2248; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3249; AVX1-NEXT: vpsubw %xmm2, %xmm3, %xmm2250; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15]251; AVX1-NEXT: vpand %xmm4, %xmm2, %xmm2252; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm2[4,4,5,5,6,6,7,7]253; AVX1-NEXT: vpslld $23, %xmm5, %xmm5254; AVX1-NEXT: vbroadcastss {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216]255; AVX1-NEXT: vpaddd %xmm6, %xmm5, %xmm5256; AVX1-NEXT: vcvttps2dq %xmm5, %xmm5257; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero258; AVX1-NEXT: vpslld $23, %xmm2, %xmm2259; AVX1-NEXT: vpaddd %xmm6, %xmm2, %xmm2260; AVX1-NEXT: vcvttps2dq %xmm2, %xmm2261; AVX1-NEXT: vpackusdw %xmm5, %xmm2, %xmm2262; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5263; AVX1-NEXT: vpmulhuw %xmm2, %xmm5, %xmm7264; AVX1-NEXT: vpmullw %xmm2, %xmm5, %xmm2265; AVX1-NEXT: vpor %xmm7, %xmm2, %xmm2266; AVX1-NEXT: vpsubw %xmm1, %xmm3, %xmm1267; AVX1-NEXT: vpand %xmm4, %xmm1, %xmm1268; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4,4,5,5,6,6,7,7]269; AVX1-NEXT: vpslld $23, %xmm3, %xmm3270; AVX1-NEXT: vpaddd %xmm6, %xmm3, %xmm3271; AVX1-NEXT: vcvttps2dq %xmm3, %xmm3272; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero273; AVX1-NEXT: vpslld $23, %xmm1, %xmm1274; AVX1-NEXT: vpaddd %xmm6, %xmm1, %xmm1275; AVX1-NEXT: vcvttps2dq %xmm1, %xmm1276; AVX1-NEXT: vpackusdw %xmm3, %xmm1, %xmm1277; AVX1-NEXT: vpmulhuw %xmm1, %xmm0, %xmm3278; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0279; AVX1-NEXT: vpor %xmm3, %xmm0, %xmm0280; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0281; AVX1-NEXT: retq282;283; AVX2-LABEL: var_funnnel_v16i16:284; AVX2: # %bb.0:285; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1286; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2287; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]288; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]289; AVX2-NEXT: vpsrlvd %ymm3, %ymm4, %ymm3290; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7],ymm3[8],ymm2[9],ymm3[10],ymm2[11],ymm3[12],ymm2[13],ymm3[14],ymm2[15]291; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]292; AVX2-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]293; AVX2-NEXT: vpsrlvd %ymm1, %ymm0, %ymm0294; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7],ymm0[8],ymm2[9],ymm0[10],ymm2[11],ymm0[12],ymm2[13],ymm0[14],ymm2[15]295; AVX2-NEXT: vpackusdw %ymm3, %ymm0, %ymm0296; AVX2-NEXT: retq297;298; AVX512F-LABEL: var_funnnel_v16i16:299; AVX512F: # %bb.0:300; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1301; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2302; AVX512F-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]303; AVX512F-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]304; AVX512F-NEXT: vpsrlvd %ymm3, %ymm4, %ymm3305; AVX512F-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7],ymm3[8],ymm2[9],ymm3[10],ymm2[11],ymm3[12],ymm2[13],ymm3[14],ymm2[15]306; AVX512F-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]307; AVX512F-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]308; AVX512F-NEXT: vpsrlvd %ymm1, %ymm0, %ymm0309; AVX512F-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7],ymm0[8],ymm2[9],ymm0[10],ymm2[11],ymm0[12],ymm2[13],ymm0[14],ymm2[15]310; AVX512F-NEXT: vpackusdw %ymm3, %ymm0, %ymm0311; AVX512F-NEXT: retq312;313; AVX512VL-LABEL: var_funnnel_v16i16:314; AVX512VL: # %bb.0:315; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1316; AVX512VL-NEXT: vpxor %xmm2, %xmm2, %xmm2317; AVX512VL-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]318; AVX512VL-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]319; AVX512VL-NEXT: vpsrlvd %ymm3, %ymm4, %ymm3320; AVX512VL-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7],ymm3[8],ymm2[9],ymm3[10],ymm2[11],ymm3[12],ymm2[13],ymm3[14],ymm2[15]321; AVX512VL-NEXT: vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]322; AVX512VL-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]323; AVX512VL-NEXT: vpsrlvd %ymm1, %ymm0, %ymm0324; AVX512VL-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7],ymm0[8],ymm2[9],ymm0[10],ymm2[11],ymm0[12],ymm2[13],ymm0[14],ymm2[15]325; AVX512VL-NEXT: vpackusdw %ymm3, %ymm0, %ymm0326; AVX512VL-NEXT: retq327;328; AVX512BW-LABEL: var_funnnel_v16i16:329; AVX512BW: # %bb.0:330; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0331; AVX512BW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1332; AVX512BW-NEXT: vpsrlvw %zmm1, %zmm0, %zmm2333; AVX512BW-NEXT: vpbroadcastw {{.*#+}} ymm3 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]334; AVX512BW-NEXT: vpsubw %ymm1, %ymm3, %ymm1335; AVX512BW-NEXT: vpsllvw %zmm1, %zmm0, %zmm0336; AVX512BW-NEXT: vpor %ymm0, %ymm2, %ymm0337; AVX512BW-NEXT: retq338;339; AVX512VLBW-LABEL: var_funnnel_v16i16:340; AVX512VLBW: # %bb.0:341; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1342; AVX512VLBW-NEXT: vpsrlvw %ymm1, %ymm0, %ymm2343; AVX512VLBW-NEXT: vpbroadcastw {{.*#+}} ymm3 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]344; AVX512VLBW-NEXT: vpsubw %ymm1, %ymm3, %ymm1345; AVX512VLBW-NEXT: vpsllvw %ymm1, %ymm0, %ymm0346; AVX512VLBW-NEXT: vpor %ymm0, %ymm2, %ymm0347; AVX512VLBW-NEXT: retq348;349; AVX512VBMI2-LABEL: var_funnnel_v16i16:350; AVX512VBMI2: # %bb.0:351; AVX512VBMI2-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1352; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0353; AVX512VBMI2-NEXT: vpshrdvw %zmm1, %zmm0, %zmm0354; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0355; AVX512VBMI2-NEXT: retq356;357; AVX512VLVBMI2-LABEL: var_funnnel_v16i16:358; AVX512VLVBMI2: # %bb.0:359; AVX512VLVBMI2-NEXT: vpshrdvw %ymm1, %ymm0, %ymm0360; AVX512VLVBMI2-NEXT: retq361;362; XOPAVX1-LABEL: var_funnnel_v16i16:363; XOPAVX1: # %bb.0:364; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2365; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3366; XOPAVX1-NEXT: vpsubw %xmm2, %xmm3, %xmm2367; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm4368; XOPAVX1-NEXT: vprotw %xmm2, %xmm4, %xmm2369; XOPAVX1-NEXT: vpsubw %xmm1, %xmm3, %xmm1370; XOPAVX1-NEXT: vprotw %xmm1, %xmm0, %xmm0371; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0372; XOPAVX1-NEXT: retq373;374; XOPAVX2-LABEL: var_funnnel_v16i16:375; XOPAVX2: # %bb.0:376; XOPAVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2377; XOPAVX2-NEXT: vpsubw %ymm1, %ymm2, %ymm1378; XOPAVX2-NEXT: vextracti128 $1, %ymm1, %xmm2379; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm3380; XOPAVX2-NEXT: vprotw %xmm2, %xmm3, %xmm2381; XOPAVX2-NEXT: vprotw %xmm1, %xmm0, %xmm0382; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0383; XOPAVX2-NEXT: retq384 %res = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %x, <16 x i16> %x, <16 x i16> %amt)385 ret <16 x i16> %res386}387 388define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {389; AVX1-LABEL: var_funnnel_v32i8:390; AVX1: # %bb.0:391; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2392; AVX1-NEXT: vpsrlw $4, %xmm2, %xmm3393; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]394; AVX1-NEXT: vpandn %xmm3, %xmm4, %xmm3395; AVX1-NEXT: vpsllw $4, %xmm2, %xmm5396; AVX1-NEXT: vpand %xmm4, %xmm5, %xmm5397; AVX1-NEXT: vpor %xmm3, %xmm5, %xmm3398; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5399; AVX1-NEXT: vpxor %xmm6, %xmm6, %xmm6400; AVX1-NEXT: vpsubb %xmm5, %xmm6, %xmm5401; AVX1-NEXT: vpsllw $5, %xmm5, %xmm5402; AVX1-NEXT: vpblendvb %xmm5, %xmm3, %xmm2, %xmm2403; AVX1-NEXT: vpsrlw $6, %xmm2, %xmm3404; AVX1-NEXT: vbroadcastss {{.*#+}} xmm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]405; AVX1-NEXT: vpandn %xmm3, %xmm7, %xmm3406; AVX1-NEXT: vpsllw $2, %xmm2, %xmm8407; AVX1-NEXT: vpand %xmm7, %xmm8, %xmm8408; AVX1-NEXT: vpor %xmm3, %xmm8, %xmm3409; AVX1-NEXT: vpaddb %xmm5, %xmm5, %xmm5410; AVX1-NEXT: vpblendvb %xmm5, %xmm3, %xmm2, %xmm2411; AVX1-NEXT: vpsrlw $7, %xmm2, %xmm3412; AVX1-NEXT: vbroadcastss {{.*#+}} xmm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]413; AVX1-NEXT: vpand %xmm3, %xmm8, %xmm3414; AVX1-NEXT: vpaddb %xmm2, %xmm2, %xmm9415; AVX1-NEXT: vpor %xmm3, %xmm9, %xmm3416; AVX1-NEXT: vpaddb %xmm5, %xmm5, %xmm5417; AVX1-NEXT: vpblendvb %xmm5, %xmm3, %xmm2, %xmm2418; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm3419; AVX1-NEXT: vpandn %xmm3, %xmm4, %xmm3420; AVX1-NEXT: vpsllw $4, %xmm0, %xmm5421; AVX1-NEXT: vpand %xmm4, %xmm5, %xmm4422; AVX1-NEXT: vpor %xmm3, %xmm4, %xmm3423; AVX1-NEXT: vpsubb %xmm1, %xmm6, %xmm1424; AVX1-NEXT: vpsllw $5, %xmm1, %xmm1425; AVX1-NEXT: vpblendvb %xmm1, %xmm3, %xmm0, %xmm0426; AVX1-NEXT: vpsrlw $6, %xmm0, %xmm3427; AVX1-NEXT: vpandn %xmm3, %xmm7, %xmm3428; AVX1-NEXT: vpsllw $2, %xmm0, %xmm4429; AVX1-NEXT: vpand %xmm7, %xmm4, %xmm4430; AVX1-NEXT: vpor %xmm3, %xmm4, %xmm3431; AVX1-NEXT: vpaddb %xmm1, %xmm1, %xmm1432; AVX1-NEXT: vpblendvb %xmm1, %xmm3, %xmm0, %xmm0433; AVX1-NEXT: vpsrlw $7, %xmm0, %xmm3434; AVX1-NEXT: vpand %xmm3, %xmm8, %xmm3435; AVX1-NEXT: vpaddb %xmm0, %xmm0, %xmm4436; AVX1-NEXT: vpor %xmm3, %xmm4, %xmm3437; AVX1-NEXT: vpaddb %xmm1, %xmm1, %xmm1438; AVX1-NEXT: vpblendvb %xmm1, %xmm3, %xmm0, %xmm0439; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0440; AVX1-NEXT: retq441;442; AVX2-LABEL: var_funnnel_v32i8:443; AVX2: # %bb.0:444; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm2445; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2446; AVX2-NEXT: vpsllw $4, %ymm0, %ymm3447; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3448; AVX2-NEXT: vpor %ymm2, %ymm3, %ymm2449; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3450; AVX2-NEXT: vpsubb %ymm1, %ymm3, %ymm1451; AVX2-NEXT: vpsllw $5, %ymm1, %ymm1452; AVX2-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0453; AVX2-NEXT: vpsrlw $6, %ymm0, %ymm2454; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2455; AVX2-NEXT: vpsllw $2, %ymm0, %ymm3456; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3457; AVX2-NEXT: vpor %ymm2, %ymm3, %ymm2458; AVX2-NEXT: vpaddb %ymm1, %ymm1, %ymm1459; AVX2-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0460; AVX2-NEXT: vpaddb %ymm0, %ymm0, %ymm2461; AVX2-NEXT: vpsrlw $7, %ymm0, %ymm3462; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3463; AVX2-NEXT: vpor %ymm3, %ymm2, %ymm2464; AVX2-NEXT: vpaddb %ymm1, %ymm1, %ymm1465; AVX2-NEXT: vpblendvb %ymm1, %ymm2, %ymm0, %ymm0466; AVX2-NEXT: retq467;468; AVX512F-LABEL: var_funnnel_v32i8:469; AVX512F: # %bb.0:470; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm2471; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm3472; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 = zmm3 ^ (m32bcst & (zmm3 ^ zmm2))473; AVX512F-NEXT: vpsllw $5, %ymm1, %ymm1474; AVX512F-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0475; AVX512F-NEXT: vpsrlw $2, %ymm0, %ymm2476; AVX512F-NEXT: vpsllw $6, %ymm0, %ymm3477; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 = zmm3 ^ (m32bcst & (zmm3 ^ zmm2))478; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1479; AVX512F-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0480; AVX512F-NEXT: vpsrlw $1, %ymm0, %ymm2481; AVX512F-NEXT: vpsllw $7, %ymm0, %ymm3482; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 = zmm3 ^ (m32bcst & (zmm3 ^ zmm2))483; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1484; AVX512F-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0485; AVX512F-NEXT: retq486;487; AVX512VL-LABEL: var_funnnel_v32i8:488; AVX512VL: # %bb.0:489; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm2490; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm3491; AVX512VL-NEXT: vpternlogd {{.*#+}} ymm3 = ymm3 ^ (m32bcst & (ymm3 ^ ymm2))492; AVX512VL-NEXT: vpsllw $5, %ymm1, %ymm1493; AVX512VL-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0494; AVX512VL-NEXT: vpsrlw $2, %ymm0, %ymm2495; AVX512VL-NEXT: vpsllw $6, %ymm0, %ymm3496; AVX512VL-NEXT: vpternlogd {{.*#+}} ymm3 = ymm3 ^ (m32bcst & (ymm3 ^ ymm2))497; AVX512VL-NEXT: vpaddb %ymm1, %ymm1, %ymm1498; AVX512VL-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0499; AVX512VL-NEXT: vpsrlw $1, %ymm0, %ymm2500; AVX512VL-NEXT: vpsllw $7, %ymm0, %ymm3501; AVX512VL-NEXT: vpternlogd {{.*#+}} ymm3 = ymm3 ^ (m32bcst & (ymm3 ^ ymm2))502; AVX512VL-NEXT: vpaddb %ymm1, %ymm1, %ymm1503; AVX512VL-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0504; AVX512VL-NEXT: retq505;506; AVX512BW-LABEL: var_funnnel_v32i8:507; AVX512BW: # %bb.0:508; AVX512BW-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]509; AVX512BW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1510; AVX512BW-NEXT: vpxor %xmm3, %xmm3, %xmm3511; AVX512BW-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm3[8],ymm1[9],ymm3[9],ymm1[10],ymm3[10],ymm1[11],ymm3[11],ymm1[12],ymm3[12],ymm1[13],ymm3[13],ymm1[14],ymm3[14],ymm1[15],ymm3[15],ymm1[24],ymm3[24],ymm1[25],ymm3[25],ymm1[26],ymm3[26],ymm1[27],ymm3[27],ymm1[28],ymm3[28],ymm1[29],ymm3[29],ymm1[30],ymm3[30],ymm1[31],ymm3[31]512; AVX512BW-NEXT: vpsrlvw %zmm4, %zmm2, %zmm2513; AVX512BW-NEXT: vpbroadcastw {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]514; AVX512BW-NEXT: vpand %ymm4, %ymm2, %ymm2515; AVX512BW-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]516; AVX512BW-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[4],ymm3[4],ymm1[5],ymm3[5],ymm1[6],ymm3[6],ymm1[7],ymm3[7],ymm1[16],ymm3[16],ymm1[17],ymm3[17],ymm1[18],ymm3[18],ymm1[19],ymm3[19],ymm1[20],ymm3[20],ymm1[21],ymm3[21],ymm1[22],ymm3[22],ymm1[23],ymm3[23]517; AVX512BW-NEXT: vpsrlvw %zmm1, %zmm0, %zmm0518; AVX512BW-NEXT: vpand %ymm4, %ymm0, %ymm0519; AVX512BW-NEXT: vpackuswb %ymm2, %ymm0, %ymm0520; AVX512BW-NEXT: retq521;522; AVX512VLBW-LABEL: var_funnnel_v32i8:523; AVX512VLBW: # %bb.0:524; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1525; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2526; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]527; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]528; AVX512VLBW-NEXT: vpsrlvw %ymm3, %ymm4, %ymm3529; AVX512VLBW-NEXT: vpbroadcastw {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]530; AVX512VLBW-NEXT: vpand %ymm4, %ymm3, %ymm3531; AVX512VLBW-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]532; AVX512VLBW-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]533; AVX512VLBW-NEXT: vpsrlvw %ymm1, %ymm0, %ymm0534; AVX512VLBW-NEXT: vpand %ymm4, %ymm0, %ymm0535; AVX512VLBW-NEXT: vpackuswb %ymm3, %ymm0, %ymm0536; AVX512VLBW-NEXT: retq537;538; AVX512VBMI2-LABEL: var_funnnel_v32i8:539; AVX512VBMI2: # %bb.0:540; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]541; AVX512VBMI2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1542; AVX512VBMI2-NEXT: vpxor %xmm3, %xmm3, %xmm3543; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm3[8],ymm1[9],ymm3[9],ymm1[10],ymm3[10],ymm1[11],ymm3[11],ymm1[12],ymm3[12],ymm1[13],ymm3[13],ymm1[14],ymm3[14],ymm1[15],ymm3[15],ymm1[24],ymm3[24],ymm1[25],ymm3[25],ymm1[26],ymm3[26],ymm1[27],ymm3[27],ymm1[28],ymm3[28],ymm1[29],ymm3[29],ymm1[30],ymm3[30],ymm1[31],ymm3[31]544; AVX512VBMI2-NEXT: vpsrlvw %zmm4, %zmm2, %zmm2545; AVX512VBMI2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]546; AVX512VBMI2-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[4],ymm3[4],ymm1[5],ymm3[5],ymm1[6],ymm3[6],ymm1[7],ymm3[7],ymm1[16],ymm3[16],ymm1[17],ymm3[17],ymm1[18],ymm3[18],ymm1[19],ymm3[19],ymm1[20],ymm3[20],ymm1[21],ymm3[21],ymm1[22],ymm3[22],ymm1[23],ymm3[23]547; AVX512VBMI2-NEXT: vpsrlvw %zmm1, %zmm0, %zmm1548; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm0 = [0,2,4,6,8,10,12,14,64,66,68,70,72,74,76,78,16,18,20,22,24,26,28,30,80,82,84,86,88,90,92,94]549; AVX512VBMI2-NEXT: vpermi2b %zmm2, %zmm1, %zmm0550; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0551; AVX512VBMI2-NEXT: retq552;553; AVX512VLVBMI2-LABEL: var_funnnel_v32i8:554; AVX512VLVBMI2: # %bb.0:555; AVX512VLVBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1556; AVX512VLVBMI2-NEXT: vpxor %xmm2, %xmm2, %xmm2557; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]558; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]559; AVX512VLVBMI2-NEXT: vpsrlvw %ymm3, %ymm4, %ymm3560; AVX512VLVBMI2-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]561; AVX512VLVBMI2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]562; AVX512VLVBMI2-NEXT: vpsrlvw %ymm1, %ymm0, %ymm1563; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} ymm0 = [0,2,4,6,8,10,12,14,32,34,36,38,40,42,44,46,16,18,20,22,24,26,28,30,48,50,52,54,56,58,60,62]564; AVX512VLVBMI2-NEXT: vpermi2b %ymm3, %ymm1, %ymm0565; AVX512VLVBMI2-NEXT: retq566;567; XOPAVX1-LABEL: var_funnnel_v32i8:568; XOPAVX1: # %bb.0:569; XOPAVX1-NEXT: vextractf128 $1, %ymm1, %xmm2570; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3571; XOPAVX1-NEXT: vpsubb %xmm2, %xmm3, %xmm2572; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm4573; XOPAVX1-NEXT: vprotb %xmm2, %xmm4, %xmm2574; XOPAVX1-NEXT: vpsubb %xmm1, %xmm3, %xmm1575; XOPAVX1-NEXT: vprotb %xmm1, %xmm0, %xmm0576; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0577; XOPAVX1-NEXT: retq578;579; XOPAVX2-LABEL: var_funnnel_v32i8:580; XOPAVX2: # %bb.0:581; XOPAVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2582; XOPAVX2-NEXT: vpsubb %ymm1, %ymm2, %ymm1583; XOPAVX2-NEXT: vextracti128 $1, %ymm1, %xmm2584; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm3585; XOPAVX2-NEXT: vprotb %xmm2, %xmm3, %xmm2586; XOPAVX2-NEXT: vprotb %xmm1, %xmm0, %xmm0587; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0588; XOPAVX2-NEXT: retq589 %res = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %x, <32 x i8> %x, <32 x i8> %amt)590 ret <32 x i8> %res591}592 593;594; Uniform Variable Shifts595;596 597define <4 x i64> @splatvar_funnnel_v4i64(<4 x i64> %x, <4 x i64> %amt) nounwind {598; AVX1-LABEL: splatvar_funnnel_v4i64:599; AVX1: # %bb.0:600; AVX1-NEXT: vpmovsxbq {{.*#+}} xmm2 = [63,63]601; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm3602; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4603; AVX1-NEXT: vpsrlq %xmm3, %xmm4, %xmm5604; AVX1-NEXT: vpsrlq %xmm3, %xmm0, %xmm3605; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm3, %ymm3606; AVX1-NEXT: vpxor %xmm5, %xmm5, %xmm5607; AVX1-NEXT: vpsubq %xmm1, %xmm5, %xmm1608; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm1609; AVX1-NEXT: vpsllq %xmm1, %xmm4, %xmm2610; AVX1-NEXT: vpsllq %xmm1, %xmm0, %xmm0611; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0612; AVX1-NEXT: vorps %ymm0, %ymm3, %ymm0613; AVX1-NEXT: retq614;615; AVX2-LABEL: splatvar_funnnel_v4i64:616; AVX2: # %bb.0:617; AVX2-NEXT: vpmovsxbq {{.*#+}} xmm2 = [63,63]618; AVX2-NEXT: vpand %xmm2, %xmm1, %xmm3619; AVX2-NEXT: vpsrlq %xmm3, %ymm0, %ymm3620; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4621; AVX2-NEXT: vpsubq %xmm1, %xmm4, %xmm1622; AVX2-NEXT: vpand %xmm2, %xmm1, %xmm1623; AVX2-NEXT: vpsllq %xmm1, %ymm0, %ymm0624; AVX2-NEXT: vpor %ymm0, %ymm3, %ymm0625; AVX2-NEXT: retq626;627; AVX512F-LABEL: splatvar_funnnel_v4i64:628; AVX512F: # %bb.0:629; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0630; AVX512F-NEXT: vpbroadcastq %xmm1, %ymm1631; AVX512F-NEXT: vprorvq %zmm1, %zmm0, %zmm0632; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0633; AVX512F-NEXT: retq634;635; AVX512VL-LABEL: splatvar_funnnel_v4i64:636; AVX512VL: # %bb.0:637; AVX512VL-NEXT: vpbroadcastq %xmm1, %ymm1638; AVX512VL-NEXT: vprorvq %ymm1, %ymm0, %ymm0639; AVX512VL-NEXT: retq640;641; AVX512BW-LABEL: splatvar_funnnel_v4i64:642; AVX512BW: # %bb.0:643; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0644; AVX512BW-NEXT: vpbroadcastq %xmm1, %ymm1645; AVX512BW-NEXT: vprorvq %zmm1, %zmm0, %zmm0646; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0647; AVX512BW-NEXT: retq648;649; AVX512VLBW-LABEL: splatvar_funnnel_v4i64:650; AVX512VLBW: # %bb.0:651; AVX512VLBW-NEXT: vpbroadcastq %xmm1, %ymm1652; AVX512VLBW-NEXT: vprorvq %ymm1, %ymm0, %ymm0653; AVX512VLBW-NEXT: retq654;655; AVX512VBMI2-LABEL: splatvar_funnnel_v4i64:656; AVX512VBMI2: # %bb.0:657; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0658; AVX512VBMI2-NEXT: vpbroadcastq %xmm1, %ymm1659; AVX512VBMI2-NEXT: vprorvq %zmm1, %zmm0, %zmm0660; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0661; AVX512VBMI2-NEXT: retq662;663; AVX512VLVBMI2-LABEL: splatvar_funnnel_v4i64:664; AVX512VLVBMI2: # %bb.0:665; AVX512VLVBMI2-NEXT: vpbroadcastq %xmm1, %ymm1666; AVX512VLVBMI2-NEXT: vprorvq %ymm1, %ymm0, %ymm0667; AVX512VLVBMI2-NEXT: retq668;669; XOPAVX1-LABEL: splatvar_funnnel_v4i64:670; XOPAVX1: # %bb.0:671; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2672; XOPAVX1-NEXT: vpsubq %xmm1, %xmm2, %xmm1673; XOPAVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]674; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2675; XOPAVX1-NEXT: vprotq %xmm1, %xmm2, %xmm2676; XOPAVX1-NEXT: vprotq %xmm1, %xmm0, %xmm0677; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0678; XOPAVX1-NEXT: retq679;680; XOPAVX2-LABEL: splatvar_funnnel_v4i64:681; XOPAVX2: # %bb.0:682; XOPAVX2-NEXT: vpbroadcastq %xmm1, %xmm1683; XOPAVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2684; XOPAVX2-NEXT: vpsubq %xmm1, %xmm2, %xmm1685; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm2686; XOPAVX2-NEXT: vprotq %xmm1, %xmm2, %xmm2687; XOPAVX2-NEXT: vprotq %xmm1, %xmm0, %xmm0688; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0689; XOPAVX2-NEXT: retq690 %splat = shufflevector <4 x i64> %amt, <4 x i64> undef, <4 x i32> zeroinitializer691 %res = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %x, <4 x i64> %x, <4 x i64> %splat)692 ret <4 x i64> %res693}694 695define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %amt) nounwind {696; AVX1-LABEL: splatvar_funnnel_v8i32:697; AVX1: # %bb.0:698; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1699; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2700; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[2,2,3,3]701; AVX1-NEXT: vpsrlq %xmm1, %xmm3, %xmm3702; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]703; AVX1-NEXT: vpsrlq %xmm1, %xmm2, %xmm2704; AVX1-NEXT: vshufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[0,2]705; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,2,3,3]706; AVX1-NEXT: vpsrlq %xmm1, %xmm3, %xmm3707; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]708; AVX1-NEXT: vpsrlq %xmm1, %xmm0, %xmm0709; AVX1-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]710; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0711; AVX1-NEXT: retq712;713; AVX2-LABEL: splatvar_funnnel_v8i32:714; AVX2: # %bb.0:715; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm0[2,2,3,3,6,6,7,7]716; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1717; AVX2-NEXT: vpsrlq %xmm1, %ymm2, %ymm2718; AVX2-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[0,0,1,1,4,4,5,5]719; AVX2-NEXT: vpsrlq %xmm1, %ymm0, %ymm0720; AVX2-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm2[0,2],ymm0[4,6],ymm2[4,6]721; AVX2-NEXT: retq722;723; AVX512F-LABEL: splatvar_funnnel_v8i32:724; AVX512F: # %bb.0:725; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0726; AVX512F-NEXT: vpbroadcastd %xmm1, %ymm1727; AVX512F-NEXT: vprorvd %zmm1, %zmm0, %zmm0728; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0729; AVX512F-NEXT: retq730;731; AVX512VL-LABEL: splatvar_funnnel_v8i32:732; AVX512VL: # %bb.0:733; AVX512VL-NEXT: vpbroadcastd %xmm1, %ymm1734; AVX512VL-NEXT: vprorvd %ymm1, %ymm0, %ymm0735; AVX512VL-NEXT: retq736;737; AVX512BW-LABEL: splatvar_funnnel_v8i32:738; AVX512BW: # %bb.0:739; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0740; AVX512BW-NEXT: vpbroadcastd %xmm1, %ymm1741; AVX512BW-NEXT: vprorvd %zmm1, %zmm0, %zmm0742; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0743; AVX512BW-NEXT: retq744;745; AVX512VLBW-LABEL: splatvar_funnnel_v8i32:746; AVX512VLBW: # %bb.0:747; AVX512VLBW-NEXT: vpbroadcastd %xmm1, %ymm1748; AVX512VLBW-NEXT: vprorvd %ymm1, %ymm0, %ymm0749; AVX512VLBW-NEXT: retq750;751; AVX512VBMI2-LABEL: splatvar_funnnel_v8i32:752; AVX512VBMI2: # %bb.0:753; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0754; AVX512VBMI2-NEXT: vpbroadcastd %xmm1, %ymm1755; AVX512VBMI2-NEXT: vprorvd %zmm1, %zmm0, %zmm0756; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0757; AVX512VBMI2-NEXT: retq758;759; AVX512VLVBMI2-LABEL: splatvar_funnnel_v8i32:760; AVX512VLVBMI2: # %bb.0:761; AVX512VLVBMI2-NEXT: vpbroadcastd %xmm1, %ymm1762; AVX512VLVBMI2-NEXT: vprorvd %ymm1, %ymm0, %ymm0763; AVX512VLVBMI2-NEXT: retq764;765; XOPAVX1-LABEL: splatvar_funnnel_v8i32:766; XOPAVX1: # %bb.0:767; XOPAVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]768; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2769; XOPAVX1-NEXT: vpsubd %xmm1, %xmm2, %xmm1770; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2771; XOPAVX1-NEXT: vprotd %xmm1, %xmm2, %xmm2772; XOPAVX1-NEXT: vprotd %xmm1, %xmm0, %xmm0773; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0774; XOPAVX1-NEXT: retq775;776; XOPAVX2-LABEL: splatvar_funnnel_v8i32:777; XOPAVX2: # %bb.0:778; XOPAVX2-NEXT: vpbroadcastd %xmm1, %xmm1779; XOPAVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2780; XOPAVX2-NEXT: vpsubd %xmm1, %xmm2, %xmm1781; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm2782; XOPAVX2-NEXT: vprotd %xmm1, %xmm2, %xmm2783; XOPAVX2-NEXT: vprotd %xmm1, %xmm0, %xmm0784; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0785; XOPAVX2-NEXT: retq786 %splat = shufflevector <8 x i32> %amt, <8 x i32> undef, <8 x i32> zeroinitializer787 %res = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %x, <8 x i32> %x, <8 x i32> %splat)788 ret <8 x i32> %res789}790 791define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounwind {792; AVX1-LABEL: splatvar_funnnel_v16i16:793; AVX1: # %bb.0:794; AVX1-NEXT: vpmovsxbq {{.*#+}} xmm2 = [15,0]795; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm3796; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4797; AVX1-NEXT: vpsrlw %xmm3, %xmm4, %xmm5798; AVX1-NEXT: vpandn %xmm2, %xmm1, %xmm1799; AVX1-NEXT: vpaddw %xmm4, %xmm4, %xmm2800; AVX1-NEXT: vpsllw %xmm1, %xmm2, %xmm2801; AVX1-NEXT: vpor %xmm5, %xmm2, %xmm2802; AVX1-NEXT: vpsrlw %xmm3, %xmm0, %xmm3803; AVX1-NEXT: vpaddw %xmm0, %xmm0, %xmm0804; AVX1-NEXT: vpsllw %xmm1, %xmm0, %xmm0805; AVX1-NEXT: vpor %xmm3, %xmm0, %xmm0806; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0807; AVX1-NEXT: retq808;809; AVX2-LABEL: splatvar_funnnel_v16i16:810; AVX2: # %bb.0:811; AVX2-NEXT: vpbroadcastq {{.*#+}} xmm2 = [15,0,0,0,15,0,0,0]812; AVX2-NEXT: vpand %xmm2, %xmm1, %xmm3813; AVX2-NEXT: vpsrlw %xmm3, %ymm0, %ymm3814; AVX2-NEXT: vpandn %xmm2, %xmm1, %xmm1815; AVX2-NEXT: vpaddw %ymm0, %ymm0, %ymm0816; AVX2-NEXT: vpsllw %xmm1, %ymm0, %ymm0817; AVX2-NEXT: vpor %ymm3, %ymm0, %ymm0818; AVX2-NEXT: retq819;820; AVX512F-LABEL: splatvar_funnnel_v16i16:821; AVX512F: # %bb.0:822; AVX512F-NEXT: vpbroadcastq {{.*#+}} xmm2 = [15,0,0,0,15,0,0,0]823; AVX512F-NEXT: vpand %xmm2, %xmm1, %xmm3824; AVX512F-NEXT: vpsrlw %xmm3, %ymm0, %ymm3825; AVX512F-NEXT: vpandn %xmm2, %xmm1, %xmm1826; AVX512F-NEXT: vpaddw %ymm0, %ymm0, %ymm0827; AVX512F-NEXT: vpsllw %xmm1, %ymm0, %ymm0828; AVX512F-NEXT: vpor %ymm3, %ymm0, %ymm0829; AVX512F-NEXT: retq830;831; AVX512VL-LABEL: splatvar_funnnel_v16i16:832; AVX512VL: # %bb.0:833; AVX512VL-NEXT: vpbroadcastq {{.*#+}} xmm2 = [15,0,0,0,15,0,0,0]834; AVX512VL-NEXT: vpand %xmm2, %xmm1, %xmm3835; AVX512VL-NEXT: vpsrlw %xmm3, %ymm0, %ymm3836; AVX512VL-NEXT: vpandn %xmm2, %xmm1, %xmm1837; AVX512VL-NEXT: vpaddw %ymm0, %ymm0, %ymm0838; AVX512VL-NEXT: vpsllw %xmm1, %ymm0, %ymm0839; AVX512VL-NEXT: vpor %ymm3, %ymm0, %ymm0840; AVX512VL-NEXT: retq841;842; AVX512BW-LABEL: splatvar_funnnel_v16i16:843; AVX512BW: # %bb.0:844; AVX512BW-NEXT: vpbroadcastq {{.*#+}} xmm2 = [15,0,0,0,15,0,0,0]845; AVX512BW-NEXT: vpand %xmm2, %xmm1, %xmm3846; AVX512BW-NEXT: vpsrlw %xmm3, %ymm0, %ymm3847; AVX512BW-NEXT: vpandn %xmm2, %xmm1, %xmm1848; AVX512BW-NEXT: vpaddw %ymm0, %ymm0, %ymm0849; AVX512BW-NEXT: vpsllw %xmm1, %ymm0, %ymm0850; AVX512BW-NEXT: vpor %ymm3, %ymm0, %ymm0851; AVX512BW-NEXT: retq852;853; AVX512VLBW-LABEL: splatvar_funnnel_v16i16:854; AVX512VLBW: # %bb.0:855; AVX512VLBW-NEXT: vpbroadcastq {{.*#+}} xmm2 = [15,0,0,0,15,0,0,0]856; AVX512VLBW-NEXT: vpand %xmm2, %xmm1, %xmm3857; AVX512VLBW-NEXT: vpsrlw %xmm3, %ymm0, %ymm3858; AVX512VLBW-NEXT: vpandn %xmm2, %xmm1, %xmm1859; AVX512VLBW-NEXT: vpaddw %ymm0, %ymm0, %ymm0860; AVX512VLBW-NEXT: vpsllw %xmm1, %ymm0, %ymm0861; AVX512VLBW-NEXT: vpor %ymm3, %ymm0, %ymm0862; AVX512VLBW-NEXT: retq863;864; AVX512VBMI2-LABEL: splatvar_funnnel_v16i16:865; AVX512VBMI2: # %bb.0:866; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0867; AVX512VBMI2-NEXT: vpbroadcastw %xmm1, %ymm1868; AVX512VBMI2-NEXT: vpshrdvw %zmm1, %zmm0, %zmm0869; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0870; AVX512VBMI2-NEXT: retq871;872; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i16:873; AVX512VLVBMI2: # %bb.0:874; AVX512VLVBMI2-NEXT: vpbroadcastw %xmm1, %ymm1875; AVX512VLVBMI2-NEXT: vpshrdvw %ymm1, %ymm0, %ymm0876; AVX512VLVBMI2-NEXT: retq877;878; XOPAVX1-LABEL: splatvar_funnnel_v16i16:879; XOPAVX1: # %bb.0:880; XOPAVX1-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,0,0,0,4,5,6,7]881; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2882; XOPAVX1-NEXT: vpsubw %xmm1, %xmm2, %xmm1883; XOPAVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]884; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2885; XOPAVX1-NEXT: vprotw %xmm1, %xmm2, %xmm2886; XOPAVX1-NEXT: vprotw %xmm1, %xmm0, %xmm0887; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0888; XOPAVX1-NEXT: retq889;890; XOPAVX2-LABEL: splatvar_funnnel_v16i16:891; XOPAVX2: # %bb.0:892; XOPAVX2-NEXT: vpbroadcastw %xmm1, %xmm1893; XOPAVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2894; XOPAVX2-NEXT: vpsubw %xmm1, %xmm2, %xmm1895; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm2896; XOPAVX2-NEXT: vprotw %xmm1, %xmm2, %xmm2897; XOPAVX2-NEXT: vprotw %xmm1, %xmm0, %xmm0898; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0899; XOPAVX2-NEXT: retq900 %splat = shufflevector <16 x i16> %amt, <16 x i16> undef, <16 x i32> zeroinitializer901 %res = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %x, <16 x i16> %x, <16 x i16> %splat)902 ret <16 x i16> %res903}904 905define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {906; AVX1-LABEL: splatvar_funnnel_v32i8:907; AVX1: # %bb.0:908; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1909; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2910; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]911; AVX1-NEXT: vpsrlw %xmm1, %xmm3, %xmm3912; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]913; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm3914; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]915; AVX1-NEXT: vpsrlw %xmm1, %xmm2, %xmm2916; AVX1-NEXT: vpand %xmm4, %xmm2, %xmm2917; AVX1-NEXT: vpackuswb %xmm3, %xmm2, %xmm2918; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]919; AVX1-NEXT: vpsrlw %xmm1, %xmm3, %xmm3920; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm3921; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]922; AVX1-NEXT: vpsrlw %xmm1, %xmm0, %xmm0923; AVX1-NEXT: vpand %xmm4, %xmm0, %xmm0924; AVX1-NEXT: vpackuswb %xmm3, %xmm0, %xmm0925; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0926; AVX1-NEXT: retq927;928; AVX2-LABEL: splatvar_funnnel_v32i8:929; AVX2: # %bb.0:930; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]931; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1932; AVX2-NEXT: vpsrlw %xmm1, %ymm2, %ymm2933; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]934; AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2935; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]936; AVX2-NEXT: vpsrlw %xmm1, %ymm0, %ymm0937; AVX2-NEXT: vpand %ymm3, %ymm0, %ymm0938; AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0939; AVX2-NEXT: retq940;941; AVX512F-LABEL: splatvar_funnnel_v32i8:942; AVX512F: # %bb.0:943; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]944; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1945; AVX512F-NEXT: vpsrlw %xmm1, %ymm2, %ymm2946; AVX512F-NEXT: vpbroadcastw {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]947; AVX512F-NEXT: vpand %ymm3, %ymm2, %ymm2948; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]949; AVX512F-NEXT: vpsrlw %xmm1, %ymm0, %ymm0950; AVX512F-NEXT: vpand %ymm3, %ymm0, %ymm0951; AVX512F-NEXT: vpackuswb %ymm2, %ymm0, %ymm0952; AVX512F-NEXT: retq953;954; AVX512VL-LABEL: splatvar_funnnel_v32i8:955; AVX512VL: # %bb.0:956; AVX512VL-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]957; AVX512VL-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1958; AVX512VL-NEXT: vpsrlw %xmm1, %ymm2, %ymm2959; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]960; AVX512VL-NEXT: vpand %ymm3, %ymm2, %ymm2961; AVX512VL-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]962; AVX512VL-NEXT: vpsrlw %xmm1, %ymm0, %ymm0963; AVX512VL-NEXT: vpand %ymm3, %ymm0, %ymm0964; AVX512VL-NEXT: vpackuswb %ymm2, %ymm0, %ymm0965; AVX512VL-NEXT: retq966;967; AVX512BW-LABEL: splatvar_funnnel_v32i8:968; AVX512BW: # %bb.0:969; AVX512BW-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]970; AVX512BW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1971; AVX512BW-NEXT: vpsrlw %xmm1, %ymm2, %ymm2972; AVX512BW-NEXT: vpbroadcastw {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]973; AVX512BW-NEXT: vpand %ymm3, %ymm2, %ymm2974; AVX512BW-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]975; AVX512BW-NEXT: vpsrlw %xmm1, %ymm0, %ymm0976; AVX512BW-NEXT: vpand %ymm3, %ymm0, %ymm0977; AVX512BW-NEXT: vpackuswb %ymm2, %ymm0, %ymm0978; AVX512BW-NEXT: retq979;980; AVX512VLBW-LABEL: splatvar_funnnel_v32i8:981; AVX512VLBW: # %bb.0:982; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]983; AVX512VLBW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1984; AVX512VLBW-NEXT: vpsrlw %xmm1, %ymm2, %ymm2985; AVX512VLBW-NEXT: vpbroadcastw {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]986; AVX512VLBW-NEXT: vpand %ymm3, %ymm2, %ymm2987; AVX512VLBW-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]988; AVX512VLBW-NEXT: vpsrlw %xmm1, %ymm0, %ymm0989; AVX512VLBW-NEXT: vpand %ymm3, %ymm0, %ymm0990; AVX512VLBW-NEXT: vpackuswb %ymm2, %ymm0, %ymm0991; AVX512VLBW-NEXT: retq992;993; AVX512VBMI2-LABEL: splatvar_funnnel_v32i8:994; AVX512VBMI2: # %bb.0:995; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0996; AVX512VBMI2-NEXT: vmovdqa {{.*#+}} ymm2 = [0,2,4,6,8,10,12,14,32,34,36,38,40,42,44,46,16,18,20,22,24,26,28,30,48,50,52,54,56,58,60,62]997; AVX512VBMI2-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]998; AVX512VBMI2-NEXT: vpermb %zmm0, %zmm3, %zmm0999; AVX512VBMI2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm11000; AVX512VBMI2-NEXT: vpsrlw %xmm1, %zmm0, %zmm01001; AVX512VBMI2-NEXT: vpermb %zmm0, %zmm2, %zmm01002; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01003; AVX512VBMI2-NEXT: retq1004;1005; AVX512VLVBMI2-LABEL: splatvar_funnnel_v32i8:1006; AVX512VLVBMI2: # %bb.0:1007; AVX512VLVBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm01008; AVX512VLVBMI2-NEXT: vmovdqa {{.*#+}} ymm2 = [32,34,36,38,40,42,44,46,0,2,4,6,8,10,12,14,48,50,52,54,56,58,60,62,16,18,20,22,24,26,28,30]1009; AVX512VLVBMI2-NEXT: vmovdqa64 {{.*#+}} zmm3 = [8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]1010; AVX512VLVBMI2-NEXT: vpermb %zmm0, %zmm3, %zmm01011; AVX512VLVBMI2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm11012; AVX512VLVBMI2-NEXT: vpsrlw %xmm1, %zmm0, %zmm01013; AVX512VLVBMI2-NEXT: vpermb %zmm0, %zmm2, %zmm01014; AVX512VLVBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01015; AVX512VLVBMI2-NEXT: retq1016;1017; XOPAVX1-LABEL: splatvar_funnnel_v32i8:1018; XOPAVX1: # %bb.0:1019; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm21020; XOPAVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm11021; XOPAVX1-NEXT: vpsubb %xmm1, %xmm2, %xmm11022; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm21023; XOPAVX1-NEXT: vprotb %xmm1, %xmm2, %xmm21024; XOPAVX1-NEXT: vprotb %xmm1, %xmm0, %xmm01025; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm01026; XOPAVX1-NEXT: retq1027;1028; XOPAVX2-LABEL: splatvar_funnnel_v32i8:1029; XOPAVX2: # %bb.0:1030; XOPAVX2-NEXT: vpbroadcastb %xmm1, %xmm11031; XOPAVX2-NEXT: vpxor %xmm2, %xmm2, %xmm21032; XOPAVX2-NEXT: vpsubb %xmm1, %xmm2, %xmm11033; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm21034; XOPAVX2-NEXT: vprotb %xmm1, %xmm2, %xmm21035; XOPAVX2-NEXT: vprotb %xmm1, %xmm0, %xmm01036; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm01037; XOPAVX2-NEXT: retq1038 %splat = shufflevector <32 x i8> %amt, <32 x i8> undef, <32 x i32> zeroinitializer1039 %res = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %x, <32 x i8> %x, <32 x i8> %splat)1040 ret <32 x i8> %res1041}1042 1043;1044; Constant Shifts1045;1046 1047define <4 x i64> @constant_funnnel_v4i64(<4 x i64> %x) nounwind {1048; AVX1-LABEL: constant_funnnel_v4i64:1049; AVX1: # %bb.0:1050; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm11051; AVX1-NEXT: vpsllq $4, %xmm1, %xmm21052; AVX1-NEXT: vpsllq $14, %xmm1, %xmm31053; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]1054; AVX1-NEXT: vpsllq $50, %xmm0, %xmm31055; AVX1-NEXT: vpsllq $60, %xmm0, %xmm41056; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]1057; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm21058; AVX1-NEXT: vpsrlq $60, %xmm1, %xmm31059; AVX1-NEXT: vpsrlq $50, %xmm1, %xmm11060; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5,6,7]1061; AVX1-NEXT: vpsrlq $14, %xmm0, %xmm31062; AVX1-NEXT: vpsrlq $4, %xmm0, %xmm01063; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm3[4,5,6,7]1064; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm01065; AVX1-NEXT: vorps %ymm2, %ymm0, %ymm01066; AVX1-NEXT: retq1067;1068; AVX2-LABEL: constant_funnnel_v4i64:1069; AVX2: # %bb.0:1070; AVX2-NEXT: vpsllvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm11071; AVX2-NEXT: vpsrlvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01072; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm01073; AVX2-NEXT: retq1074;1075; AVX512F-LABEL: constant_funnnel_v4i64:1076; AVX512F: # %bb.0:1077; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm01078; AVX512F-NEXT: vpmovsxbq {{.*#+}} ymm1 = [4,14,50,60]1079; AVX512F-NEXT: vprorvq %zmm1, %zmm0, %zmm01080; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01081; AVX512F-NEXT: retq1082;1083; AVX512VL-LABEL: constant_funnnel_v4i64:1084; AVX512VL: # %bb.0:1085; AVX512VL-NEXT: vprorvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01086; AVX512VL-NEXT: retq1087;1088; AVX512BW-LABEL: constant_funnnel_v4i64:1089; AVX512BW: # %bb.0:1090; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm01091; AVX512BW-NEXT: vpmovsxbq {{.*#+}} ymm1 = [4,14,50,60]1092; AVX512BW-NEXT: vprorvq %zmm1, %zmm0, %zmm01093; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01094; AVX512BW-NEXT: retq1095;1096; AVX512VLBW-LABEL: constant_funnnel_v4i64:1097; AVX512VLBW: # %bb.0:1098; AVX512VLBW-NEXT: vprorvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01099; AVX512VLBW-NEXT: retq1100;1101; AVX512VBMI2-LABEL: constant_funnnel_v4i64:1102; AVX512VBMI2: # %bb.0:1103; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm01104; AVX512VBMI2-NEXT: vpmovsxbq {{.*#+}} ymm1 = [4,14,50,60]1105; AVX512VBMI2-NEXT: vprorvq %zmm1, %zmm0, %zmm01106; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01107; AVX512VBMI2-NEXT: retq1108;1109; AVX512VLVBMI2-LABEL: constant_funnnel_v4i64:1110; AVX512VLVBMI2: # %bb.0:1111; AVX512VLVBMI2-NEXT: vprorvq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01112; AVX512VLVBMI2-NEXT: retq1113;1114; XOPAVX1-LABEL: constant_funnnel_v4i64:1115; XOPAVX1: # %bb.0:1116; XOPAVX1-NEXT: vprotq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm11117; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm01118; XOPAVX1-NEXT: vprotq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm01119; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm01120; XOPAVX1-NEXT: retq1121;1122; XOPAVX2-LABEL: constant_funnnel_v4i64:1123; XOPAVX2: # %bb.0:1124; XOPAVX2-NEXT: vprotq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm11125; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm01126; XOPAVX2-NEXT: vprotq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm01127; XOPAVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm01128; XOPAVX2-NEXT: retq1129 %res = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %x, <4 x i64> %x, <4 x i64> <i64 4, i64 14, i64 50, i64 60>)1130 ret <4 x i64> %res1131}1132 1133define <8 x i32> @constant_funnnel_v8i32(<8 x i32> %x) nounwind {1134; AVX1-LABEL: constant_funnnel_v8i32:1135; AVX1: # %bb.0:1136; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]1137; AVX1-NEXT: vpmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [134217728,u,33554432,u]1138; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm21139; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]1140; AVX1-NEXT: vpmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3 # [8388608,u,2097152,u]1141; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm11142; AVX1-NEXT: vpmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [268435456,134217728,67108864,33554432]1143; AVX1-NEXT: vpmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 # [16777216,8388608,4194304,2097152]1144; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm01145; AVX1-NEXT: vmovshdup {{.*#+}} ymm2 = ymm0[1,1,3,3,5,5,7,7]1146; AVX1-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4],ymm1[5],ymm2[6],ymm1[7]1147; AVX1-NEXT: vmovsldup {{.*#+}} ymm1 = ymm1[0,0,2,2,4,4,6,6]1148; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]1149; AVX1-NEXT: vorps %ymm2, %ymm0, %ymm01150; AVX1-NEXT: retq1151;1152; AVX2-LABEL: constant_funnnel_v8i32:1153; AVX2: # %bb.0:1154; AVX2-NEXT: vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm11155; AVX2-NEXT: vpsllvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01156; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm01157; AVX2-NEXT: retq1158;1159; AVX512F-LABEL: constant_funnnel_v8i32:1160; AVX512F: # %bb.0:1161; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm01162; AVX512F-NEXT: vpmovsxbd {{.*#+}} ymm1 = [4,5,6,7,8,9,10,11]1163; AVX512F-NEXT: vprorvd %zmm1, %zmm0, %zmm01164; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01165; AVX512F-NEXT: retq1166;1167; AVX512VL-LABEL: constant_funnnel_v8i32:1168; AVX512VL: # %bb.0:1169; AVX512VL-NEXT: vprorvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01170; AVX512VL-NEXT: retq1171;1172; AVX512BW-LABEL: constant_funnnel_v8i32:1173; AVX512BW: # %bb.0:1174; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm01175; AVX512BW-NEXT: vpmovsxbd {{.*#+}} ymm1 = [4,5,6,7,8,9,10,11]1176; AVX512BW-NEXT: vprorvd %zmm1, %zmm0, %zmm01177; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01178; AVX512BW-NEXT: retq1179;1180; AVX512VLBW-LABEL: constant_funnnel_v8i32:1181; AVX512VLBW: # %bb.0:1182; AVX512VLBW-NEXT: vprorvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01183; AVX512VLBW-NEXT: retq1184;1185; AVX512VBMI2-LABEL: constant_funnnel_v8i32:1186; AVX512VBMI2: # %bb.0:1187; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm01188; AVX512VBMI2-NEXT: vpmovsxbd {{.*#+}} ymm1 = [4,5,6,7,8,9,10,11]1189; AVX512VBMI2-NEXT: vprorvd %zmm1, %zmm0, %zmm01190; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01191; AVX512VBMI2-NEXT: retq1192;1193; AVX512VLVBMI2-LABEL: constant_funnnel_v8i32:1194; AVX512VLVBMI2: # %bb.0:1195; AVX512VLVBMI2-NEXT: vprorvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01196; AVX512VLVBMI2-NEXT: retq1197;1198; XOPAVX1-LABEL: constant_funnnel_v8i32:1199; XOPAVX1: # %bb.0:1200; XOPAVX1-NEXT: vprotd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm11201; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm01202; XOPAVX1-NEXT: vprotd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm01203; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm01204; XOPAVX1-NEXT: retq1205;1206; XOPAVX2-LABEL: constant_funnnel_v8i32:1207; XOPAVX2: # %bb.0:1208; XOPAVX2-NEXT: vprotd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm11209; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm01210; XOPAVX2-NEXT: vprotd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm01211; XOPAVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm01212; XOPAVX2-NEXT: retq1213 %res = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %x, <8 x i32> %x, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>)1214 ret <8 x i32> %res1215}1216 1217define <16 x i16> @constant_funnnel_v16i16(<16 x i16> %x) nounwind {1218; AVX1-LABEL: constant_funnnel_v16i16:1219; AVX1: # %bb.0:1220; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm11221; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [256,128,64,32,16,8,4,2]1222; AVX1-NEXT: vpmulhuw %xmm2, %xmm1, %xmm31223; AVX1-NEXT: vpmullw %xmm2, %xmm1, %xmm11224; AVX1-NEXT: vpor %xmm3, %xmm1, %xmm11225; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [1,32768,16384,8192,4096,2048,1024,512]1226; AVX1-NEXT: vpmulhuw %xmm2, %xmm0, %xmm31227; AVX1-NEXT: vpmullw %xmm2, %xmm0, %xmm01228; AVX1-NEXT: vpor %xmm3, %xmm0, %xmm01229; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm01230; AVX1-NEXT: retq1231;1232; AVX2-LABEL: constant_funnnel_v16i16:1233; AVX2: # %bb.0:1234; AVX2-NEXT: vmovdqa {{.*#+}} ymm1 = [1,32768,16384,8192,4096,2048,1024,512,256,128,64,32,16,8,4,2]1235; AVX2-NEXT: vpmulhuw %ymm1, %ymm0, %ymm21236; AVX2-NEXT: vpmullw %ymm1, %ymm0, %ymm01237; AVX2-NEXT: vpor %ymm2, %ymm0, %ymm01238; AVX2-NEXT: retq1239;1240; AVX512F-LABEL: constant_funnnel_v16i16:1241; AVX512F: # %bb.0:1242; AVX512F-NEXT: vmovdqa {{.*#+}} ymm1 = [1,32768,16384,8192,4096,2048,1024,512,256,128,64,32,16,8,4,2]1243; AVX512F-NEXT: vpmulhuw %ymm1, %ymm0, %ymm21244; AVX512F-NEXT: vpmullw %ymm1, %ymm0, %ymm01245; AVX512F-NEXT: vpor %ymm2, %ymm0, %ymm01246; AVX512F-NEXT: retq1247;1248; AVX512VL-LABEL: constant_funnnel_v16i16:1249; AVX512VL: # %bb.0:1250; AVX512VL-NEXT: vmovdqa {{.*#+}} ymm1 = [1,32768,16384,8192,4096,2048,1024,512,256,128,64,32,16,8,4,2]1251; AVX512VL-NEXT: vpmulhuw %ymm1, %ymm0, %ymm21252; AVX512VL-NEXT: vpmullw %ymm1, %ymm0, %ymm01253; AVX512VL-NEXT: vpor %ymm2, %ymm0, %ymm01254; AVX512VL-NEXT: retq1255;1256; AVX512BW-LABEL: constant_funnnel_v16i16:1257; AVX512BW: # %bb.0:1258; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm01259; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm1 = [16,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]1260; AVX512BW-NEXT: vpsrlvw %zmm1, %zmm0, %zmm11261; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [0,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1]1262; AVX512BW-NEXT: vpsllvw %zmm2, %zmm0, %zmm01263; AVX512BW-NEXT: vpor %ymm1, %ymm0, %ymm01264; AVX512BW-NEXT: retq1265;1266; AVX512VLBW-LABEL: constant_funnnel_v16i16:1267; AVX512VLBW: # %bb.0:1268; AVX512VLBW-NEXT: vpsrlvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm11269; AVX512VLBW-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01270; AVX512VLBW-NEXT: vpor %ymm1, %ymm0, %ymm01271; AVX512VLBW-NEXT: retq1272;1273; AVX512VBMI2-LABEL: constant_funnnel_v16i16:1274; AVX512VBMI2: # %bb.0:1275; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm01276; AVX512VBMI2-NEXT: vpmovsxbw {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]1277; AVX512VBMI2-NEXT: vpshrdvw %zmm1, %zmm0, %zmm01278; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01279; AVX512VBMI2-NEXT: retq1280;1281; AVX512VLVBMI2-LABEL: constant_funnnel_v16i16:1282; AVX512VLVBMI2: # %bb.0:1283; AVX512VLVBMI2-NEXT: vpshrdvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01284; AVX512VLVBMI2-NEXT: retq1285;1286; XOPAVX1-LABEL: constant_funnnel_v16i16:1287; XOPAVX1: # %bb.0:1288; XOPAVX1-NEXT: vprotw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm11289; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm01290; XOPAVX1-NEXT: vprotw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm01291; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm01292; XOPAVX1-NEXT: retq1293;1294; XOPAVX2-LABEL: constant_funnnel_v16i16:1295; XOPAVX2: # %bb.0:1296; XOPAVX2-NEXT: vprotw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm11297; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm01298; XOPAVX2-NEXT: vprotw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm01299; XOPAVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm01300; XOPAVX2-NEXT: retq1301 %res = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %x, <16 x i16> %x, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>)1302 ret <16 x i16> %res1303}1304 1305define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x) nounwind {1306; AVX1-LABEL: constant_funnnel_v32i8:1307; AVX1: # %bb.0:1308; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm11309; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1310; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm3 = [1,2,4,8,16,32,64,128]1311; AVX1-NEXT: vpmullw %xmm3, %xmm2, %xmm21312; AVX1-NEXT: vpsrlw $8, %xmm2, %xmm21313; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1314; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm4 = [1,128,64,32,16,8,4,2]1315; AVX1-NEXT: vpmullw %xmm4, %xmm1, %xmm11316; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm11317; AVX1-NEXT: vpackuswb %xmm2, %xmm1, %xmm11318; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1319; AVX1-NEXT: vpmullw %xmm3, %xmm2, %xmm21320; AVX1-NEXT: vpsrlw $8, %xmm2, %xmm21321; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1322; AVX1-NEXT: vpmullw %xmm4, %xmm0, %xmm01323; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm01324; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm01325; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm01326; AVX1-NEXT: retq1327;1328; AVX2-LABEL: constant_funnnel_v32i8:1329; AVX2: # %bb.0:1330; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]1331; AVX2-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]1332; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm11333; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]1334; AVX2-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2]1335; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm01336; AVX2-NEXT: vpackuswb %ymm1, %ymm0, %ymm01337; AVX2-NEXT: retq1338;1339; AVX512F-LABEL: constant_funnnel_v32i8:1340; AVX512F: # %bb.0:1341; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]1342; AVX512F-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]1343; AVX512F-NEXT: vpsrlw $8, %ymm1, %ymm11344; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]1345; AVX512F-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2]1346; AVX512F-NEXT: vpsrlw $8, %ymm0, %ymm01347; AVX512F-NEXT: vpackuswb %ymm1, %ymm0, %ymm01348; AVX512F-NEXT: retq1349;1350; AVX512VL-LABEL: constant_funnnel_v32i8:1351; AVX512VL: # %bb.0:1352; AVX512VL-NEXT: vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]1353; AVX512VL-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]1354; AVX512VL-NEXT: vpsrlw $8, %ymm1, %ymm11355; AVX512VL-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]1356; AVX512VL-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2]1357; AVX512VL-NEXT: vpsrlw $8, %ymm0, %ymm01358; AVX512VL-NEXT: vpackuswb %ymm1, %ymm0, %ymm01359; AVX512VL-NEXT: retq1360;1361; AVX512BW-LABEL: constant_funnnel_v32i8:1362; AVX512BW: # %bb.0:1363; AVX512BW-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [0,0,1,0,2,0,3,0,4,0,5,0,6,0,7,0,0,0,1,0,2,0,3,0,4,0,5,0,6,0,7,0]1364; AVX512BW-NEXT: # ymm1 = mem[0,1,0,1]1365; AVX512BW-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]1366; AVX512BW-NEXT: vpsllvw %zmm1, %zmm2, %zmm11367; AVX512BW-NEXT: vpsrlw $8, %ymm1, %ymm11368; AVX512BW-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [0,0,7,0,6,0,5,0,4,0,3,0,2,0,1,0,0,0,7,0,6,0,5,0,4,0,3,0,2,0,1,0]1369; AVX512BW-NEXT: # ymm2 = mem[0,1,0,1]1370; AVX512BW-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]1371; AVX512BW-NEXT: vpsllvw %zmm2, %zmm0, %zmm01372; AVX512BW-NEXT: vpsrlw $8, %ymm0, %ymm01373; AVX512BW-NEXT: vpackuswb %ymm1, %ymm0, %ymm01374; AVX512BW-NEXT: retq1375;1376; AVX512VLBW-LABEL: constant_funnnel_v32i8:1377; AVX512VLBW: # %bb.0:1378; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]1379; AVX512VLBW-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm11380; AVX512VLBW-NEXT: vpsrlw $8, %ymm1, %ymm11381; AVX512VLBW-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]1382; AVX512VLBW-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01383; AVX512VLBW-NEXT: vpsrlw $8, %ymm0, %ymm01384; AVX512VLBW-NEXT: vpackuswb %ymm1, %ymm0, %ymm01385; AVX512VLBW-NEXT: retq1386;1387; AVX512VBMI2-LABEL: constant_funnnel_v32i8:1388; AVX512VBMI2: # %bb.0:1389; AVX512VBMI2-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [0,0,1,0,2,0,3,0,4,0,5,0,6,0,7,0,0,0,1,0,2,0,3,0,4,0,5,0,6,0,7,0]1390; AVX512VBMI2-NEXT: # ymm1 = mem[0,1,0,1]1391; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]1392; AVX512VBMI2-NEXT: vpsllvw %zmm1, %zmm2, %zmm11393; AVX512VBMI2-NEXT: vpsrlw $8, %ymm1, %ymm11394; AVX512VBMI2-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [0,0,7,0,6,0,5,0,4,0,3,0,2,0,1,0,0,0,7,0,6,0,5,0,4,0,3,0,2,0,1,0]1395; AVX512VBMI2-NEXT: # ymm2 = mem[0,1,0,1]1396; AVX512VBMI2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]1397; AVX512VBMI2-NEXT: vpsllvw %zmm2, %zmm0, %zmm01398; AVX512VBMI2-NEXT: vpsrlw $8, %ymm0, %ymm01399; AVX512VBMI2-NEXT: vpackuswb %ymm1, %ymm0, %ymm01400; AVX512VBMI2-NEXT: retq1401;1402; AVX512VLVBMI2-LABEL: constant_funnnel_v32i8:1403; AVX512VLVBMI2: # %bb.0:1404; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]1405; AVX512VLVBMI2-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm11406; AVX512VLVBMI2-NEXT: vpsrlw $8, %ymm1, %ymm11407; AVX512VLVBMI2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]1408; AVX512VLVBMI2-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01409; AVX512VLVBMI2-NEXT: vpsrlw $8, %ymm0, %ymm01410; AVX512VLVBMI2-NEXT: vpackuswb %ymm1, %ymm0, %ymm01411; AVX512VLVBMI2-NEXT: retq1412;1413; XOPAVX1-LABEL: constant_funnnel_v32i8:1414; XOPAVX1: # %bb.0:1415; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm11416; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [0,7,6,5,4,3,2,1,0,1,2,3,4,5,6,7]1417; XOPAVX1-NEXT: vprotb %xmm2, %xmm1, %xmm11418; XOPAVX1-NEXT: vprotb %xmm2, %xmm0, %xmm01419; XOPAVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm01420; XOPAVX1-NEXT: retq1421;1422; XOPAVX2-LABEL: constant_funnnel_v32i8:1423; XOPAVX2: # %bb.0:1424; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm11425; XOPAVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [0,7,6,5,4,3,2,1,0,1,2,3,4,5,6,7]1426; XOPAVX2-NEXT: vprotb %xmm2, %xmm1, %xmm11427; XOPAVX2-NEXT: vprotb %xmm2, %xmm0, %xmm01428; XOPAVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm01429; XOPAVX2-NEXT: retq1430 %res = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %x, <32 x i8> %x, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1>)1431 ret <32 x i8> %res1432}1433 1434;1435; Uniform Constant Shifts1436;1437 1438define <4 x i64> @splatconstant_funnnel_v4i64(<4 x i64> %x) nounwind {1439; AVX1-LABEL: splatconstant_funnnel_v4i64:1440; AVX1: # %bb.0:1441; AVX1-NEXT: vpsllq $50, %xmm0, %xmm11442; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm21443; AVX1-NEXT: vpsllq $50, %xmm2, %xmm31444; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm11445; AVX1-NEXT: vpsrlq $14, %xmm0, %xmm01446; AVX1-NEXT: vpsrlq $14, %xmm2, %xmm21447; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm01448; AVX1-NEXT: vorps %ymm1, %ymm0, %ymm01449; AVX1-NEXT: retq1450;1451; AVX2-LABEL: splatconstant_funnnel_v4i64:1452; AVX2: # %bb.0:1453; AVX2-NEXT: vpsllq $50, %ymm0, %ymm11454; AVX2-NEXT: vpsrlq $14, %ymm0, %ymm01455; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm01456; AVX2-NEXT: retq1457;1458; AVX512F-LABEL: splatconstant_funnnel_v4i64:1459; AVX512F: # %bb.0:1460; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm01461; AVX512F-NEXT: vprorq $14, %zmm0, %zmm01462; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01463; AVX512F-NEXT: retq1464;1465; AVX512VL-LABEL: splatconstant_funnnel_v4i64:1466; AVX512VL: # %bb.0:1467; AVX512VL-NEXT: vprorq $14, %ymm0, %ymm01468; AVX512VL-NEXT: retq1469;1470; AVX512BW-LABEL: splatconstant_funnnel_v4i64:1471; AVX512BW: # %bb.0:1472; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm01473; AVX512BW-NEXT: vprorq $14, %zmm0, %zmm01474; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01475; AVX512BW-NEXT: retq1476;1477; AVX512VLBW-LABEL: splatconstant_funnnel_v4i64:1478; AVX512VLBW: # %bb.0:1479; AVX512VLBW-NEXT: vprorq $14, %ymm0, %ymm01480; AVX512VLBW-NEXT: retq1481;1482; AVX512VBMI2-LABEL: splatconstant_funnnel_v4i64:1483; AVX512VBMI2: # %bb.0:1484; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm01485; AVX512VBMI2-NEXT: vprorq $14, %zmm0, %zmm01486; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01487; AVX512VBMI2-NEXT: retq1488;1489; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v4i64:1490; AVX512VLVBMI2: # %bb.0:1491; AVX512VLVBMI2-NEXT: vprorq $14, %ymm0, %ymm01492; AVX512VLVBMI2-NEXT: retq1493;1494; XOPAVX1-LABEL: splatconstant_funnnel_v4i64:1495; XOPAVX1: # %bb.0:1496; XOPAVX1-NEXT: vprotq $50, %xmm0, %xmm11497; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm01498; XOPAVX1-NEXT: vprotq $50, %xmm0, %xmm01499; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm01500; XOPAVX1-NEXT: retq1501;1502; XOPAVX2-LABEL: splatconstant_funnnel_v4i64:1503; XOPAVX2: # %bb.0:1504; XOPAVX2-NEXT: vprotq $50, %xmm0, %xmm11505; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm01506; XOPAVX2-NEXT: vprotq $50, %xmm0, %xmm01507; XOPAVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm01508; XOPAVX2-NEXT: retq1509 %res = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %x, <4 x i64> %x, <4 x i64> <i64 14, i64 14, i64 14, i64 14>)1510 ret <4 x i64> %res1511}1512 1513define <8 x i32> @splatconstant_funnnel_v8i32(<8 x i32> %x) nounwind {1514; AVX1-LABEL: splatconstant_funnnel_v8i32:1515; AVX1: # %bb.0:1516; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm11517; AVX1-NEXT: vpsrld $4, %xmm1, %xmm21518; AVX1-NEXT: vpslld $28, %xmm1, %xmm11519; AVX1-NEXT: vpor %xmm2, %xmm1, %xmm11520; AVX1-NEXT: vpsrld $4, %xmm0, %xmm21521; AVX1-NEXT: vpslld $28, %xmm0, %xmm01522; AVX1-NEXT: vpor %xmm2, %xmm0, %xmm01523; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm01524; AVX1-NEXT: retq1525;1526; AVX2-LABEL: splatconstant_funnnel_v8i32:1527; AVX2: # %bb.0:1528; AVX2-NEXT: vpsrld $4, %ymm0, %ymm11529; AVX2-NEXT: vpslld $28, %ymm0, %ymm01530; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm01531; AVX2-NEXT: retq1532;1533; AVX512F-LABEL: splatconstant_funnnel_v8i32:1534; AVX512F: # %bb.0:1535; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm01536; AVX512F-NEXT: vprord $4, %zmm0, %zmm01537; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01538; AVX512F-NEXT: retq1539;1540; AVX512VL-LABEL: splatconstant_funnnel_v8i32:1541; AVX512VL: # %bb.0:1542; AVX512VL-NEXT: vprord $4, %ymm0, %ymm01543; AVX512VL-NEXT: retq1544;1545; AVX512BW-LABEL: splatconstant_funnnel_v8i32:1546; AVX512BW: # %bb.0:1547; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm01548; AVX512BW-NEXT: vprord $4, %zmm0, %zmm01549; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01550; AVX512BW-NEXT: retq1551;1552; AVX512VLBW-LABEL: splatconstant_funnnel_v8i32:1553; AVX512VLBW: # %bb.0:1554; AVX512VLBW-NEXT: vprord $4, %ymm0, %ymm01555; AVX512VLBW-NEXT: retq1556;1557; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i32:1558; AVX512VBMI2: # %bb.0:1559; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm01560; AVX512VBMI2-NEXT: vprord $4, %zmm0, %zmm01561; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01562; AVX512VBMI2-NEXT: retq1563;1564; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v8i32:1565; AVX512VLVBMI2: # %bb.0:1566; AVX512VLVBMI2-NEXT: vprord $4, %ymm0, %ymm01567; AVX512VLVBMI2-NEXT: retq1568;1569; XOPAVX1-LABEL: splatconstant_funnnel_v8i32:1570; XOPAVX1: # %bb.0:1571; XOPAVX1-NEXT: vprotd $28, %xmm0, %xmm11572; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm01573; XOPAVX1-NEXT: vprotd $28, %xmm0, %xmm01574; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm01575; XOPAVX1-NEXT: retq1576;1577; XOPAVX2-LABEL: splatconstant_funnnel_v8i32:1578; XOPAVX2: # %bb.0:1579; XOPAVX2-NEXT: vprotd $28, %xmm0, %xmm11580; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm01581; XOPAVX2-NEXT: vprotd $28, %xmm0, %xmm01582; XOPAVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm01583; XOPAVX2-NEXT: retq1584 %res = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %x, <8 x i32> %x, <8 x i32> <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>)1585 ret <8 x i32> %res1586}1587 1588define <16 x i16> @splatconstant_funnnel_v16i16(<16 x i16> %x) nounwind {1589; AVX1-LABEL: splatconstant_funnnel_v16i16:1590; AVX1: # %bb.0:1591; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm11592; AVX1-NEXT: vpsrlw $7, %xmm1, %xmm21593; AVX1-NEXT: vpsllw $9, %xmm1, %xmm11594; AVX1-NEXT: vpor %xmm2, %xmm1, %xmm11595; AVX1-NEXT: vpsrlw $7, %xmm0, %xmm21596; AVX1-NEXT: vpsllw $9, %xmm0, %xmm01597; AVX1-NEXT: vpor %xmm2, %xmm0, %xmm01598; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm01599; AVX1-NEXT: retq1600;1601; AVX2-LABEL: splatconstant_funnnel_v16i16:1602; AVX2: # %bb.0:1603; AVX2-NEXT: vpsrlw $7, %ymm0, %ymm11604; AVX2-NEXT: vpsllw $9, %ymm0, %ymm01605; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm01606; AVX2-NEXT: retq1607;1608; AVX512F-LABEL: splatconstant_funnnel_v16i16:1609; AVX512F: # %bb.0:1610; AVX512F-NEXT: vpsrlw $7, %ymm0, %ymm11611; AVX512F-NEXT: vpsllw $9, %ymm0, %ymm01612; AVX512F-NEXT: vpor %ymm1, %ymm0, %ymm01613; AVX512F-NEXT: retq1614;1615; AVX512VL-LABEL: splatconstant_funnnel_v16i16:1616; AVX512VL: # %bb.0:1617; AVX512VL-NEXT: vpsrlw $7, %ymm0, %ymm11618; AVX512VL-NEXT: vpsllw $9, %ymm0, %ymm01619; AVX512VL-NEXT: vpor %ymm1, %ymm0, %ymm01620; AVX512VL-NEXT: retq1621;1622; AVX512BW-LABEL: splatconstant_funnnel_v16i16:1623; AVX512BW: # %bb.0:1624; AVX512BW-NEXT: vpsrlw $7, %ymm0, %ymm11625; AVX512BW-NEXT: vpsllw $9, %ymm0, %ymm01626; AVX512BW-NEXT: vpor %ymm1, %ymm0, %ymm01627; AVX512BW-NEXT: retq1628;1629; AVX512VLBW-LABEL: splatconstant_funnnel_v16i16:1630; AVX512VLBW: # %bb.0:1631; AVX512VLBW-NEXT: vpsrlw $7, %ymm0, %ymm11632; AVX512VLBW-NEXT: vpsllw $9, %ymm0, %ymm01633; AVX512VLBW-NEXT: vpor %ymm1, %ymm0, %ymm01634; AVX512VLBW-NEXT: retq1635;1636; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i16:1637; AVX512VBMI2: # %bb.0:1638; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm01639; AVX512VBMI2-NEXT: vpshrdw $7, %zmm0, %zmm0, %zmm01640; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01641; AVX512VBMI2-NEXT: retq1642;1643; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i16:1644; AVX512VLVBMI2: # %bb.0:1645; AVX512VLVBMI2-NEXT: vpshrdw $7, %ymm0, %ymm0, %ymm01646; AVX512VLVBMI2-NEXT: retq1647;1648; XOPAVX1-LABEL: splatconstant_funnnel_v16i16:1649; XOPAVX1: # %bb.0:1650; XOPAVX1-NEXT: vprotw $9, %xmm0, %xmm11651; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm01652; XOPAVX1-NEXT: vprotw $9, %xmm0, %xmm01653; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm01654; XOPAVX1-NEXT: retq1655;1656; XOPAVX2-LABEL: splatconstant_funnnel_v16i16:1657; XOPAVX2: # %bb.0:1658; XOPAVX2-NEXT: vprotw $9, %xmm0, %xmm11659; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm01660; XOPAVX2-NEXT: vprotw $9, %xmm0, %xmm01661; XOPAVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm01662; XOPAVX2-NEXT: retq1663 %res = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %x, <16 x i16> %x, <16 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>)1664 ret <16 x i16> %res1665}1666 1667define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind {1668; AVX1-LABEL: splatconstant_funnnel_v32i8:1669; AVX1: # %bb.0:1670; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm11671; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm21672; AVX1-NEXT: vpsrlw $4, %xmm2, %xmm31673; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm11674; AVX1-NEXT: vbroadcastss {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]1675; AVX1-NEXT: vandnps %ymm1, %ymm3, %ymm11676; AVX1-NEXT: vpsllw $4, %xmm0, %xmm01677; AVX1-NEXT: vpsllw $4, %xmm2, %xmm21678; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm01679; AVX1-NEXT: vandps %ymm3, %ymm0, %ymm01680; AVX1-NEXT: vorps %ymm1, %ymm0, %ymm01681; AVX1-NEXT: retq1682;1683; AVX2-LABEL: splatconstant_funnnel_v32i8:1684; AVX2: # %bb.0:1685; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm11686; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm11687; AVX2-NEXT: vpsllw $4, %ymm0, %ymm01688; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01689; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm01690; AVX2-NEXT: retq1691;1692; AVX512F-LABEL: splatconstant_funnnel_v32i8:1693; AVX512F: # %bb.0:1694; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm11695; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm01696; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))1697; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01698; AVX512F-NEXT: retq1699;1700; AVX512VL-LABEL: splatconstant_funnnel_v32i8:1701; AVX512VL: # %bb.0:1702; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm11703; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm01704; AVX512VL-NEXT: vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))1705; AVX512VL-NEXT: retq1706;1707; AVX512BW-LABEL: splatconstant_funnnel_v32i8:1708; AVX512BW: # %bb.0:1709; AVX512BW-NEXT: vpsllw $4, %ymm0, %ymm11710; AVX512BW-NEXT: vpsrlw $4, %ymm0, %ymm01711; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))1712; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01713; AVX512BW-NEXT: retq1714;1715; AVX512VLBW-LABEL: splatconstant_funnnel_v32i8:1716; AVX512VLBW: # %bb.0:1717; AVX512VLBW-NEXT: vpsllw $4, %ymm0, %ymm11718; AVX512VLBW-NEXT: vpsrlw $4, %ymm0, %ymm01719; AVX512VLBW-NEXT: vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))1720; AVX512VLBW-NEXT: retq1721;1722; AVX512VBMI2-LABEL: splatconstant_funnnel_v32i8:1723; AVX512VBMI2: # %bb.0:1724; AVX512VBMI2-NEXT: vpsllw $4, %ymm0, %ymm11725; AVX512VBMI2-NEXT: vpsrlw $4, %ymm0, %ymm01726; AVX512VBMI2-NEXT: vpternlogd {{.*#+}} zmm0 = zmm0 ^ (m32bcst & (zmm0 ^ zmm1))1727; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01728; AVX512VBMI2-NEXT: retq1729;1730; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v32i8:1731; AVX512VLVBMI2: # %bb.0:1732; AVX512VLVBMI2-NEXT: vpsllw $4, %ymm0, %ymm11733; AVX512VLVBMI2-NEXT: vpsrlw $4, %ymm0, %ymm01734; AVX512VLVBMI2-NEXT: vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))1735; AVX512VLVBMI2-NEXT: retq1736;1737; XOPAVX1-LABEL: splatconstant_funnnel_v32i8:1738; XOPAVX1: # %bb.0:1739; XOPAVX1-NEXT: vprotb $4, %xmm0, %xmm11740; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm01741; XOPAVX1-NEXT: vprotb $4, %xmm0, %xmm01742; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm01743; XOPAVX1-NEXT: retq1744;1745; XOPAVX2-LABEL: splatconstant_funnnel_v32i8:1746; XOPAVX2: # %bb.0:1747; XOPAVX2-NEXT: vprotb $4, %xmm0, %xmm11748; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm01749; XOPAVX2-NEXT: vprotb $4, %xmm0, %xmm01750; XOPAVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm01751; XOPAVX2-NEXT: retq1752 %res = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %x, <32 x i8> %x, <32 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)1753 ret <32 x i8> %res1754}1755