437 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=i686-- -mattr=sse2 | FileCheck %s --check-prefixes=CHECK,X86-SSE23; RUN: llc < %s -mtriple=x86_64-- -mattr=avx2 | FileCheck %s --check-prefixes=CHECK,X64-AVX24 5declare i8 @llvm.fshl.i8(i8, i8, i8)6declare i16 @llvm.fshl.i16(i16, i16, i16)7declare i32 @llvm.fshl.i32(i32, i32, i32)8declare i64 @llvm.fshl.i64(i64, i64, i64)9declare <4 x i32> @llvm.fshl.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)10 11declare i8 @llvm.fshr.i8(i8, i8, i8)12declare i16 @llvm.fshr.i16(i16, i16, i16)13declare i32 @llvm.fshr.i32(i32, i32, i32)14declare i64 @llvm.fshr.i64(i64, i64, i64)15declare <4 x i32> @llvm.fshr.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)16 17; When first 2 operands match, it's a rotate.18 19define i8 @rotl_i8_const_shift(i8 %x) nounwind {20; X86-SSE2-LABEL: rotl_i8_const_shift:21; X86-SSE2: # %bb.0:22; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax23; X86-SSE2-NEXT: rolb $3, %al24; X86-SSE2-NEXT: retl25;26; X64-AVX2-LABEL: rotl_i8_const_shift:27; X64-AVX2: # %bb.0:28; X64-AVX2-NEXT: movl %edi, %eax29; X64-AVX2-NEXT: rolb $3, %al30; X64-AVX2-NEXT: # kill: def $al killed $al killed $eax31; X64-AVX2-NEXT: retq32 %f = call i8 @llvm.fshl.i8(i8 %x, i8 %x, i8 3)33 ret i8 %f34}35 36define i8 @rotl_i8_const_shift1(i8 %x) nounwind {37; X86-SSE2-LABEL: rotl_i8_const_shift1:38; X86-SSE2: # %bb.0:39; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax40; X86-SSE2-NEXT: rolb %al41; X86-SSE2-NEXT: retl42;43; X64-AVX2-LABEL: rotl_i8_const_shift1:44; X64-AVX2: # %bb.0:45; X64-AVX2-NEXT: movl %edi, %eax46; X64-AVX2-NEXT: rolb %al47; X64-AVX2-NEXT: # kill: def $al killed $al killed $eax48; X64-AVX2-NEXT: retq49 %f = call i8 @llvm.fshl.i8(i8 %x, i8 %x, i8 1)50 ret i8 %f51}52 53define i8 @rotl_i8_const_shift7(i8 %x) nounwind {54; X86-SSE2-LABEL: rotl_i8_const_shift7:55; X86-SSE2: # %bb.0:56; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax57; X86-SSE2-NEXT: rorb %al58; X86-SSE2-NEXT: retl59;60; X64-AVX2-LABEL: rotl_i8_const_shift7:61; X64-AVX2: # %bb.0:62; X64-AVX2-NEXT: movl %edi, %eax63; X64-AVX2-NEXT: rorb %al64; X64-AVX2-NEXT: # kill: def $al killed $al killed $eax65; X64-AVX2-NEXT: retq66 %f = call i8 @llvm.fshl.i8(i8 %x, i8 %x, i8 7)67 ret i8 %f68}69 70define i64 @rotl_i64_const_shift(i64 %x) nounwind {71; X86-SSE2-LABEL: rotl_i64_const_shift:72; X86-SSE2: # %bb.0:73; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx74; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edx75; X86-SSE2-NEXT: movl %ecx, %eax76; X86-SSE2-NEXT: shldl $3, %edx, %eax77; X86-SSE2-NEXT: shldl $3, %ecx, %edx78; X86-SSE2-NEXT: retl79;80; X64-AVX2-LABEL: rotl_i64_const_shift:81; X64-AVX2: # %bb.0:82; X64-AVX2-NEXT: movq %rdi, %rax83; X64-AVX2-NEXT: rolq $3, %rax84; X64-AVX2-NEXT: retq85 %f = call i64 @llvm.fshl.i64(i64 %x, i64 %x, i64 3)86 ret i64 %f87}88 89define i16 @rotl_i16(i16 %x, i16 %z) nounwind {90; X86-SSE2-LABEL: rotl_i16:91; X86-SSE2: # %bb.0:92; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %ecx93; X86-SSE2-NEXT: movzwl {{[0-9]+}}(%esp), %eax94; X86-SSE2-NEXT: rolw %cl, %ax95; X86-SSE2-NEXT: retl96;97; X64-AVX2-LABEL: rotl_i16:98; X64-AVX2: # %bb.0:99; X64-AVX2-NEXT: movl %esi, %ecx100; X64-AVX2-NEXT: movl %edi, %eax101; X64-AVX2-NEXT: # kill: def $cl killed $cl killed $ecx102; X64-AVX2-NEXT: rolw %cl, %ax103; X64-AVX2-NEXT: # kill: def $ax killed $ax killed $eax104; X64-AVX2-NEXT: retq105 %f = call i16 @llvm.fshl.i16(i16 %x, i16 %x, i16 %z)106 ret i16 %f107}108 109define i32 @rotl_i32(i32 %x, i32 %z) nounwind {110; X86-SSE2-LABEL: rotl_i32:111; X86-SSE2: # %bb.0:112; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %ecx113; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax114; X86-SSE2-NEXT: roll %cl, %eax115; X86-SSE2-NEXT: retl116;117; X64-AVX2-LABEL: rotl_i32:118; X64-AVX2: # %bb.0:119; X64-AVX2-NEXT: movl %esi, %ecx120; X64-AVX2-NEXT: movl %edi, %eax121; X64-AVX2-NEXT: # kill: def $cl killed $cl killed $ecx122; X64-AVX2-NEXT: roll %cl, %eax123; X64-AVX2-NEXT: retq124 %f = call i32 @llvm.fshl.i32(i32 %x, i32 %x, i32 %z)125 ret i32 %f126}127 128; Vector rotate.129 130define <4 x i32> @rotl_v4i32(<4 x i32> %x, <4 x i32> %z) nounwind {131; X86-SSE2-LABEL: rotl_v4i32:132; X86-SSE2: # %bb.0:133; X86-SSE2-NEXT: pslld $23, %xmm1134; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1135; X86-SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1136; X86-SSE2-NEXT: cvttps2dq %xmm1, %xmm1137; X86-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]138; X86-SSE2-NEXT: pmuludq %xmm1, %xmm0139; X86-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]140; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]141; X86-SSE2-NEXT: pmuludq %xmm2, %xmm1142; X86-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]143; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]144; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]145; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]146; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]147; X86-SSE2-NEXT: por %xmm3, %xmm0148; X86-SSE2-NEXT: retl149;150; X64-AVX2-LABEL: rotl_v4i32:151; X64-AVX2: # %bb.0:152; X64-AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]153; X64-AVX2-NEXT: vpand %xmm2, %xmm1, %xmm1154; X64-AVX2-NEXT: vpsllvd %xmm1, %xmm0, %xmm2155; X64-AVX2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [32,32,32,32]156; X64-AVX2-NEXT: vpsubd %xmm1, %xmm3, %xmm1157; X64-AVX2-NEXT: vpsrlvd %xmm1, %xmm0, %xmm0158; X64-AVX2-NEXT: vpor %xmm0, %xmm2, %xmm0159; X64-AVX2-NEXT: retq160 %f = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> %z)161 ret <4 x i32> %f162}163 164; Vector rotate by constant splat amount.165 166define <4 x i32> @rotl_v4i32_const_shift(<4 x i32> %x) nounwind {167; X86-SSE2-LABEL: rotl_v4i32_const_shift:168; X86-SSE2: # %bb.0:169; X86-SSE2-NEXT: movdqa %xmm0, %xmm1170; X86-SSE2-NEXT: psrld $29, %xmm1171; X86-SSE2-NEXT: pslld $3, %xmm0172; X86-SSE2-NEXT: por %xmm1, %xmm0173; X86-SSE2-NEXT: retl174;175; X64-AVX2-LABEL: rotl_v4i32_const_shift:176; X64-AVX2: # %bb.0:177; X64-AVX2-NEXT: vpsrld $29, %xmm0, %xmm1178; X64-AVX2-NEXT: vpslld $3, %xmm0, %xmm0179; X64-AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0180; X64-AVX2-NEXT: retq181 %f = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> <i32 3, i32 3, i32 3, i32 3>)182 ret <4 x i32> %f183}184 185; Repeat everything for funnel shift right.186 187define i8 @rotr_i8_const_shift(i8 %x) nounwind {188; X86-SSE2-LABEL: rotr_i8_const_shift:189; X86-SSE2: # %bb.0:190; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax191; X86-SSE2-NEXT: rorb $3, %al192; X86-SSE2-NEXT: retl193;194; X64-AVX2-LABEL: rotr_i8_const_shift:195; X64-AVX2: # %bb.0:196; X64-AVX2-NEXT: movl %edi, %eax197; X64-AVX2-NEXT: rorb $3, %al198; X64-AVX2-NEXT: # kill: def $al killed $al killed $eax199; X64-AVX2-NEXT: retq200 %f = call i8 @llvm.fshr.i8(i8 %x, i8 %x, i8 3)201 ret i8 %f202}203 204define i8 @rotr_i8_const_shift1(i8 %x) nounwind {205; X86-SSE2-LABEL: rotr_i8_const_shift1:206; X86-SSE2: # %bb.0:207; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax208; X86-SSE2-NEXT: rorb %al209; X86-SSE2-NEXT: retl210;211; X64-AVX2-LABEL: rotr_i8_const_shift1:212; X64-AVX2: # %bb.0:213; X64-AVX2-NEXT: movl %edi, %eax214; X64-AVX2-NEXT: rorb %al215; X64-AVX2-NEXT: # kill: def $al killed $al killed $eax216; X64-AVX2-NEXT: retq217 %f = call i8 @llvm.fshr.i8(i8 %x, i8 %x, i8 1)218 ret i8 %f219}220 221define i8 @rotr_i8_const_shift7(i8 %x) nounwind {222; X86-SSE2-LABEL: rotr_i8_const_shift7:223; X86-SSE2: # %bb.0:224; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %eax225; X86-SSE2-NEXT: rolb %al226; X86-SSE2-NEXT: retl227;228; X64-AVX2-LABEL: rotr_i8_const_shift7:229; X64-AVX2: # %bb.0:230; X64-AVX2-NEXT: movl %edi, %eax231; X64-AVX2-NEXT: rolb %al232; X64-AVX2-NEXT: # kill: def $al killed $al killed $eax233; X64-AVX2-NEXT: retq234 %f = call i8 @llvm.fshr.i8(i8 %x, i8 %x, i8 7)235 ret i8 %f236}237 238define i32 @rotr_i32_const_shift(i32 %x) nounwind {239; X86-SSE2-LABEL: rotr_i32_const_shift:240; X86-SSE2: # %bb.0:241; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax242; X86-SSE2-NEXT: rorl $3, %eax243; X86-SSE2-NEXT: retl244;245; X64-AVX2-LABEL: rotr_i32_const_shift:246; X64-AVX2: # %bb.0:247; X64-AVX2-NEXT: movl %edi, %eax248; X64-AVX2-NEXT: rorl $3, %eax249; X64-AVX2-NEXT: retq250 %f = call i32 @llvm.fshr.i32(i32 %x, i32 %x, i32 3)251 ret i32 %f252}253 254; When first 2 operands match, it's a rotate (by variable amount).255 256define i16 @rotr_i16(i16 %x, i16 %z) nounwind {257; X86-SSE2-LABEL: rotr_i16:258; X86-SSE2: # %bb.0:259; X86-SSE2-NEXT: movzbl {{[0-9]+}}(%esp), %ecx260; X86-SSE2-NEXT: movzwl {{[0-9]+}}(%esp), %eax261; X86-SSE2-NEXT: rorw %cl, %ax262; X86-SSE2-NEXT: retl263;264; X64-AVX2-LABEL: rotr_i16:265; X64-AVX2: # %bb.0:266; X64-AVX2-NEXT: movl %esi, %ecx267; X64-AVX2-NEXT: movl %edi, %eax268; X64-AVX2-NEXT: # kill: def $cl killed $cl killed $ecx269; X64-AVX2-NEXT: rorw %cl, %ax270; X64-AVX2-NEXT: # kill: def $ax killed $ax killed $eax271; X64-AVX2-NEXT: retq272 %f = call i16 @llvm.fshr.i16(i16 %x, i16 %x, i16 %z)273 ret i16 %f274}275 276define i64 @rotr_i64(i64 %x, i64 %z) nounwind {277; X86-SSE2-LABEL: rotr_i64:278; X86-SSE2: # %bb.0:279; X86-SSE2-NEXT: pushl %esi280; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax281; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %esi282; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx283; X86-SSE2-NEXT: testb $32, %cl284; X86-SSE2-NEXT: movl %eax, %edx285; X86-SSE2-NEXT: cmovel %esi, %edx286; X86-SSE2-NEXT: cmovel %eax, %esi287; X86-SSE2-NEXT: movl %esi, %eax288; X86-SSE2-NEXT: shrdl %cl, %edx, %eax289; X86-SSE2-NEXT: # kill: def $cl killed $cl killed $ecx290; X86-SSE2-NEXT: shrdl %cl, %esi, %edx291; X86-SSE2-NEXT: popl %esi292; X86-SSE2-NEXT: retl293;294; X64-AVX2-LABEL: rotr_i64:295; X64-AVX2: # %bb.0:296; X64-AVX2-NEXT: movq %rsi, %rcx297; X64-AVX2-NEXT: movq %rdi, %rax298; X64-AVX2-NEXT: # kill: def $cl killed $cl killed $rcx299; X64-AVX2-NEXT: rorq %cl, %rax300; X64-AVX2-NEXT: retq301 %f = call i64 @llvm.fshr.i64(i64 %x, i64 %x, i64 %z)302 ret i64 %f303}304 305; Vector rotate.306 307define <4 x i32> @rotr_v4i32(<4 x i32> %x, <4 x i32> %z) nounwind {308; X86-SSE2-LABEL: rotr_v4i32:309; X86-SSE2: # %bb.0:310; X86-SSE2-NEXT: pxor %xmm2, %xmm2311; X86-SSE2-NEXT: psubd %xmm1, %xmm2312; X86-SSE2-NEXT: pslld $23, %xmm2313; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm2314; X86-SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm2315; X86-SSE2-NEXT: cvttps2dq %xmm2, %xmm1316; X86-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]317; X86-SSE2-NEXT: pmuludq %xmm1, %xmm0318; X86-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]319; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]320; X86-SSE2-NEXT: pmuludq %xmm2, %xmm1321; X86-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]322; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]323; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]324; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]325; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]326; X86-SSE2-NEXT: por %xmm3, %xmm0327; X86-SSE2-NEXT: retl328;329; X64-AVX2-LABEL: rotr_v4i32:330; X64-AVX2: # %bb.0:331; X64-AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]332; X64-AVX2-NEXT: vpand %xmm2, %xmm1, %xmm1333; X64-AVX2-NEXT: vpsrlvd %xmm1, %xmm0, %xmm2334; X64-AVX2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [32,32,32,32]335; X64-AVX2-NEXT: vpsubd %xmm1, %xmm3, %xmm1336; X64-AVX2-NEXT: vpsllvd %xmm1, %xmm0, %xmm0337; X64-AVX2-NEXT: vpor %xmm0, %xmm2, %xmm0338; X64-AVX2-NEXT: retq339 %f = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> %z)340 ret <4 x i32> %f341}342 343; Vector rotate by constant splat amount.344 345define <4 x i32> @rotr_v4i32_const_shift(<4 x i32> %x) nounwind {346; X86-SSE2-LABEL: rotr_v4i32_const_shift:347; X86-SSE2: # %bb.0:348; X86-SSE2-NEXT: movdqa %xmm0, %xmm1349; X86-SSE2-NEXT: psrld $3, %xmm1350; X86-SSE2-NEXT: pslld $29, %xmm0351; X86-SSE2-NEXT: por %xmm1, %xmm0352; X86-SSE2-NEXT: retl353;354; X64-AVX2-LABEL: rotr_v4i32_const_shift:355; X64-AVX2: # %bb.0:356; X64-AVX2-NEXT: vpsrld $3, %xmm0, %xmm1357; X64-AVX2-NEXT: vpslld $29, %xmm0, %xmm0358; X64-AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0359; X64-AVX2-NEXT: retq360 %f = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> <i32 3, i32 3, i32 3, i32 3>)361 ret <4 x i32> %f362}363 364define i32 @rotl_i32_shift_by_bitwidth(i32 %x) nounwind {365; X86-SSE2-LABEL: rotl_i32_shift_by_bitwidth:366; X86-SSE2: # %bb.0:367; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax368; X86-SSE2-NEXT: retl369;370; X64-AVX2-LABEL: rotl_i32_shift_by_bitwidth:371; X64-AVX2: # %bb.0:372; X64-AVX2-NEXT: movl %edi, %eax373; X64-AVX2-NEXT: retq374 %f = call i32 @llvm.fshl.i32(i32 %x, i32 %x, i32 32)375 ret i32 %f376}377 378define i32 @rotr_i32_shift_by_bitwidth(i32 %x) nounwind {379; X86-SSE2-LABEL: rotr_i32_shift_by_bitwidth:380; X86-SSE2: # %bb.0:381; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax382; X86-SSE2-NEXT: retl383;384; X64-AVX2-LABEL: rotr_i32_shift_by_bitwidth:385; X64-AVX2: # %bb.0:386; X64-AVX2-NEXT: movl %edi, %eax387; X64-AVX2-NEXT: retq388 %f = call i32 @llvm.fshr.i32(i32 %x, i32 %x, i32 32)389 ret i32 %f390}391 392define <4 x i32> @rotl_v4i32_shift_by_bitwidth(<4 x i32> %x) nounwind {393; CHECK-LABEL: rotl_v4i32_shift_by_bitwidth:394; CHECK: # %bb.0:395; CHECK-NEXT: ret{{[l|q]}}396 %f = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> <i32 32, i32 32, i32 32, i32 32>)397 ret <4 x i32> %f398}399 400define <4 x i32> @rotr_v4i32_shift_by_bitwidth(<4 x i32> %x) nounwind {401; CHECK-LABEL: rotr_v4i32_shift_by_bitwidth:402; CHECK: # %bb.0:403; CHECK-NEXT: ret{{[l|q]}}404 %f = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> <i32 32, i32 32, i32 32, i32 32>)405 ret <4 x i32> %f406}407 408; Non power-of-2 types can't use the negated shift amount to avoid a select.409 410declare i7 @llvm.fshl.i7(i7, i7, i7)411declare i7 @llvm.fshr.i7(i7, i7, i7)412 413; extract(concat(0b1110000, 0b1110000) << 9) = 0b1000011414; Try an oversized shift to test modulo functionality.415 416define i7 @fshl_i7() {417; CHECK-LABEL: fshl_i7:418; CHECK: # %bb.0:419; CHECK-NEXT: movb $67, %al420; CHECK-NEXT: ret{{[l|q]}}421 %f = call i7 @llvm.fshl.i7(i7 112, i7 112, i7 9)422 ret i7 %f423}424 425; extract(concat(0b1110001, 0b1110001) >> 16) = 0b0111100426; Try an oversized shift to test modulo functionality.427 428define i7 @fshr_i7() {429; CHECK-LABEL: fshr_i7:430; CHECK: # %bb.0:431; CHECK-NEXT: movb $60, %al432; CHECK-NEXT: ret{{[l|q]}}433 %f = call i7 @llvm.fshr.i7(i7 113, i7 113, i7 16)434 ret i7 %f435}436 437