318 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=CHECK,X863; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=CHECK,X644 5; Check that under certain conditions we can factor out a rotate6; from the following idioms:7; (a*c0) >> s1 | (a*c1)8; (a/c0) << s1 | (a/c1)9; This targets cases where instcombine has folded a shl/srl/mul/udiv10; with one of the shifts from the rotate idiom11 12define <4 x i32> @vroll_v4i32_extract_shl(<4 x i32> %i) {13; CHECK-LABEL: vroll_v4i32_extract_shl:14; CHECK: # %bb.0:15; CHECK-NEXT: vpslld $3, %xmm0, %xmm016; CHECK-NEXT: vprold $7, %xmm0, %xmm017; CHECK-NEXT: ret{{[l|q]}}18 %lhs_mul = shl <4 x i32> %i, <i32 3, i32 3, i32 3, i32 3>19 %rhs_mul = shl <4 x i32> %i, <i32 10, i32 10, i32 10, i32 10>20 %lhs_shift = lshr <4 x i32> %lhs_mul, <i32 25, i32 25, i32 25, i32 25>21 %out = or <4 x i32> %lhs_shift, %rhs_mul22 ret <4 x i32> %out23}24 25define <4 x i64> @vrolq_v4i64_extract_shrl(<4 x i64> %i) nounwind {26; CHECK-LABEL: vrolq_v4i64_extract_shrl:27; CHECK: # %bb.0:28; CHECK-NEXT: vpsrlq $5, %ymm0, %ymm029; CHECK-NEXT: vprolq $29, %ymm0, %ymm030; CHECK-NEXT: ret{{[l|q]}}31 %lhs_div = lshr <4 x i64> %i, <i64 40, i64 40, i64 40, i64 40>32 %rhs_div = lshr <4 x i64> %i, <i64 5, i64 5, i64 5, i64 5>33 %rhs_shift = shl <4 x i64> %rhs_div, <i64 29, i64 29, i64 29, i64 29>34 %out = or <4 x i64> %lhs_div, %rhs_shift35 ret <4 x i64> %out36}37 38define <8 x i32> @vroll_extract_mul(<8 x i32> %i) nounwind {39; X86-LABEL: vroll_extract_mul:40; X86: # %bb.0:41; X86-NEXT: vpmulld {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %ymm0, %ymm042; X86-NEXT: vprold $6, %ymm0, %ymm043; X86-NEXT: retl44;45; X64-LABEL: vroll_extract_mul:46; X64: # %bb.0:47; X64-NEXT: vpmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm048; X64-NEXT: vprold $6, %ymm0, %ymm049; X64-NEXT: retq50 %lhs_mul = mul <8 x i32> %i, <i32 640, i32 640, i32 640, i32 640, i32 640, i32 640, i32 640, i32 640>51 %rhs_mul = mul <8 x i32> %i, <i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>52 %rhs_shift = lshr <8 x i32> %rhs_mul, <i32 26, i32 26, i32 26, i32 26, i32 26, i32 26, i32 26, i32 26>53 %out = or <8 x i32> %lhs_mul, %rhs_shift54 ret <8 x i32> %out55}56 57define <2 x i64> @vrolq_extract_udiv(<2 x i64> %i) nounwind {58; X86-LABEL: vrolq_extract_udiv:59; X86: # %bb.0:60; X86-NEXT: subl $32, %esp61; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill62; X86-NEXT: vextractps $1, %xmm0, {{[0-9]+}}(%esp)63; X86-NEXT: vmovss %xmm0, (%esp)64; X86-NEXT: movl $0, {{[0-9]+}}(%esp)65; X86-NEXT: movl $3, {{[0-9]+}}(%esp)66; X86-NEXT: calll __udivdi367; X86-NEXT: vmovups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload68; X86-NEXT: vextractps $3, %xmm0, {{[0-9]+}}(%esp)69; X86-NEXT: vextractps $2, %xmm0, (%esp)70; X86-NEXT: movl $0, {{[0-9]+}}(%esp)71; X86-NEXT: movl $3, {{[0-9]+}}(%esp)72; X86-NEXT: vmovd %eax, %xmm073; X86-NEXT: vpinsrd $1, %edx, %xmm0, %xmm074; X86-NEXT: vmovdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill75; X86-NEXT: calll __udivdi376; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload77; X86-NEXT: vpinsrd $2, %eax, %xmm0, %xmm078; X86-NEXT: vpinsrd $3, %edx, %xmm0, %xmm079; X86-NEXT: vprolq $57, %xmm0, %xmm080; X86-NEXT: addl $32, %esp81; X86-NEXT: retl82;83; X64-LABEL: vrolq_extract_udiv:84; X64: # %bb.0:85; X64-NEXT: vpextrq $1, %xmm0, %rax86; X64-NEXT: movabsq $-6148914691236517205, %rcx # imm = 0xAAAAAAAAAAAAAAAB87; X64-NEXT: mulq %rcx88; X64-NEXT: vmovq %rdx, %xmm189; X64-NEXT: vmovq %xmm0, %rax90; X64-NEXT: mulq %rcx91; X64-NEXT: vmovq %rdx, %xmm092; X64-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]93; X64-NEXT: vpsrlq $1, %xmm0, %xmm094; X64-NEXT: vprolq $57, %xmm0, %xmm095; X64-NEXT: retq96 %lhs_div = udiv <2 x i64> %i, <i64 3, i64 3>97 %rhs_div = udiv <2 x i64> %i, <i64 384, i64 384>98 %lhs_shift = shl <2 x i64> %lhs_div, <i64 57, i64 57>99 %out = or <2 x i64> %lhs_shift, %rhs_div100 ret <2 x i64> %out101}102 103define <4 x i32> @vrolw_extract_mul_with_mask(<4 x i32> %i) nounwind {104; X86-LABEL: vrolw_extract_mul_with_mask:105; X86: # %bb.0:106; X86-NEXT: vpslld $3, %xmm0, %xmm1107; X86-NEXT: vpaddd %xmm1, %xmm0, %xmm0108; X86-NEXT: vprold $7, %xmm0, %xmm0109; X86-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}{1to4}, %xmm0, %xmm0110; X86-NEXT: retl111;112; X64-LABEL: vrolw_extract_mul_with_mask:113; X64: # %bb.0:114; X64-NEXT: vpslld $3, %xmm0, %xmm1115; X64-NEXT: vpaddd %xmm1, %xmm0, %xmm0116; X64-NEXT: vprold $7, %xmm0, %xmm0117; X64-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0118; X64-NEXT: retq119 %lhs_mul = mul <4 x i32> %i, <i32 1152, i32 1152, i32 1152, i32 1152>120 %rhs_mul = mul <4 x i32> %i, <i32 9, i32 9, i32 9, i32 9>121 %lhs_and = and <4 x i32> %lhs_mul, <i32 160, i32 160, i32 160, i32 160>122 %rhs_shift = lshr <4 x i32> %rhs_mul, <i32 25, i32 25, i32 25, i32 25>123 %out = or <4 x i32> %lhs_and, %rhs_shift124 ret <4 x i32> %out125}126 127define <32 x i16> @illegal_no_extract_mul(<32 x i16> %i) nounwind {128; X86-LABEL: illegal_no_extract_mul:129; X86: # %bb.0:130; X86-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}, %zmm0, %zmm0 # [10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10]131; X86-NEXT: vpsrlw $10, %zmm0, %zmm1132; X86-NEXT: vpsllw $6, %zmm0, %zmm0133; X86-NEXT: vporq %zmm1, %zmm0, %zmm0134; X86-NEXT: retl135;136; X64-LABEL: illegal_no_extract_mul:137; X64: # %bb.0:138; X64-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10]139; X64-NEXT: vpsrlw $10, %zmm0, %zmm1140; X64-NEXT: vpsllw $6, %zmm0, %zmm0141; X64-NEXT: vporq %zmm1, %zmm0, %zmm0142; X64-NEXT: retq143 %lhs_mul = mul <32 x i16> %i, <i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640, i16 640>144 %rhs_mul = mul <32 x i16> %i, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>145 %rhs_shift = lshr <32 x i16> %rhs_mul, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>146 %out = or <32 x i16> %lhs_mul, %rhs_shift147 ret <32 x i16> %out148}149 150; Result would undershift151define <4 x i64> @no_extract_shl(<4 x i64> %i) nounwind {152; CHECK-LABEL: no_extract_shl:153; CHECK: # %bb.0:154; CHECK-NEXT: vpsllq $24, %ymm0, %ymm1155; CHECK-NEXT: vpsrlq $39, %ymm0, %ymm0156; CHECK-NEXT: vpternlogq {{.*#+}} ymm0 = (ymm0 & m64bcst) | ymm1157; CHECK-NEXT: ret{{[l|q]}}158 %lhs_mul = shl <4 x i64> %i, <i64 11, i64 11, i64 11, i64 11>159 %rhs_mul = shl <4 x i64> %i, <i64 24, i64 24, i64 24, i64 24>160 %lhs_shift = lshr <4 x i64> %lhs_mul, <i64 50, i64 50, i64 50, i64 50>161 %out = or <4 x i64> %lhs_shift, %rhs_mul162 ret <4 x i64> %out163}164 165; Result would overshift166define <4 x i32> @no_extract_shrl(<4 x i32> %i) nounwind {167; CHECK-LABEL: no_extract_shrl:168; CHECK: # %bb.0:169; CHECK-NEXT: vpsrld $9, %xmm0, %xmm1170; CHECK-NEXT: vpslld $25, %xmm0, %xmm0171; CHECK-NEXT: vpternlogd {{.*#+}} xmm0 = (xmm0 & m32bcst) | xmm1172; CHECK-NEXT: ret{{[l|q]}}173 %lhs_div = lshr <4 x i32> %i, <i32 3, i32 3, i32 3, i32 3>174 %rhs_div = lshr <4 x i32> %i, <i32 9, i32 9, i32 9, i32 9>175 %lhs_shift = shl <4 x i32> %lhs_div, <i32 28, i32 28, i32 28, i32 28>176 %out = or <4 x i32> %lhs_shift, %rhs_div177 ret <4 x i32> %out178}179 180; Can factor 512 from 1536, but result is 3 instead of 9181define <8 x i32> @no_extract_mul(<8 x i32> %i) nounwind {182; X86-LABEL: no_extract_mul:183; X86: # %bb.0:184; X86-NEXT: vpmulld {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %ymm0, %ymm1185; X86-NEXT: vpslld $3, %ymm0, %ymm2186; X86-NEXT: vpaddd %ymm2, %ymm0, %ymm0187; X86-NEXT: vpsrld $23, %ymm0, %ymm0188; X86-NEXT: vpor %ymm0, %ymm1, %ymm0189; X86-NEXT: retl190;191; X64-LABEL: no_extract_mul:192; X64: # %bb.0:193; X64-NEXT: vpmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm1194; X64-NEXT: vpslld $3, %ymm0, %ymm2195; X64-NEXT: vpaddd %ymm2, %ymm0, %ymm0196; X64-NEXT: vpsrld $23, %ymm0, %ymm0197; X64-NEXT: vpor %ymm0, %ymm1, %ymm0198; X64-NEXT: retq199 %lhs_mul = mul <8 x i32> %i, <i32 1536, i32 1536, i32 1536, i32 1536, i32 1536, i32 1536, i32 1536, i32 1536>200 %rhs_mul = mul <8 x i32> %i, <i32 9, i32 9, i32 9, i32 9, i32 9, i32 9, i32 9, i32 9>201 %rhs_shift = lshr <8 x i32> %rhs_mul, <i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23>202 %out = or <8 x i32> %lhs_mul, %rhs_shift203 ret <8 x i32> %out204}205 206; Can't evenly factor 256 from 770207define <2 x i64> @no_extract_udiv(<2 x i64> %i) nounwind {208; X86-LABEL: no_extract_udiv:209; X86: # %bb.0:210; X86-NEXT: subl $48, %esp211; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill212; X86-NEXT: vextractps $1, %xmm0, {{[0-9]+}}(%esp)213; X86-NEXT: vmovss %xmm0, (%esp)214; X86-NEXT: movl $0, {{[0-9]+}}(%esp)215; X86-NEXT: movl $3, {{[0-9]+}}(%esp)216; X86-NEXT: calll __udivdi3217; X86-NEXT: vmovups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload218; X86-NEXT: vextractps $3, %xmm0, {{[0-9]+}}(%esp)219; X86-NEXT: vextractps $2, %xmm0, (%esp)220; X86-NEXT: movl $0, {{[0-9]+}}(%esp)221; X86-NEXT: movl $3, {{[0-9]+}}(%esp)222; X86-NEXT: vmovd %eax, %xmm0223; X86-NEXT: vmovdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill224; X86-NEXT: calll __udivdi3225; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload226; X86-NEXT: vpinsrd $2, %eax, %xmm0, %xmm0227; X86-NEXT: vmovdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill228; X86-NEXT: vmovups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload229; X86-NEXT: vextractps $1, %xmm0, {{[0-9]+}}(%esp)230; X86-NEXT: vmovss %xmm0, (%esp)231; X86-NEXT: movl $0, {{[0-9]+}}(%esp)232; X86-NEXT: movl $770, {{[0-9]+}}(%esp) # imm = 0x302233; X86-NEXT: calll __udivdi3234; X86-NEXT: vmovups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload235; X86-NEXT: vextractps $3, %xmm0, {{[0-9]+}}(%esp)236; X86-NEXT: vextractps $2, %xmm0, (%esp)237; X86-NEXT: movl $0, {{[0-9]+}}(%esp)238; X86-NEXT: movl $770, {{[0-9]+}}(%esp) # imm = 0x302239; X86-NEXT: vmovd %eax, %xmm0240; X86-NEXT: vpinsrd $1, %edx, %xmm0, %xmm0241; X86-NEXT: vmovdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill242; X86-NEXT: calll __udivdi3243; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload244; X86-NEXT: vpinsrd $2, %eax, %xmm0, %xmm0245; X86-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0246; X86-NEXT: vpsllq $56, {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Folded Reload247; X86-NEXT: vpor %xmm0, %xmm1, %xmm0248; X86-NEXT: addl $48, %esp249; X86-NEXT: retl250;251; X64-LABEL: no_extract_udiv:252; X64: # %bb.0:253; X64-NEXT: vpextrq $1, %xmm0, %rcx254; X64-NEXT: movabsq $-6148914691236517205, %rdi # imm = 0xAAAAAAAAAAAAAAAB255; X64-NEXT: movq %rcx, %rax256; X64-NEXT: mulq %rdi257; X64-NEXT: vmovq %rdx, %xmm1258; X64-NEXT: vmovq %xmm0, %rsi259; X64-NEXT: movq %rsi, %rax260; X64-NEXT: mulq %rdi261; X64-NEXT: vmovq %rdx, %xmm0262; X64-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]263; X64-NEXT: vpsrlq $1, %xmm0, %xmm0264; X64-NEXT: movabsq $-6180857105216966645, %rdi # imm = 0xAA392F35DC17F00B265; X64-NEXT: movq %rcx, %rax266; X64-NEXT: mulq %rdi267; X64-NEXT: vmovq %rdx, %xmm1268; X64-NEXT: movq %rsi, %rax269; X64-NEXT: mulq %rdi270; X64-NEXT: vmovq %rdx, %xmm2271; X64-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]272; X64-NEXT: vpsrlq $9, %xmm1, %xmm1273; X64-NEXT: vpsllq $56, %xmm0, %xmm0274; X64-NEXT: vpor %xmm1, %xmm0, %xmm0275; X64-NEXT: retq276 %lhs_div = udiv <2 x i64> %i, <i64 3, i64 3>277 %rhs_div = udiv <2 x i64> %i, <i64 770, i64 770>278 %lhs_shift = shl <2 x i64> %lhs_div, <i64 56, i64 56>279 %out = or <2 x i64> %lhs_shift, %rhs_div280 ret <2 x i64> %out281}282 283; DAGCombiner transforms shl X, 1 into add X, X.284define <4 x i32> @extract_add_1(<4 x i32> %i) nounwind {285; CHECK-LABEL: extract_add_1:286; CHECK: # %bb.0:287; CHECK-NEXT: vprold $1, %xmm0, %xmm0288; CHECK-NEXT: ret{{[l|q]}}289 %ii = add <4 x i32> %i, %i290 %rhs = lshr <4 x i32> %i, <i32 31, i32 31, i32 31, i32 31>291 %out = or <4 x i32> %ii, %rhs292 ret <4 x i32> %out293}294 295define <4 x i32> @extract_add_1_comut(<4 x i32> %i) nounwind {296; CHECK-LABEL: extract_add_1_comut:297; CHECK: # %bb.0:298; CHECK-NEXT: vprold $1, %xmm0, %xmm0299; CHECK-NEXT: ret{{[l|q]}}300 %ii = add <4 x i32> %i, %i301 %lhs = lshr <4 x i32> %i, <i32 31, i32 31, i32 31, i32 31>302 %out = or <4 x i32> %lhs, %ii303 ret <4 x i32> %out304}305 306define <4 x i32> @no_extract_add_1(<4 x i32> %i) nounwind {307; CHECK-LABEL: no_extract_add_1:308; CHECK: # %bb.0:309; CHECK-NEXT: vpaddd %xmm0, %xmm0, %xmm1310; CHECK-NEXT: vpsrld $27, %xmm0, %xmm0311; CHECK-NEXT: vpor %xmm0, %xmm1, %xmm0312; CHECK-NEXT: ret{{[l|q]}}313 %ii = add <4 x i32> %i, %i314 %rhs = lshr <4 x i32> %i, <i32 27, i32 27, i32 27, i32 27>315 %out = or <4 x i32> %ii, %rhs316 ret <4 x i32> %out317}318