2356 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -O0 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mcpu=skylake | FileCheck --check-prefixes=CHECK,CHECK-O0 %s3; RUN: llc -O3 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mcpu=skylake | FileCheck --check-prefixes=CHECK,CHECK-O3 %s4 5define i8 @load_i8(ptr %ptr) {6; CHECK-O0-LABEL: load_i8:7; CHECK-O0: # %bb.0:8; CHECK-O0-NEXT: movb (%rdi), %al9; CHECK-O0-NEXT: retq10;11; CHECK-O3-LABEL: load_i8:12; CHECK-O3: # %bb.0:13; CHECK-O3-NEXT: movzbl (%rdi), %eax14; CHECK-O3-NEXT: retq15 %v = load atomic i8, ptr %ptr unordered, align 116 ret i8 %v17}18 19define void @store_i8(ptr %ptr, i8 %v) {20; CHECK-O0-LABEL: store_i8:21; CHECK-O0: # %bb.0:22; CHECK-O0-NEXT: movb %sil, %al23; CHECK-O0-NEXT: movb %al, (%rdi)24; CHECK-O0-NEXT: retq25;26; CHECK-O3-LABEL: store_i8:27; CHECK-O3: # %bb.0:28; CHECK-O3-NEXT: movb %sil, (%rdi)29; CHECK-O3-NEXT: retq30 store atomic i8 %v, ptr %ptr unordered, align 131 ret void32}33 34define i16 @load_i16(ptr %ptr) {35; CHECK-O0-LABEL: load_i16:36; CHECK-O0: # %bb.0:37; CHECK-O0-NEXT: movw (%rdi), %ax38; CHECK-O0-NEXT: retq39;40; CHECK-O3-LABEL: load_i16:41; CHECK-O3: # %bb.0:42; CHECK-O3-NEXT: movzwl (%rdi), %eax43; CHECK-O3-NEXT: retq44 %v = load atomic i16, ptr %ptr unordered, align 245 ret i16 %v46}47 48 49define void @store_i16(ptr %ptr, i16 %v) {50; CHECK-O0-LABEL: store_i16:51; CHECK-O0: # %bb.0:52; CHECK-O0-NEXT: movw %si, %ax53; CHECK-O0-NEXT: movw %ax, (%rdi)54; CHECK-O0-NEXT: retq55;56; CHECK-O3-LABEL: store_i16:57; CHECK-O3: # %bb.0:58; CHECK-O3-NEXT: movw %si, (%rdi)59; CHECK-O3-NEXT: retq60 store atomic i16 %v, ptr %ptr unordered, align 261 ret void62}63 64define i32 @load_i32(ptr %ptr) {65; CHECK-LABEL: load_i32:66; CHECK: # %bb.0:67; CHECK-NEXT: movl (%rdi), %eax68; CHECK-NEXT: retq69 %v = load atomic i32, ptr %ptr unordered, align 470 ret i32 %v71}72 73define void @store_i32(ptr %ptr, i32 %v) {74; CHECK-LABEL: store_i32:75; CHECK: # %bb.0:76; CHECK-NEXT: movl %esi, (%rdi)77; CHECK-NEXT: retq78 store atomic i32 %v, ptr %ptr unordered, align 479 ret void80}81 82define i64 @load_i64(ptr %ptr) {83; CHECK-LABEL: load_i64:84; CHECK: # %bb.0:85; CHECK-NEXT: movq (%rdi), %rax86; CHECK-NEXT: retq87 %v = load atomic i64, ptr %ptr unordered, align 888 ret i64 %v89}90 91define void @store_i64(ptr %ptr, i64 %v) {92; CHECK-LABEL: store_i64:93; CHECK: # %bb.0:94; CHECK-NEXT: movq %rsi, (%rdi)95; CHECK-NEXT: retq96 store atomic i64 %v, ptr %ptr unordered, align 897 ret void98}99 100;; The tests in the rest of this file are intended to show transforms which we101;; either *can't* do for legality, or don't currently implement. The later102;; are noted carefully where relevant.103 104;; Start w/some clearly illegal ones.105 106; Must use a full width op, not a byte op107define void @narrow_writeback_or(ptr %ptr) {108; CHECK-O0-LABEL: narrow_writeback_or:109; CHECK-O0: # %bb.0:110; CHECK-O0-NEXT: movq (%rdi), %rax111; CHECK-O0-NEXT: orq $7, %rax112; CHECK-O0-NEXT: movq %rax, (%rdi)113; CHECK-O0-NEXT: retq114;115; CHECK-O3-LABEL: narrow_writeback_or:116; CHECK-O3: # %bb.0:117; CHECK-O3-NEXT: orq $7, (%rdi)118; CHECK-O3-NEXT: retq119 %v = load atomic i64, ptr %ptr unordered, align 8120 %v.new = or i64 %v, 7121 store atomic i64 %v.new, ptr %ptr unordered, align 8122 ret void123}124 125; Must use a full width op, not a byte op126define void @narrow_writeback_and(ptr %ptr) {127; CHECK-O0-LABEL: narrow_writeback_and:128; CHECK-O0: # %bb.0:129; CHECK-O0-NEXT: movq (%rdi), %rax130; CHECK-O0-NEXT: # kill: def $eax killed $eax killed $rax131; CHECK-O0-NEXT: andl $-256, %eax132; CHECK-O0-NEXT: # kill: def $rax killed $eax133; CHECK-O0-NEXT: movq %rax, (%rdi)134; CHECK-O0-NEXT: retq135;136; CHECK-O3-LABEL: narrow_writeback_and:137; CHECK-O3: # %bb.0:138; CHECK-O3-NEXT: movl $4294967040, %eax # imm = 0xFFFFFF00139; CHECK-O3-NEXT: andq %rax, (%rdi)140; CHECK-O3-NEXT: retq141 %v = load atomic i64, ptr %ptr unordered, align 8142 %v.new = and i64 %v, 4294967040 ;; 0xFFFF_FF00143 store atomic i64 %v.new, ptr %ptr unordered, align 8144 ret void145}146 147; Must use a full width op, not a byte op148define void @narrow_writeback_xor(ptr %ptr) {149; CHECK-O0-LABEL: narrow_writeback_xor:150; CHECK-O0: # %bb.0:151; CHECK-O0-NEXT: movq (%rdi), %rax152; CHECK-O0-NEXT: xorq $7, %rax153; CHECK-O0-NEXT: movq %rax, (%rdi)154; CHECK-O0-NEXT: retq155;156; CHECK-O3-LABEL: narrow_writeback_xor:157; CHECK-O3: # %bb.0:158; CHECK-O3-NEXT: xorq $7, (%rdi)159; CHECK-O3-NEXT: retq160 %v = load atomic i64, ptr %ptr unordered, align 8161 %v.new = xor i64 %v, 7162 store atomic i64 %v.new, ptr %ptr unordered, align 8163 ret void164}165 166;; Next batch of tests are exercising cases where store widening would167;; improve codegeneration. Note that widening is only legal if the168;; resulting type would be atomic. Each tests has a well aligned, and169;; unaligned variant to ensure we get correct codegen here.170;; Note: It's not a legality issue, but there's a gotcha here to be aware171;; of. Once we widen a pair of atomic stores, we loose the information172;; that the original atomicity requirement was half the width. Given that,173;; we can't then split the load again. This challenges our usual iterative174;; approach to incremental improvement.175 176; Legal if wider type is also atomic (TODO)177define void @widen_store(ptr %p0, i32 %v1, i32 %v2) {178; CHECK-LABEL: widen_store:179; CHECK: # %bb.0:180; CHECK-NEXT: movl %esi, (%rdi)181; CHECK-NEXT: movl %edx, 4(%rdi)182; CHECK-NEXT: retq183 %p1 = getelementptr i32, ptr %p0, i64 1184 store atomic i32 %v1, ptr %p0 unordered, align 8185 store atomic i32 %v2, ptr %p1 unordered, align 4186 ret void187}188 189; This one is *NOT* legal to widen. With weaker alignment,190; the wider type might cross a cache line and violate the191; atomicity requirement.192define void @widen_store_unaligned(ptr %p0, i32 %v1, i32 %v2) {193; CHECK-LABEL: widen_store_unaligned:194; CHECK: # %bb.0:195; CHECK-NEXT: movl %esi, (%rdi)196; CHECK-NEXT: movl %edx, 4(%rdi)197; CHECK-NEXT: retq198 %p1 = getelementptr i32, ptr %p0, i64 1199 store atomic i32 %v1, ptr %p0 unordered, align 4200 store atomic i32 %v2, ptr %p1 unordered, align 4201 ret void202}203 204; Legal if wider type is also atomic (TODO)205define void @widen_broadcast(ptr %p0, i32 %v) {206; CHECK-LABEL: widen_broadcast:207; CHECK: # %bb.0:208; CHECK-NEXT: movl %esi, (%rdi)209; CHECK-NEXT: movl %esi, 4(%rdi)210; CHECK-NEXT: retq211 %p1 = getelementptr i32, ptr %p0, i64 1212 store atomic i32 %v, ptr %p0 unordered, align 8213 store atomic i32 %v, ptr %p1 unordered, align 4214 ret void215}216 217; Not legal to widen due to alignment restriction218define void @widen_broadcast_unaligned(ptr %p0, i32 %v) {219; CHECK-LABEL: widen_broadcast_unaligned:220; CHECK: # %bb.0:221; CHECK-NEXT: movl %esi, (%rdi)222; CHECK-NEXT: movl %esi, 4(%rdi)223; CHECK-NEXT: retq224 %p1 = getelementptr i32, ptr %p0, i64 1225 store atomic i32 %v, ptr %p0 unordered, align 4226 store atomic i32 %v, ptr %p1 unordered, align 4227 ret void228}229 230define i128 @load_i128(ptr %ptr) {231; CHECK-LABEL: load_i128:232; CHECK: # %bb.0:233; CHECK-NEXT: vmovdqa (%rdi), %xmm0234; CHECK-NEXT: vmovq %xmm0, %rax235; CHECK-NEXT: vpextrq $1, %xmm0, %rdx236; CHECK-NEXT: retq237 %v = load atomic i128, ptr %ptr unordered, align 16238 ret i128 %v239}240 241define void @store_i128(ptr %ptr, i128 %v) {242; CHECK-O0-LABEL: store_i128:243; CHECK-O0: # %bb.0:244; CHECK-O0-NEXT: vmovq %rsi, %xmm0245; CHECK-O0-NEXT: vmovq %rdx, %xmm1246; CHECK-O0-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]247; CHECK-O0-NEXT: vmovdqa %xmm0, (%rdi)248; CHECK-O0-NEXT: retq249;250; CHECK-O3-LABEL: store_i128:251; CHECK-O3: # %bb.0:252; CHECK-O3-NEXT: vmovq %rdx, %xmm0253; CHECK-O3-NEXT: vmovq %rsi, %xmm1254; CHECK-O3-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]255; CHECK-O3-NEXT: vmovdqa %xmm0, (%rdi)256; CHECK-O3-NEXT: retq257 store atomic i128 %v, ptr %ptr unordered, align 16258 ret void259}260 261define i256 @load_i256(ptr %ptr) {262; CHECK-O0-LABEL: load_i256:263; CHECK-O0: # %bb.0:264; CHECK-O0-NEXT: subq $56, %rsp265; CHECK-O0-NEXT: .cfi_def_cfa_offset 64266; CHECK-O0-NEXT: movq %rdi, %rax267; CHECK-O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill268; CHECK-O0-NEXT: movq %rdi, (%rsp) # 8-byte Spill269; CHECK-O0-NEXT: movl $32, %edi270; CHECK-O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx271; CHECK-O0-NEXT: xorl %ecx, %ecx272; CHECK-O0-NEXT: callq __atomic_load@PLT273; CHECK-O0-NEXT: movq (%rsp), %rdi # 8-byte Reload274; CHECK-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload275; CHECK-O0-NEXT: movq {{[0-9]+}}(%rsp), %rcx276; CHECK-O0-NEXT: movq {{[0-9]+}}(%rsp), %rdx277; CHECK-O0-NEXT: movq {{[0-9]+}}(%rsp), %rsi278; CHECK-O0-NEXT: movq {{[0-9]+}}(%rsp), %r8279; CHECK-O0-NEXT: movq %r8, 24(%rdi)280; CHECK-O0-NEXT: movq %rsi, 16(%rdi)281; CHECK-O0-NEXT: movq %rdx, 8(%rdi)282; CHECK-O0-NEXT: movq %rcx, (%rdi)283; CHECK-O0-NEXT: addq $56, %rsp284; CHECK-O0-NEXT: .cfi_def_cfa_offset 8285; CHECK-O0-NEXT: retq286;287; CHECK-O3-LABEL: load_i256:288; CHECK-O3: # %bb.0:289; CHECK-O3-NEXT: pushq %rbx290; CHECK-O3-NEXT: .cfi_def_cfa_offset 16291; CHECK-O3-NEXT: subq $32, %rsp292; CHECK-O3-NEXT: .cfi_def_cfa_offset 48293; CHECK-O3-NEXT: .cfi_offset %rbx, -16294; CHECK-O3-NEXT: movq %rdi, %rbx295; CHECK-O3-NEXT: movq %rsp, %rdx296; CHECK-O3-NEXT: movl $32, %edi297; CHECK-O3-NEXT: xorl %ecx, %ecx298; CHECK-O3-NEXT: callq __atomic_load@PLT299; CHECK-O3-NEXT: vmovups (%rsp), %ymm0300; CHECK-O3-NEXT: vmovups %ymm0, (%rbx)301; CHECK-O3-NEXT: movq %rbx, %rax302; CHECK-O3-NEXT: addq $32, %rsp303; CHECK-O3-NEXT: .cfi_def_cfa_offset 16304; CHECK-O3-NEXT: popq %rbx305; CHECK-O3-NEXT: .cfi_def_cfa_offset 8306; CHECK-O3-NEXT: vzeroupper307; CHECK-O3-NEXT: retq308 %v = load atomic i256, ptr %ptr unordered, align 16309 ret i256 %v310}311 312define void @store_i256(ptr %ptr, i256 %v) {313; CHECK-O0-LABEL: store_i256:314; CHECK-O0: # %bb.0:315; CHECK-O0-NEXT: subq $40, %rsp316; CHECK-O0-NEXT: .cfi_def_cfa_offset 48317; CHECK-O0-NEXT: movq %rsi, %rax318; CHECK-O0-NEXT: movq %rdi, %rsi319; CHECK-O0-NEXT: movq %rax, (%rsp)320; CHECK-O0-NEXT: movq %rdx, {{[0-9]+}}(%rsp)321; CHECK-O0-NEXT: movq %rcx, {{[0-9]+}}(%rsp)322; CHECK-O0-NEXT: movq %r8, {{[0-9]+}}(%rsp)323; CHECK-O0-NEXT: movl $32, %edi324; CHECK-O0-NEXT: movq %rsp, %rdx325; CHECK-O0-NEXT: xorl %ecx, %ecx326; CHECK-O0-NEXT: callq __atomic_store@PLT327; CHECK-O0-NEXT: addq $40, %rsp328; CHECK-O0-NEXT: .cfi_def_cfa_offset 8329; CHECK-O0-NEXT: retq330;331; CHECK-O3-LABEL: store_i256:332; CHECK-O3: # %bb.0:333; CHECK-O3-NEXT: subq $40, %rsp334; CHECK-O3-NEXT: .cfi_def_cfa_offset 48335; CHECK-O3-NEXT: movq %rdi, %rax336; CHECK-O3-NEXT: movq %r8, {{[0-9]+}}(%rsp)337; CHECK-O3-NEXT: movq %rcx, {{[0-9]+}}(%rsp)338; CHECK-O3-NEXT: movq %rdx, {{[0-9]+}}(%rsp)339; CHECK-O3-NEXT: movq %rsi, (%rsp)340; CHECK-O3-NEXT: movq %rsp, %rdx341; CHECK-O3-NEXT: movl $32, %edi342; CHECK-O3-NEXT: movq %rax, %rsi343; CHECK-O3-NEXT: xorl %ecx, %ecx344; CHECK-O3-NEXT: callq __atomic_store@PLT345; CHECK-O3-NEXT: addq $40, %rsp346; CHECK-O3-NEXT: .cfi_def_cfa_offset 8347; CHECK-O3-NEXT: retq348 store atomic i256 %v, ptr %ptr unordered, align 16349 ret void350}351 352; Legal if wider type is also atomic (TODO)353define void @vec_store(ptr %p0, <2 x i32> %vec) {354; CHECK-O0-LABEL: vec_store:355; CHECK-O0: # %bb.0:356; CHECK-O0-NEXT: vmovd %xmm0, %ecx357; CHECK-O0-NEXT: vpextrd $1, %xmm0, %eax358; CHECK-O0-NEXT: movl %ecx, (%rdi)359; CHECK-O0-NEXT: movl %eax, 4(%rdi)360; CHECK-O0-NEXT: retq361;362; CHECK-O3-LABEL: vec_store:363; CHECK-O3: # %bb.0:364; CHECK-O3-NEXT: vmovd %xmm0, %eax365; CHECK-O3-NEXT: vpextrd $1, %xmm0, %ecx366; CHECK-O3-NEXT: movl %eax, (%rdi)367; CHECK-O3-NEXT: movl %ecx, 4(%rdi)368; CHECK-O3-NEXT: retq369 %v1 = extractelement <2 x i32> %vec, i32 0370 %v2 = extractelement <2 x i32> %vec, i32 1371 %p1 = getelementptr i32, ptr %p0, i64 1372 store atomic i32 %v1, ptr %p0 unordered, align 8373 store atomic i32 %v2, ptr %p1 unordered, align 4374 ret void375}376 377; Not legal to widen due to alignment restriction378define void @vec_store_unaligned(ptr %p0, <2 x i32> %vec) {379; CHECK-O0-LABEL: vec_store_unaligned:380; CHECK-O0: # %bb.0:381; CHECK-O0-NEXT: vmovd %xmm0, %ecx382; CHECK-O0-NEXT: vpextrd $1, %xmm0, %eax383; CHECK-O0-NEXT: movl %ecx, (%rdi)384; CHECK-O0-NEXT: movl %eax, 4(%rdi)385; CHECK-O0-NEXT: retq386;387; CHECK-O3-LABEL: vec_store_unaligned:388; CHECK-O3: # %bb.0:389; CHECK-O3-NEXT: vmovd %xmm0, %eax390; CHECK-O3-NEXT: vpextrd $1, %xmm0, %ecx391; CHECK-O3-NEXT: movl %eax, (%rdi)392; CHECK-O3-NEXT: movl %ecx, 4(%rdi)393; CHECK-O3-NEXT: retq394 %v1 = extractelement <2 x i32> %vec, i32 0395 %v2 = extractelement <2 x i32> %vec, i32 1396 %p1 = getelementptr i32, ptr %p0, i64 1397 store atomic i32 %v1, ptr %p0 unordered, align 4398 store atomic i32 %v2, ptr %p1 unordered, align 4399 ret void400}401 402 403 404; Legal if wider type is also atomic (TODO)405; Also, can avoid register move from xmm to eax (TODO)406define void @widen_broadcast2(ptr %p0, <2 x i32> %vec) {407; CHECK-LABEL: widen_broadcast2:408; CHECK: # %bb.0:409; CHECK-NEXT: vmovd %xmm0, %eax410; CHECK-NEXT: movl %eax, (%rdi)411; CHECK-NEXT: movl %eax, 4(%rdi)412; CHECK-NEXT: retq413 %v1 = extractelement <2 x i32> %vec, i32 0414 %p1 = getelementptr i32, ptr %p0, i64 1415 store atomic i32 %v1, ptr %p0 unordered, align 8416 store atomic i32 %v1, ptr %p1 unordered, align 4417 ret void418}419 420; Not legal to widen due to alignment restriction421define void @widen_broadcast2_unaligned(ptr %p0, <2 x i32> %vec) {422; CHECK-LABEL: widen_broadcast2_unaligned:423; CHECK: # %bb.0:424; CHECK-NEXT: vmovd %xmm0, %eax425; CHECK-NEXT: movl %eax, (%rdi)426; CHECK-NEXT: movl %eax, 4(%rdi)427; CHECK-NEXT: retq428 %v1 = extractelement <2 x i32> %vec, i32 0429 %p1 = getelementptr i32, ptr %p0, i64 1430 store atomic i32 %v1, ptr %p0 unordered, align 4431 store atomic i32 %v1, ptr %p1 unordered, align 4432 ret void433}434 435; Legal if wider type is also atomic (TODO)436define void @widen_zero_init(ptr %p0, i32 %v1, i32 %v2) {437; CHECK-LABEL: widen_zero_init:438; CHECK: # %bb.0:439; CHECK-NEXT: movl $0, (%rdi)440; CHECK-NEXT: movl $0, 4(%rdi)441; CHECK-NEXT: retq442 %p1 = getelementptr i32, ptr %p0, i64 1443 store atomic i32 0, ptr %p0 unordered, align 8444 store atomic i32 0, ptr %p1 unordered, align 4445 ret void446}447 448; Not legal to widen due to alignment restriction449define void @widen_zero_init_unaligned(ptr %p0, i32 %v1, i32 %v2) {450; CHECK-LABEL: widen_zero_init_unaligned:451; CHECK: # %bb.0:452; CHECK-NEXT: movl $0, (%rdi)453; CHECK-NEXT: movl $0, 4(%rdi)454; CHECK-NEXT: retq455 %p1 = getelementptr i32, ptr %p0, i64 1456 store atomic i32 0, ptr %p0 unordered, align 4457 store atomic i32 0, ptr %p1 unordered, align 4458 ret void459}460 461;; The next batch of tests are stressing load folding. Folding is legal462;; on x86, so these are simply checking optimization quality.463 464; Legal, as expected465define i64 @load_fold_add1(ptr %p) {466; CHECK-LABEL: load_fold_add1:467; CHECK: # %bb.0:468; CHECK-NEXT: movq (%rdi), %rax469; CHECK-NEXT: addq $15, %rax470; CHECK-NEXT: retq471 %v = load atomic i64, ptr %p unordered, align 8472 %ret = add i64 %v, 15473 ret i64 %ret474}475 476define i64 @load_fold_add2(ptr %p, i64 %v2) {477; CHECK-LABEL: load_fold_add2:478; CHECK: # %bb.0:479; CHECK-NEXT: movq %rsi, %rax480; CHECK-NEXT: addq (%rdi), %rax481; CHECK-NEXT: retq482 %v = load atomic i64, ptr %p unordered, align 8483 %ret = add i64 %v, %v2484 ret i64 %ret485}486 487define i64 @load_fold_add3(ptr %p1, ptr %p2) {488; CHECK-O0-LABEL: load_fold_add3:489; CHECK-O0: # %bb.0:490; CHECK-O0-NEXT: movq (%rdi), %rax491; CHECK-O0-NEXT: addq (%rsi), %rax492; CHECK-O0-NEXT: retq493;494; CHECK-O3-LABEL: load_fold_add3:495; CHECK-O3: # %bb.0:496; CHECK-O3-NEXT: movq (%rsi), %rax497; CHECK-O3-NEXT: addq (%rdi), %rax498; CHECK-O3-NEXT: retq499 %v = load atomic i64, ptr %p1 unordered, align 8500 %v2 = load atomic i64, ptr %p2 unordered, align 8501 %ret = add i64 %v, %v2502 ret i64 %ret503}504 505; Legal, as expected506define i64 @load_fold_sub1(ptr %p) {507; CHECK-O0-LABEL: load_fold_sub1:508; CHECK-O0: # %bb.0:509; CHECK-O0-NEXT: movq (%rdi), %rax510; CHECK-O0-NEXT: subq $15, %rax511; CHECK-O0-NEXT: retq512;513; CHECK-O3-LABEL: load_fold_sub1:514; CHECK-O3: # %bb.0:515; CHECK-O3-NEXT: movq (%rdi), %rax516; CHECK-O3-NEXT: addq $-15, %rax517; CHECK-O3-NEXT: retq518 %v = load atomic i64, ptr %p unordered, align 8519 %ret = sub i64 %v, 15520 ret i64 %ret521}522 523define i64 @load_fold_sub2(ptr %p, i64 %v2) {524; CHECK-LABEL: load_fold_sub2:525; CHECK: # %bb.0:526; CHECK-NEXT: movq (%rdi), %rax527; CHECK-NEXT: subq %rsi, %rax528; CHECK-NEXT: retq529 %v = load atomic i64, ptr %p unordered, align 8530 %ret = sub i64 %v, %v2531 ret i64 %ret532}533 534define i64 @load_fold_sub3(ptr %p1, ptr %p2) {535; CHECK-LABEL: load_fold_sub3:536; CHECK: # %bb.0:537; CHECK-NEXT: movq (%rdi), %rax538; CHECK-NEXT: subq (%rsi), %rax539; CHECK-NEXT: retq540 %v = load atomic i64, ptr %p1 unordered, align 8541 %v2 = load atomic i64, ptr %p2 unordered, align 8542 %ret = sub i64 %v, %v2543 ret i64 %ret544}545 546; Legal, as expected547define i64 @load_fold_mul1(ptr %p) {548; CHECK-O0-LABEL: load_fold_mul1:549; CHECK-O0: # %bb.0:550; CHECK-O0-NEXT: imulq $15, (%rdi), %rax551; CHECK-O0-NEXT: retq552;553; CHECK-O3-LABEL: load_fold_mul1:554; CHECK-O3: # %bb.0:555; CHECK-O3-NEXT: movq (%rdi), %rax556; CHECK-O3-NEXT: leaq (%rax,%rax,4), %rax557; CHECK-O3-NEXT: leaq (%rax,%rax,2), %rax558; CHECK-O3-NEXT: retq559 %v = load atomic i64, ptr %p unordered, align 8560 %ret = mul i64 %v, 15561 ret i64 %ret562}563 564define i64 @load_fold_mul2(ptr %p, i64 %v2) {565; CHECK-LABEL: load_fold_mul2:566; CHECK: # %bb.0:567; CHECK-NEXT: movq %rsi, %rax568; CHECK-NEXT: imulq (%rdi), %rax569; CHECK-NEXT: retq570 %v = load atomic i64, ptr %p unordered, align 8571 %ret = mul i64 %v, %v2572 ret i64 %ret573}574 575define i64 @load_fold_mul3(ptr %p1, ptr %p2) {576; CHECK-O0-LABEL: load_fold_mul3:577; CHECK-O0: # %bb.0:578; CHECK-O0-NEXT: movq (%rdi), %rax579; CHECK-O0-NEXT: imulq (%rsi), %rax580; CHECK-O0-NEXT: retq581;582; CHECK-O3-LABEL: load_fold_mul3:583; CHECK-O3: # %bb.0:584; CHECK-O3-NEXT: movq (%rsi), %rax585; CHECK-O3-NEXT: imulq (%rdi), %rax586; CHECK-O3-NEXT: retq587 %v = load atomic i64, ptr %p1 unordered, align 8588 %v2 = load atomic i64, ptr %p2 unordered, align 8589 %ret = mul i64 %v, %v2590 ret i64 %ret591}592 593; Legal to fold (TODO)594define i64 @load_fold_sdiv1(ptr %p) {595; CHECK-O0-LABEL: load_fold_sdiv1:596; CHECK-O0: # %bb.0:597; CHECK-O0-NEXT: movq (%rdi), %rax598; CHECK-O0-NEXT: movl $15, %ecx599; CHECK-O0-NEXT: cqto600; CHECK-O0-NEXT: idivq %rcx601; CHECK-O0-NEXT: retq602;603; CHECK-O3-LABEL: load_fold_sdiv1:604; CHECK-O3: # %bb.0:605; CHECK-O3-NEXT: movq (%rdi), %rcx606; CHECK-O3-NEXT: movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889607; CHECK-O3-NEXT: movq %rcx, %rax608; CHECK-O3-NEXT: imulq %rdx609; CHECK-O3-NEXT: addq %rdx, %rcx610; CHECK-O3-NEXT: movq %rcx, %rax611; CHECK-O3-NEXT: shrq $63, %rax612; CHECK-O3-NEXT: sarq $3, %rcx613; CHECK-O3-NEXT: addq %rax, %rcx614; CHECK-O3-NEXT: movq %rcx, %rax615; CHECK-O3-NEXT: retq616 %v = load atomic i64, ptr %p unordered, align 8617 %ret = sdiv i64 %v, 15618 ret i64 %ret619}620 621; Legal to fold (TODO)622define i64 @load_fold_sdiv2(ptr %p, i64 %v2) {623; CHECK-O0-LABEL: load_fold_sdiv2:624; CHECK-O0: # %bb.0:625; CHECK-O0-NEXT: movq (%rdi), %rax626; CHECK-O0-NEXT: cqto627; CHECK-O0-NEXT: idivq %rsi628; CHECK-O0-NEXT: retq629;630; CHECK-O3-LABEL: load_fold_sdiv2:631; CHECK-O3: # %bb.0:632; CHECK-O3-NEXT: movq (%rdi), %rax633; CHECK-O3-NEXT: movq %rax, %rcx634; CHECK-O3-NEXT: orq %rsi, %rcx635; CHECK-O3-NEXT: shrq $32, %rcx636; CHECK-O3-NEXT: je .LBB35_1637; CHECK-O3-NEXT: # %bb.2:638; CHECK-O3-NEXT: cqto639; CHECK-O3-NEXT: idivq %rsi640; CHECK-O3-NEXT: retq641; CHECK-O3-NEXT: .LBB35_1:642; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax643; CHECK-O3-NEXT: xorl %edx, %edx644; CHECK-O3-NEXT: divl %esi645; CHECK-O3-NEXT: # kill: def $eax killed $eax def $rax646; CHECK-O3-NEXT: retq647 %v = load atomic i64, ptr %p unordered, align 8648 %ret = sdiv i64 %v, %v2649 ret i64 %ret650}651 652define i64 @load_fold_sdiv3(ptr %p1, ptr %p2) {653; CHECK-O0-LABEL: load_fold_sdiv3:654; CHECK-O0: # %bb.0:655; CHECK-O0-NEXT: movq (%rdi), %rax656; CHECK-O0-NEXT: cqto657; CHECK-O0-NEXT: idivq (%rsi)658; CHECK-O0-NEXT: retq659;660; CHECK-O3-LABEL: load_fold_sdiv3:661; CHECK-O3: # %bb.0:662; CHECK-O3-NEXT: movq (%rdi), %rax663; CHECK-O3-NEXT: movq (%rsi), %rcx664; CHECK-O3-NEXT: movq %rax, %rdx665; CHECK-O3-NEXT: orq %rcx, %rdx666; CHECK-O3-NEXT: shrq $32, %rdx667; CHECK-O3-NEXT: je .LBB36_1668; CHECK-O3-NEXT: # %bb.2:669; CHECK-O3-NEXT: cqto670; CHECK-O3-NEXT: idivq %rcx671; CHECK-O3-NEXT: retq672; CHECK-O3-NEXT: .LBB36_1:673; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax674; CHECK-O3-NEXT: xorl %edx, %edx675; CHECK-O3-NEXT: divl %ecx676; CHECK-O3-NEXT: # kill: def $eax killed $eax def $rax677; CHECK-O3-NEXT: retq678 %v = load atomic i64, ptr %p1 unordered, align 8679 %v2 = load atomic i64, ptr %p2 unordered, align 8680 %ret = sdiv i64 %v, %v2681 ret i64 %ret682}683 684; Legal to fold (TODO)685define i64 @load_fold_udiv1(ptr %p) {686; CHECK-O0-LABEL: load_fold_udiv1:687; CHECK-O0: # %bb.0:688; CHECK-O0-NEXT: movq (%rdi), %rax689; CHECK-O0-NEXT: movl $15, %ecx690; CHECK-O0-NEXT: xorl %edx, %edx691; CHECK-O0-NEXT: # kill: def $rdx killed $edx692; CHECK-O0-NEXT: divq %rcx693; CHECK-O0-NEXT: retq694;695; CHECK-O3-LABEL: load_fold_udiv1:696; CHECK-O3: # %bb.0:697; CHECK-O3-NEXT: movq (%rdi), %rdx698; CHECK-O3-NEXT: movabsq $-8608480567731124087, %rax # imm = 0x8888888888888889699; CHECK-O3-NEXT: mulxq %rax, %rax, %rax700; CHECK-O3-NEXT: shrq $3, %rax701; CHECK-O3-NEXT: retq702 %v = load atomic i64, ptr %p unordered, align 8703 %ret = udiv i64 %v, 15704 ret i64 %ret705}706 707define i64 @load_fold_udiv2(ptr %p, i64 %v2) {708; CHECK-O0-LABEL: load_fold_udiv2:709; CHECK-O0: # %bb.0:710; CHECK-O0-NEXT: movq (%rdi), %rax711; CHECK-O0-NEXT: xorl %ecx, %ecx712; CHECK-O0-NEXT: movl %ecx, %edx713; CHECK-O0-NEXT: divq %rsi714; CHECK-O0-NEXT: retq715;716; CHECK-O3-LABEL: load_fold_udiv2:717; CHECK-O3: # %bb.0:718; CHECK-O3-NEXT: movq (%rdi), %rax719; CHECK-O3-NEXT: movq %rax, %rcx720; CHECK-O3-NEXT: orq %rsi, %rcx721; CHECK-O3-NEXT: shrq $32, %rcx722; CHECK-O3-NEXT: je .LBB38_1723; CHECK-O3-NEXT: # %bb.2:724; CHECK-O3-NEXT: xorl %edx, %edx725; CHECK-O3-NEXT: divq %rsi726; CHECK-O3-NEXT: retq727; CHECK-O3-NEXT: .LBB38_1:728; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax729; CHECK-O3-NEXT: xorl %edx, %edx730; CHECK-O3-NEXT: divl %esi731; CHECK-O3-NEXT: # kill: def $eax killed $eax def $rax732; CHECK-O3-NEXT: retq733 %v = load atomic i64, ptr %p unordered, align 8734 %ret = udiv i64 %v, %v2735 ret i64 %ret736}737 738define i64 @load_fold_udiv3(ptr %p1, ptr %p2) {739; CHECK-O0-LABEL: load_fold_udiv3:740; CHECK-O0: # %bb.0:741; CHECK-O0-NEXT: movq (%rdi), %rax742; CHECK-O0-NEXT: xorl %ecx, %ecx743; CHECK-O0-NEXT: movl %ecx, %edx744; CHECK-O0-NEXT: divq (%rsi)745; CHECK-O0-NEXT: retq746;747; CHECK-O3-LABEL: load_fold_udiv3:748; CHECK-O3: # %bb.0:749; CHECK-O3-NEXT: movq (%rdi), %rax750; CHECK-O3-NEXT: movq (%rsi), %rcx751; CHECK-O3-NEXT: movq %rax, %rdx752; CHECK-O3-NEXT: orq %rcx, %rdx753; CHECK-O3-NEXT: shrq $32, %rdx754; CHECK-O3-NEXT: je .LBB39_1755; CHECK-O3-NEXT: # %bb.2:756; CHECK-O3-NEXT: xorl %edx, %edx757; CHECK-O3-NEXT: divq %rcx758; CHECK-O3-NEXT: retq759; CHECK-O3-NEXT: .LBB39_1:760; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax761; CHECK-O3-NEXT: xorl %edx, %edx762; CHECK-O3-NEXT: divl %ecx763; CHECK-O3-NEXT: # kill: def $eax killed $eax def $rax764; CHECK-O3-NEXT: retq765 %v = load atomic i64, ptr %p1 unordered, align 8766 %v2 = load atomic i64, ptr %p2 unordered, align 8767 %ret = udiv i64 %v, %v2768 ret i64 %ret769}770 771; Legal to fold (TODO)772define i64 @load_fold_srem1(ptr %p) {773; CHECK-O0-LABEL: load_fold_srem1:774; CHECK-O0: # %bb.0:775; CHECK-O0-NEXT: movq (%rdi), %rax776; CHECK-O0-NEXT: movl $15, %ecx777; CHECK-O0-NEXT: cqto778; CHECK-O0-NEXT: idivq %rcx779; CHECK-O0-NEXT: movq %rdx, %rax780; CHECK-O0-NEXT: retq781;782; CHECK-O3-LABEL: load_fold_srem1:783; CHECK-O3: # %bb.0:784; CHECK-O3-NEXT: movq (%rdi), %rcx785; CHECK-O3-NEXT: movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889786; CHECK-O3-NEXT: movq %rcx, %rax787; CHECK-O3-NEXT: imulq %rdx788; CHECK-O3-NEXT: addq %rcx, %rdx789; CHECK-O3-NEXT: movq %rdx, %rax790; CHECK-O3-NEXT: shrq $63, %rax791; CHECK-O3-NEXT: sarq $3, %rdx792; CHECK-O3-NEXT: addq %rax, %rdx793; CHECK-O3-NEXT: leaq (%rdx,%rdx,4), %rax794; CHECK-O3-NEXT: leaq (%rax,%rax,2), %rax795; CHECK-O3-NEXT: subq %rax, %rcx796; CHECK-O3-NEXT: movq %rcx, %rax797; CHECK-O3-NEXT: retq798 %v = load atomic i64, ptr %p unordered, align 8799 %ret = srem i64 %v, 15800 ret i64 %ret801}802 803; Legal, as expected804define i64 @load_fold_srem2(ptr %p, i64 %v2) {805; CHECK-O0-LABEL: load_fold_srem2:806; CHECK-O0: # %bb.0:807; CHECK-O0-NEXT: movq (%rdi), %rax808; CHECK-O0-NEXT: cqto809; CHECK-O0-NEXT: idivq %rsi810; CHECK-O0-NEXT: movq %rdx, %rax811; CHECK-O0-NEXT: retq812;813; CHECK-O3-LABEL: load_fold_srem2:814; CHECK-O3: # %bb.0:815; CHECK-O3-NEXT: movq (%rdi), %rax816; CHECK-O3-NEXT: movq %rax, %rcx817; CHECK-O3-NEXT: orq %rsi, %rcx818; CHECK-O3-NEXT: shrq $32, %rcx819; CHECK-O3-NEXT: je .LBB41_1820; CHECK-O3-NEXT: # %bb.2:821; CHECK-O3-NEXT: cqto822; CHECK-O3-NEXT: idivq %rsi823; CHECK-O3-NEXT: movq %rdx, %rax824; CHECK-O3-NEXT: retq825; CHECK-O3-NEXT: .LBB41_1:826; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax827; CHECK-O3-NEXT: xorl %edx, %edx828; CHECK-O3-NEXT: divl %esi829; CHECK-O3-NEXT: movl %edx, %eax830; CHECK-O3-NEXT: retq831 %v = load atomic i64, ptr %p unordered, align 8832 %ret = srem i64 %v, %v2833 ret i64 %ret834}835 836define i64 @load_fold_srem3(ptr %p1, ptr %p2) {837; CHECK-O0-LABEL: load_fold_srem3:838; CHECK-O0: # %bb.0:839; CHECK-O0-NEXT: movq (%rdi), %rax840; CHECK-O0-NEXT: cqto841; CHECK-O0-NEXT: idivq (%rsi)842; CHECK-O0-NEXT: movq %rdx, %rax843; CHECK-O0-NEXT: retq844;845; CHECK-O3-LABEL: load_fold_srem3:846; CHECK-O3: # %bb.0:847; CHECK-O3-NEXT: movq (%rdi), %rax848; CHECK-O3-NEXT: movq (%rsi), %rcx849; CHECK-O3-NEXT: movq %rax, %rdx850; CHECK-O3-NEXT: orq %rcx, %rdx851; CHECK-O3-NEXT: shrq $32, %rdx852; CHECK-O3-NEXT: je .LBB42_1853; CHECK-O3-NEXT: # %bb.2:854; CHECK-O3-NEXT: cqto855; CHECK-O3-NEXT: idivq %rcx856; CHECK-O3-NEXT: movq %rdx, %rax857; CHECK-O3-NEXT: retq858; CHECK-O3-NEXT: .LBB42_1:859; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax860; CHECK-O3-NEXT: xorl %edx, %edx861; CHECK-O3-NEXT: divl %ecx862; CHECK-O3-NEXT: movl %edx, %eax863; CHECK-O3-NEXT: retq864 %v = load atomic i64, ptr %p1 unordered, align 8865 %v2 = load atomic i64, ptr %p2 unordered, align 8866 %ret = srem i64 %v, %v2867 ret i64 %ret868}869 870; Legal to fold (TODO)871define i64 @load_fold_urem1(ptr %p) {872; CHECK-O0-LABEL: load_fold_urem1:873; CHECK-O0: # %bb.0:874; CHECK-O0-NEXT: movq (%rdi), %rax875; CHECK-O0-NEXT: movl $15, %ecx876; CHECK-O0-NEXT: xorl %edx, %edx877; CHECK-O0-NEXT: # kill: def $rdx killed $edx878; CHECK-O0-NEXT: divq %rcx879; CHECK-O0-NEXT: movq %rdx, %rax880; CHECK-O0-NEXT: retq881;882; CHECK-O3-LABEL: load_fold_urem1:883; CHECK-O3: # %bb.0:884; CHECK-O3-NEXT: movq (%rdi), %rax885; CHECK-O3-NEXT: movabsq $-8608480567731124087, %rcx # imm = 0x8888888888888889886; CHECK-O3-NEXT: movq %rax, %rdx887; CHECK-O3-NEXT: mulxq %rcx, %rcx, %rcx888; CHECK-O3-NEXT: shrq $3, %rcx889; CHECK-O3-NEXT: leaq (%rcx,%rcx,4), %rcx890; CHECK-O3-NEXT: leaq (%rcx,%rcx,2), %rcx891; CHECK-O3-NEXT: subq %rcx, %rax892; CHECK-O3-NEXT: retq893 %v = load atomic i64, ptr %p unordered, align 8894 %ret = urem i64 %v, 15895 ret i64 %ret896}897 898; Legal, as expected899define i64 @load_fold_urem2(ptr %p, i64 %v2) {900; CHECK-O0-LABEL: load_fold_urem2:901; CHECK-O0: # %bb.0:902; CHECK-O0-NEXT: movq (%rdi), %rax903; CHECK-O0-NEXT: xorl %ecx, %ecx904; CHECK-O0-NEXT: movl %ecx, %edx905; CHECK-O0-NEXT: divq %rsi906; CHECK-O0-NEXT: movq %rdx, %rax907; CHECK-O0-NEXT: retq908;909; CHECK-O3-LABEL: load_fold_urem2:910; CHECK-O3: # %bb.0:911; CHECK-O3-NEXT: movq (%rdi), %rax912; CHECK-O3-NEXT: movq %rax, %rcx913; CHECK-O3-NEXT: orq %rsi, %rcx914; CHECK-O3-NEXT: shrq $32, %rcx915; CHECK-O3-NEXT: je .LBB44_1916; CHECK-O3-NEXT: # %bb.2:917; CHECK-O3-NEXT: xorl %edx, %edx918; CHECK-O3-NEXT: divq %rsi919; CHECK-O3-NEXT: movq %rdx, %rax920; CHECK-O3-NEXT: retq921; CHECK-O3-NEXT: .LBB44_1:922; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax923; CHECK-O3-NEXT: xorl %edx, %edx924; CHECK-O3-NEXT: divl %esi925; CHECK-O3-NEXT: movl %edx, %eax926; CHECK-O3-NEXT: retq927 %v = load atomic i64, ptr %p unordered, align 8928 %ret = urem i64 %v, %v2929 ret i64 %ret930}931 932define i64 @load_fold_urem3(ptr %p1, ptr %p2) {933; CHECK-O0-LABEL: load_fold_urem3:934; CHECK-O0: # %bb.0:935; CHECK-O0-NEXT: movq (%rdi), %rax936; CHECK-O0-NEXT: xorl %ecx, %ecx937; CHECK-O0-NEXT: movl %ecx, %edx938; CHECK-O0-NEXT: divq (%rsi)939; CHECK-O0-NEXT: movq %rdx, %rax940; CHECK-O0-NEXT: retq941;942; CHECK-O3-LABEL: load_fold_urem3:943; CHECK-O3: # %bb.0:944; CHECK-O3-NEXT: movq (%rdi), %rax945; CHECK-O3-NEXT: movq (%rsi), %rcx946; CHECK-O3-NEXT: movq %rax, %rdx947; CHECK-O3-NEXT: orq %rcx, %rdx948; CHECK-O3-NEXT: shrq $32, %rdx949; CHECK-O3-NEXT: je .LBB45_1950; CHECK-O3-NEXT: # %bb.2:951; CHECK-O3-NEXT: xorl %edx, %edx952; CHECK-O3-NEXT: divq %rcx953; CHECK-O3-NEXT: movq %rdx, %rax954; CHECK-O3-NEXT: retq955; CHECK-O3-NEXT: .LBB45_1:956; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax957; CHECK-O3-NEXT: xorl %edx, %edx958; CHECK-O3-NEXT: divl %ecx959; CHECK-O3-NEXT: movl %edx, %eax960; CHECK-O3-NEXT: retq961 %v = load atomic i64, ptr %p1 unordered, align 8962 %v2 = load atomic i64, ptr %p2 unordered, align 8963 %ret = urem i64 %v, %v2964 ret i64 %ret965}966 967; Legal, as expected968define i64 @load_fold_shl1(ptr %p) {969; CHECK-LABEL: load_fold_shl1:970; CHECK: # %bb.0:971; CHECK-NEXT: movq (%rdi), %rax972; CHECK-NEXT: shlq $15, %rax973; CHECK-NEXT: retq974 %v = load atomic i64, ptr %p unordered, align 8975 %ret = shl i64 %v, 15976 ret i64 %ret977}978 979define i64 @load_fold_shl2(ptr %p, i64 %v2) {980; CHECK-O0-LABEL: load_fold_shl2:981; CHECK-O0: # %bb.0:982; CHECK-O0-NEXT: movq %rsi, %rcx983; CHECK-O0-NEXT: movq (%rdi), %rax984; CHECK-O0-NEXT: # kill: def $cl killed $rcx985; CHECK-O0-NEXT: shlq %cl, %rax986; CHECK-O0-NEXT: retq987;988; CHECK-O3-LABEL: load_fold_shl2:989; CHECK-O3: # %bb.0:990; CHECK-O3-NEXT: shlxq %rsi, (%rdi), %rax991; CHECK-O3-NEXT: retq992 %v = load atomic i64, ptr %p unordered, align 8993 %ret = shl i64 %v, %v2994 ret i64 %ret995}996 997define i64 @load_fold_shl3(ptr %p1, ptr %p2) {998; CHECK-O0-LABEL: load_fold_shl3:999; CHECK-O0: # %bb.0:1000; CHECK-O0-NEXT: movq (%rdi), %rax1001; CHECK-O0-NEXT: movq (%rsi), %rcx1002; CHECK-O0-NEXT: # kill: def $cl killed $rcx1003; CHECK-O0-NEXT: shlq %cl, %rax1004; CHECK-O0-NEXT: retq1005;1006; CHECK-O3-LABEL: load_fold_shl3:1007; CHECK-O3: # %bb.0:1008; CHECK-O3-NEXT: movq (%rsi), %rax1009; CHECK-O3-NEXT: shlxq %rax, (%rdi), %rax1010; CHECK-O3-NEXT: retq1011 %v = load atomic i64, ptr %p1 unordered, align 81012 %v2 = load atomic i64, ptr %p2 unordered, align 81013 %ret = shl i64 %v, %v21014 ret i64 %ret1015}1016 1017; Legal, as expected1018define i64 @load_fold_lshr1(ptr %p) {1019; CHECK-LABEL: load_fold_lshr1:1020; CHECK: # %bb.0:1021; CHECK-NEXT: movq (%rdi), %rax1022; CHECK-NEXT: shrq $15, %rax1023; CHECK-NEXT: retq1024 %v = load atomic i64, ptr %p unordered, align 81025 %ret = lshr i64 %v, 151026 ret i64 %ret1027}1028 1029define i64 @load_fold_lshr2(ptr %p, i64 %v2) {1030; CHECK-O0-LABEL: load_fold_lshr2:1031; CHECK-O0: # %bb.0:1032; CHECK-O0-NEXT: movq %rsi, %rcx1033; CHECK-O0-NEXT: movq (%rdi), %rax1034; CHECK-O0-NEXT: # kill: def $cl killed $rcx1035; CHECK-O0-NEXT: shrq %cl, %rax1036; CHECK-O0-NEXT: retq1037;1038; CHECK-O3-LABEL: load_fold_lshr2:1039; CHECK-O3: # %bb.0:1040; CHECK-O3-NEXT: shrxq %rsi, (%rdi), %rax1041; CHECK-O3-NEXT: retq1042 %v = load atomic i64, ptr %p unordered, align 81043 %ret = lshr i64 %v, %v21044 ret i64 %ret1045}1046 1047define i64 @load_fold_lshr3(ptr %p1, ptr %p2) {1048; CHECK-O0-LABEL: load_fold_lshr3:1049; CHECK-O0: # %bb.0:1050; CHECK-O0-NEXT: movq (%rdi), %rax1051; CHECK-O0-NEXT: movq (%rsi), %rcx1052; CHECK-O0-NEXT: # kill: def $cl killed $rcx1053; CHECK-O0-NEXT: shrq %cl, %rax1054; CHECK-O0-NEXT: retq1055;1056; CHECK-O3-LABEL: load_fold_lshr3:1057; CHECK-O3: # %bb.0:1058; CHECK-O3-NEXT: movq (%rsi), %rax1059; CHECK-O3-NEXT: shrxq %rax, (%rdi), %rax1060; CHECK-O3-NEXT: retq1061 %v = load atomic i64, ptr %p1 unordered, align 81062 %v2 = load atomic i64, ptr %p2 unordered, align 81063 %ret = lshr i64 %v, %v21064 ret i64 %ret1065}1066 1067; Legal, as expected1068define i64 @load_fold_ashr1(ptr %p) {1069; CHECK-LABEL: load_fold_ashr1:1070; CHECK: # %bb.0:1071; CHECK-NEXT: movq (%rdi), %rax1072; CHECK-NEXT: sarq $15, %rax1073; CHECK-NEXT: retq1074 %v = load atomic i64, ptr %p unordered, align 81075 %ret = ashr i64 %v, 151076 ret i64 %ret1077}1078 1079define i64 @load_fold_ashr2(ptr %p, i64 %v2) {1080; CHECK-O0-LABEL: load_fold_ashr2:1081; CHECK-O0: # %bb.0:1082; CHECK-O0-NEXT: movq %rsi, %rcx1083; CHECK-O0-NEXT: movq (%rdi), %rax1084; CHECK-O0-NEXT: # kill: def $cl killed $rcx1085; CHECK-O0-NEXT: sarq %cl, %rax1086; CHECK-O0-NEXT: retq1087;1088; CHECK-O3-LABEL: load_fold_ashr2:1089; CHECK-O3: # %bb.0:1090; CHECK-O3-NEXT: sarxq %rsi, (%rdi), %rax1091; CHECK-O3-NEXT: retq1092 %v = load atomic i64, ptr %p unordered, align 81093 %ret = ashr i64 %v, %v21094 ret i64 %ret1095}1096 1097define i64 @load_fold_ashr3(ptr %p1, ptr %p2) {1098; CHECK-O0-LABEL: load_fold_ashr3:1099; CHECK-O0: # %bb.0:1100; CHECK-O0-NEXT: movq (%rdi), %rax1101; CHECK-O0-NEXT: movq (%rsi), %rcx1102; CHECK-O0-NEXT: # kill: def $cl killed $rcx1103; CHECK-O0-NEXT: sarq %cl, %rax1104; CHECK-O0-NEXT: retq1105;1106; CHECK-O3-LABEL: load_fold_ashr3:1107; CHECK-O3: # %bb.0:1108; CHECK-O3-NEXT: movq (%rsi), %rax1109; CHECK-O3-NEXT: sarxq %rax, (%rdi), %rax1110; CHECK-O3-NEXT: retq1111 %v = load atomic i64, ptr %p1 unordered, align 81112 %v2 = load atomic i64, ptr %p2 unordered, align 81113 %ret = ashr i64 %v, %v21114 ret i64 %ret1115}1116 1117; Legal, as expected1118define i64 @load_fold_and1(ptr %p) {1119; CHECK-O0-LABEL: load_fold_and1:1120; CHECK-O0: # %bb.0:1121; CHECK-O0-NEXT: movq (%rdi), %rax1122; CHECK-O0-NEXT: andq $15, %rax1123; CHECK-O0-NEXT: retq1124;1125; CHECK-O3-LABEL: load_fold_and1:1126; CHECK-O3: # %bb.0:1127; CHECK-O3-NEXT: movq (%rdi), %rax1128; CHECK-O3-NEXT: andl $15, %eax1129; CHECK-O3-NEXT: retq1130 %v = load atomic i64, ptr %p unordered, align 81131 %ret = and i64 %v, 151132 ret i64 %ret1133}1134 1135define i64 @load_fold_and2(ptr %p, i64 %v2) {1136; CHECK-LABEL: load_fold_and2:1137; CHECK: # %bb.0:1138; CHECK-NEXT: movq %rsi, %rax1139; CHECK-NEXT: andq (%rdi), %rax1140; CHECK-NEXT: retq1141 %v = load atomic i64, ptr %p unordered, align 81142 %ret = and i64 %v, %v21143 ret i64 %ret1144}1145 1146define i64 @load_fold_and3(ptr %p1, ptr %p2) {1147; CHECK-O0-LABEL: load_fold_and3:1148; CHECK-O0: # %bb.0:1149; CHECK-O0-NEXT: movq (%rdi), %rax1150; CHECK-O0-NEXT: andq (%rsi), %rax1151; CHECK-O0-NEXT: retq1152;1153; CHECK-O3-LABEL: load_fold_and3:1154; CHECK-O3: # %bb.0:1155; CHECK-O3-NEXT: movq (%rsi), %rax1156; CHECK-O3-NEXT: andq (%rdi), %rax1157; CHECK-O3-NEXT: retq1158 %v = load atomic i64, ptr %p1 unordered, align 81159 %v2 = load atomic i64, ptr %p2 unordered, align 81160 %ret = and i64 %v, %v21161 ret i64 %ret1162}1163 1164; Legal, as expected1165define i64 @load_fold_or1(ptr %p) {1166; CHECK-LABEL: load_fold_or1:1167; CHECK: # %bb.0:1168; CHECK-NEXT: movq (%rdi), %rax1169; CHECK-NEXT: orq $15, %rax1170; CHECK-NEXT: retq1171 %v = load atomic i64, ptr %p unordered, align 81172 %ret = or i64 %v, 151173 ret i64 %ret1174}1175 1176define i64 @load_fold_or2(ptr %p, i64 %v2) {1177; CHECK-LABEL: load_fold_or2:1178; CHECK: # %bb.0:1179; CHECK-NEXT: movq %rsi, %rax1180; CHECK-NEXT: orq (%rdi), %rax1181; CHECK-NEXT: retq1182 %v = load atomic i64, ptr %p unordered, align 81183 %ret = or i64 %v, %v21184 ret i64 %ret1185}1186 1187define i64 @load_fold_or3(ptr %p1, ptr %p2) {1188; CHECK-O0-LABEL: load_fold_or3:1189; CHECK-O0: # %bb.0:1190; CHECK-O0-NEXT: movq (%rdi), %rax1191; CHECK-O0-NEXT: orq (%rsi), %rax1192; CHECK-O0-NEXT: retq1193;1194; CHECK-O3-LABEL: load_fold_or3:1195; CHECK-O3: # %bb.0:1196; CHECK-O3-NEXT: movq (%rsi), %rax1197; CHECK-O3-NEXT: orq (%rdi), %rax1198; CHECK-O3-NEXT: retq1199 %v = load atomic i64, ptr %p1 unordered, align 81200 %v2 = load atomic i64, ptr %p2 unordered, align 81201 %ret = or i64 %v, %v21202 ret i64 %ret1203}1204 1205; Legal, as expected1206define i64 @load_fold_xor1(ptr %p) {1207; CHECK-LABEL: load_fold_xor1:1208; CHECK: # %bb.0:1209; CHECK-NEXT: movq (%rdi), %rax1210; CHECK-NEXT: xorq $15, %rax1211; CHECK-NEXT: retq1212 %v = load atomic i64, ptr %p unordered, align 81213 %ret = xor i64 %v, 151214 ret i64 %ret1215}1216 1217define i64 @load_fold_xor2(ptr %p, i64 %v2) {1218; CHECK-LABEL: load_fold_xor2:1219; CHECK: # %bb.0:1220; CHECK-NEXT: movq %rsi, %rax1221; CHECK-NEXT: xorq (%rdi), %rax1222; CHECK-NEXT: retq1223 %v = load atomic i64, ptr %p unordered, align 81224 %ret = xor i64 %v, %v21225 ret i64 %ret1226}1227 1228define i64 @load_fold_xor3(ptr %p1, ptr %p2) {1229; CHECK-O0-LABEL: load_fold_xor3:1230; CHECK-O0: # %bb.0:1231; CHECK-O0-NEXT: movq (%rdi), %rax1232; CHECK-O0-NEXT: xorq (%rsi), %rax1233; CHECK-O0-NEXT: retq1234;1235; CHECK-O3-LABEL: load_fold_xor3:1236; CHECK-O3: # %bb.0:1237; CHECK-O3-NEXT: movq (%rsi), %rax1238; CHECK-O3-NEXT: xorq (%rdi), %rax1239; CHECK-O3-NEXT: retq1240 %v = load atomic i64, ptr %p1 unordered, align 81241 %v2 = load atomic i64, ptr %p2 unordered, align 81242 %ret = xor i64 %v, %v21243 ret i64 %ret1244}1245 1246define i1 @load_fold_icmp1(ptr %p) {1247; CHECK-O0-LABEL: load_fold_icmp1:1248; CHECK-O0: # %bb.0:1249; CHECK-O0-NEXT: movq (%rdi), %rax1250; CHECK-O0-NEXT: subq $15, %rax1251; CHECK-O0-NEXT: sete %al1252; CHECK-O0-NEXT: retq1253;1254; CHECK-O3-LABEL: load_fold_icmp1:1255; CHECK-O3: # %bb.0:1256; CHECK-O3-NEXT: cmpq $15, (%rdi)1257; CHECK-O3-NEXT: sete %al1258; CHECK-O3-NEXT: retq1259 %v = load atomic i64, ptr %p unordered, align 81260 %ret = icmp eq i64 %v, 151261 ret i1 %ret1262}1263 1264define i1 @load_fold_icmp2(ptr %p, i64 %v2) {1265; CHECK-O0-LABEL: load_fold_icmp2:1266; CHECK-O0: # %bb.0:1267; CHECK-O0-NEXT: movq (%rdi), %rax1268; CHECK-O0-NEXT: subq %rsi, %rax1269; CHECK-O0-NEXT: sete %al1270; CHECK-O0-NEXT: retq1271;1272; CHECK-O3-LABEL: load_fold_icmp2:1273; CHECK-O3: # %bb.0:1274; CHECK-O3-NEXT: cmpq %rsi, (%rdi)1275; CHECK-O3-NEXT: sete %al1276; CHECK-O3-NEXT: retq1277 %v = load atomic i64, ptr %p unordered, align 81278 %ret = icmp eq i64 %v, %v21279 ret i1 %ret1280}1281 1282define i1 @load_fold_icmp3(ptr %p1, ptr %p2) {1283; CHECK-O0-LABEL: load_fold_icmp3:1284; CHECK-O0: # %bb.0:1285; CHECK-O0-NEXT: movq (%rdi), %rax1286; CHECK-O0-NEXT: movq (%rsi), %rcx1287; CHECK-O0-NEXT: subq %rcx, %rax1288; CHECK-O0-NEXT: sete %al1289; CHECK-O0-NEXT: retq1290;1291; CHECK-O3-LABEL: load_fold_icmp3:1292; CHECK-O3: # %bb.0:1293; CHECK-O3-NEXT: movq (%rsi), %rax1294; CHECK-O3-NEXT: cmpq %rax, (%rdi)1295; CHECK-O3-NEXT: sete %al1296; CHECK-O3-NEXT: retq1297 %v = load atomic i64, ptr %p1 unordered, align 81298 %v2 = load atomic i64, ptr %p2 unordered, align 81299 %ret = icmp eq i64 %v, %v21300 ret i1 %ret1301}1302 1303 1304;; The next batch of tests check for read-modify-write patterns1305;; Legally, it's okay to use a memory operand here as long as the operand1306;; is well aligned (i.e. doesn't cross a cache line boundary). We are1307;; required not to narrow the store though!1308 1309; Legal, as expected1310define void @rmw_fold_add1(ptr %p, i64 %v) {1311; CHECK-O0-LABEL: rmw_fold_add1:1312; CHECK-O0: # %bb.0:1313; CHECK-O0-NEXT: movq (%rdi), %rax1314; CHECK-O0-NEXT: addq $15, %rax1315; CHECK-O0-NEXT: movq %rax, (%rdi)1316; CHECK-O0-NEXT: retq1317;1318; CHECK-O3-LABEL: rmw_fold_add1:1319; CHECK-O3: # %bb.0:1320; CHECK-O3-NEXT: addq $15, (%rdi)1321; CHECK-O3-NEXT: retq1322 %prev = load atomic i64, ptr %p unordered, align 81323 %val = add i64 %prev, 151324 store atomic i64 %val, ptr %p unordered, align 81325 ret void1326}1327 1328; Legal, as expected1329define void @rmw_fold_add2(ptr %p, i64 %v) {1330; CHECK-O0-LABEL: rmw_fold_add2:1331; CHECK-O0: # %bb.0:1332; CHECK-O0-NEXT: movq (%rdi), %rax1333; CHECK-O0-NEXT: addq %rsi, %rax1334; CHECK-O0-NEXT: movq %rax, (%rdi)1335; CHECK-O0-NEXT: retq1336;1337; CHECK-O3-LABEL: rmw_fold_add2:1338; CHECK-O3: # %bb.0:1339; CHECK-O3-NEXT: addq %rsi, (%rdi)1340; CHECK-O3-NEXT: retq1341 %prev = load atomic i64, ptr %p unordered, align 81342 %val = add i64 %prev, %v1343 store atomic i64 %val, ptr %p unordered, align 81344 ret void1345}1346 1347; Legal, as expected1348define void @rmw_fold_sub1(ptr %p, i64 %v) {1349; CHECK-O0-LABEL: rmw_fold_sub1:1350; CHECK-O0: # %bb.0:1351; CHECK-O0-NEXT: movq (%rdi), %rax1352; CHECK-O0-NEXT: addq $-15, %rax1353; CHECK-O0-NEXT: movq %rax, (%rdi)1354; CHECK-O0-NEXT: retq1355;1356; CHECK-O3-LABEL: rmw_fold_sub1:1357; CHECK-O3: # %bb.0:1358; CHECK-O3-NEXT: addq $-15, (%rdi)1359; CHECK-O3-NEXT: retq1360 %prev = load atomic i64, ptr %p unordered, align 81361 %val = sub i64 %prev, 151362 store atomic i64 %val, ptr %p unordered, align 81363 ret void1364}1365 1366; Legal, as expected1367define void @rmw_fold_sub2(ptr %p, i64 %v) {1368; CHECK-O0-LABEL: rmw_fold_sub2:1369; CHECK-O0: # %bb.0:1370; CHECK-O0-NEXT: movq (%rdi), %rax1371; CHECK-O0-NEXT: subq %rsi, %rax1372; CHECK-O0-NEXT: movq %rax, (%rdi)1373; CHECK-O0-NEXT: retq1374;1375; CHECK-O3-LABEL: rmw_fold_sub2:1376; CHECK-O3: # %bb.0:1377; CHECK-O3-NEXT: subq %rsi, (%rdi)1378; CHECK-O3-NEXT: retq1379 %prev = load atomic i64, ptr %p unordered, align 81380 %val = sub i64 %prev, %v1381 store atomic i64 %val, ptr %p unordered, align 81382 ret void1383}1384 1385; Legal, as expected1386define void @rmw_fold_mul1(ptr %p, i64 %v) {1387; CHECK-LABEL: rmw_fold_mul1:1388; CHECK: # %bb.0:1389; CHECK-NEXT: movq (%rdi), %rax1390; CHECK-NEXT: leaq (%rax,%rax,4), %rax1391; CHECK-NEXT: leaq (%rax,%rax,2), %rax1392; CHECK-NEXT: movq %rax, (%rdi)1393; CHECK-NEXT: retq1394 %prev = load atomic i64, ptr %p unordered, align 81395 %val = mul i64 %prev, 151396 store atomic i64 %val, ptr %p unordered, align 81397 ret void1398}1399 1400; Legal to fold (TODO)1401define void @rmw_fold_mul2(ptr %p, i64 %v) {1402; CHECK-O0-LABEL: rmw_fold_mul2:1403; CHECK-O0: # %bb.0:1404; CHECK-O0-NEXT: movq (%rdi), %rax1405; CHECK-O0-NEXT: imulq %rsi, %rax1406; CHECK-O0-NEXT: movq %rax, (%rdi)1407; CHECK-O0-NEXT: retq1408;1409; CHECK-O3-LABEL: rmw_fold_mul2:1410; CHECK-O3: # %bb.0:1411; CHECK-O3-NEXT: imulq (%rdi), %rsi1412; CHECK-O3-NEXT: movq %rsi, (%rdi)1413; CHECK-O3-NEXT: retq1414 %prev = load atomic i64, ptr %p unordered, align 81415 %val = mul i64 %prev, %v1416 store atomic i64 %val, ptr %p unordered, align 81417 ret void1418}1419 1420; Legal, as expected1421define void @rmw_fold_sdiv1(ptr %p, i64 %v) {1422; CHECK-O0-LABEL: rmw_fold_sdiv1:1423; CHECK-O0: # %bb.0:1424; CHECK-O0-NEXT: movq (%rdi), %rcx1425; CHECK-O0-NEXT: movabsq $-8608480567731124087, %rdx # imm = 0x88888888888888891426; CHECK-O0-NEXT: movq %rcx, %rax1427; CHECK-O0-NEXT: imulq %rdx1428; CHECK-O0-NEXT: movq %rdx, %rax1429; CHECK-O0-NEXT: addq %rcx, %rax1430; CHECK-O0-NEXT: movq %rax, %rcx1431; CHECK-O0-NEXT: shrq $63, %rcx1432; CHECK-O0-NEXT: sarq $3, %rax1433; CHECK-O0-NEXT: addq %rcx, %rax1434; CHECK-O0-NEXT: movq %rax, (%rdi)1435; CHECK-O0-NEXT: retq1436;1437; CHECK-O3-LABEL: rmw_fold_sdiv1:1438; CHECK-O3: # %bb.0:1439; CHECK-O3-NEXT: movq (%rdi), %rcx1440; CHECK-O3-NEXT: movabsq $-8608480567731124087, %rdx # imm = 0x88888888888888891441; CHECK-O3-NEXT: movq %rcx, %rax1442; CHECK-O3-NEXT: imulq %rdx1443; CHECK-O3-NEXT: addq %rcx, %rdx1444; CHECK-O3-NEXT: movq %rdx, %rax1445; CHECK-O3-NEXT: shrq $63, %rax1446; CHECK-O3-NEXT: sarq $3, %rdx1447; CHECK-O3-NEXT: addq %rax, %rdx1448; CHECK-O3-NEXT: movq %rdx, (%rdi)1449; CHECK-O3-NEXT: retq1450 %prev = load atomic i64, ptr %p unordered, align 81451 %val = sdiv i64 %prev, 151452 store atomic i64 %val, ptr %p unordered, align 81453 ret void1454}1455 1456; Legal, as expected1457define void @rmw_fold_sdiv2(ptr %p, i64 %v) {1458; CHECK-O0-LABEL: rmw_fold_sdiv2:1459; CHECK-O0: # %bb.0:1460; CHECK-O0-NEXT: movq (%rdi), %rax1461; CHECK-O0-NEXT: cqto1462; CHECK-O0-NEXT: idivq %rsi1463; CHECK-O0-NEXT: movq %rax, (%rdi)1464; CHECK-O0-NEXT: retq1465;1466; CHECK-O3-LABEL: rmw_fold_sdiv2:1467; CHECK-O3: # %bb.0:1468; CHECK-O3-NEXT: movq (%rdi), %rax1469; CHECK-O3-NEXT: movq %rax, %rcx1470; CHECK-O3-NEXT: orq %rsi, %rcx1471; CHECK-O3-NEXT: shrq $32, %rcx1472; CHECK-O3-NEXT: je .LBB74_11473; CHECK-O3-NEXT: # %bb.2:1474; CHECK-O3-NEXT: cqto1475; CHECK-O3-NEXT: idivq %rsi1476; CHECK-O3-NEXT: movq %rax, (%rdi)1477; CHECK-O3-NEXT: retq1478; CHECK-O3-NEXT: .LBB74_1:1479; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax1480; CHECK-O3-NEXT: xorl %edx, %edx1481; CHECK-O3-NEXT: divl %esi1482; CHECK-O3-NEXT: # kill: def $eax killed $eax def $rax1483; CHECK-O3-NEXT: movq %rax, (%rdi)1484; CHECK-O3-NEXT: retq1485 %prev = load atomic i64, ptr %p unordered, align 81486 %val = sdiv i64 %prev, %v1487 store atomic i64 %val, ptr %p unordered, align 81488 ret void1489}1490 1491; Legal, as expected1492define void @rmw_fold_udiv1(ptr %p, i64 %v) {1493; CHECK-LABEL: rmw_fold_udiv1:1494; CHECK: # %bb.0:1495; CHECK-NEXT: movq (%rdi), %rdx1496; CHECK-NEXT: movabsq $-8608480567731124087, %rax # imm = 0x88888888888888891497; CHECK-NEXT: mulxq %rax, %rax, %rax1498; CHECK-NEXT: shrq $3, %rax1499; CHECK-NEXT: movq %rax, (%rdi)1500; CHECK-NEXT: retq1501 %prev = load atomic i64, ptr %p unordered, align 81502 %val = udiv i64 %prev, 151503 store atomic i64 %val, ptr %p unordered, align 81504 ret void1505}1506 1507; Legal, as expected1508define void @rmw_fold_udiv2(ptr %p, i64 %v) {1509; CHECK-O0-LABEL: rmw_fold_udiv2:1510; CHECK-O0: # %bb.0:1511; CHECK-O0-NEXT: movq (%rdi), %rax1512; CHECK-O0-NEXT: xorl %ecx, %ecx1513; CHECK-O0-NEXT: movl %ecx, %edx1514; CHECK-O0-NEXT: divq %rsi1515; CHECK-O0-NEXT: movq %rax, (%rdi)1516; CHECK-O0-NEXT: retq1517;1518; CHECK-O3-LABEL: rmw_fold_udiv2:1519; CHECK-O3: # %bb.0:1520; CHECK-O3-NEXT: movq (%rdi), %rax1521; CHECK-O3-NEXT: movq %rax, %rcx1522; CHECK-O3-NEXT: orq %rsi, %rcx1523; CHECK-O3-NEXT: shrq $32, %rcx1524; CHECK-O3-NEXT: je .LBB76_11525; CHECK-O3-NEXT: # %bb.2:1526; CHECK-O3-NEXT: xorl %edx, %edx1527; CHECK-O3-NEXT: divq %rsi1528; CHECK-O3-NEXT: movq %rax, (%rdi)1529; CHECK-O3-NEXT: retq1530; CHECK-O3-NEXT: .LBB76_1:1531; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax1532; CHECK-O3-NEXT: xorl %edx, %edx1533; CHECK-O3-NEXT: divl %esi1534; CHECK-O3-NEXT: # kill: def $eax killed $eax def $rax1535; CHECK-O3-NEXT: movq %rax, (%rdi)1536; CHECK-O3-NEXT: retq1537 %prev = load atomic i64, ptr %p unordered, align 81538 %val = udiv i64 %prev, %v1539 store atomic i64 %val, ptr %p unordered, align 81540 ret void1541}1542 1543; Legal, as expected1544define void @rmw_fold_srem1(ptr %p, i64 %v) {1545; CHECK-O0-LABEL: rmw_fold_srem1:1546; CHECK-O0: # %bb.0:1547; CHECK-O0-NEXT: movq (%rdi), %rax1548; CHECK-O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill1549; CHECK-O0-NEXT: movabsq $-8608480567731124087, %rcx # imm = 0x88888888888888891550; CHECK-O0-NEXT: imulq %rcx1551; CHECK-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload1552; CHECK-O0-NEXT: movq %rdx, %rcx1553; CHECK-O0-NEXT: addq %rax, %rcx1554; CHECK-O0-NEXT: movq %rcx, %rdx1555; CHECK-O0-NEXT: shrq $63, %rdx1556; CHECK-O0-NEXT: sarq $3, %rcx1557; CHECK-O0-NEXT: addq %rdx, %rcx1558; CHECK-O0-NEXT: leaq (%rcx,%rcx,4), %rcx1559; CHECK-O0-NEXT: leaq (%rcx,%rcx,2), %rcx1560; CHECK-O0-NEXT: subq %rcx, %rax1561; CHECK-O0-NEXT: movq %rax, (%rdi)1562; CHECK-O0-NEXT: retq1563;1564; CHECK-O3-LABEL: rmw_fold_srem1:1565; CHECK-O3: # %bb.0:1566; CHECK-O3-NEXT: movq (%rdi), %rcx1567; CHECK-O3-NEXT: movabsq $-8608480567731124087, %rdx # imm = 0x88888888888888891568; CHECK-O3-NEXT: movq %rcx, %rax1569; CHECK-O3-NEXT: imulq %rdx1570; CHECK-O3-NEXT: addq %rcx, %rdx1571; CHECK-O3-NEXT: movq %rdx, %rax1572; CHECK-O3-NEXT: shrq $63, %rax1573; CHECK-O3-NEXT: sarq $3, %rdx1574; CHECK-O3-NEXT: addq %rax, %rdx1575; CHECK-O3-NEXT: leaq (%rdx,%rdx,4), %rax1576; CHECK-O3-NEXT: leaq (%rax,%rax,2), %rax1577; CHECK-O3-NEXT: subq %rax, %rcx1578; CHECK-O3-NEXT: movq %rcx, (%rdi)1579; CHECK-O3-NEXT: retq1580 %prev = load atomic i64, ptr %p unordered, align 81581 %val = srem i64 %prev, 151582 store atomic i64 %val, ptr %p unordered, align 81583 ret void1584}1585 1586; Legal, as expected1587define void @rmw_fold_srem2(ptr %p, i64 %v) {1588; CHECK-O0-LABEL: rmw_fold_srem2:1589; CHECK-O0: # %bb.0:1590; CHECK-O0-NEXT: movq (%rdi), %rax1591; CHECK-O0-NEXT: cqto1592; CHECK-O0-NEXT: idivq %rsi1593; CHECK-O0-NEXT: movq %rdx, (%rdi)1594; CHECK-O0-NEXT: retq1595;1596; CHECK-O3-LABEL: rmw_fold_srem2:1597; CHECK-O3: # %bb.0:1598; CHECK-O3-NEXT: movq (%rdi), %rax1599; CHECK-O3-NEXT: movq %rax, %rcx1600; CHECK-O3-NEXT: orq %rsi, %rcx1601; CHECK-O3-NEXT: shrq $32, %rcx1602; CHECK-O3-NEXT: je .LBB78_11603; CHECK-O3-NEXT: # %bb.2:1604; CHECK-O3-NEXT: cqto1605; CHECK-O3-NEXT: idivq %rsi1606; CHECK-O3-NEXT: movq %rdx, (%rdi)1607; CHECK-O3-NEXT: retq1608; CHECK-O3-NEXT: .LBB78_1:1609; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax1610; CHECK-O3-NEXT: xorl %edx, %edx1611; CHECK-O3-NEXT: divl %esi1612; CHECK-O3-NEXT: # kill: def $edx killed $edx def $rdx1613; CHECK-O3-NEXT: movq %rdx, (%rdi)1614; CHECK-O3-NEXT: retq1615 %prev = load atomic i64, ptr %p unordered, align 81616 %val = srem i64 %prev, %v1617 store atomic i64 %val, ptr %p unordered, align 81618 ret void1619}1620 1621; Legal, as expected1622define void @rmw_fold_urem1(ptr %p, i64 %v) {1623; CHECK-O0-LABEL: rmw_fold_urem1:1624; CHECK-O0: # %bb.0:1625; CHECK-O0-NEXT: movq (%rdi), %rax1626; CHECK-O0-NEXT: movabsq $-8608480567731124087, %rcx # imm = 0x88888888888888891627; CHECK-O0-NEXT: movq %rax, %rdx1628; CHECK-O0-NEXT: mulxq %rcx, %rcx, %rcx1629; CHECK-O0-NEXT: shrq $3, %rcx1630; CHECK-O0-NEXT: leaq (%rcx,%rcx,4), %rcx1631; CHECK-O0-NEXT: leaq (%rcx,%rcx,2), %rcx1632; CHECK-O0-NEXT: subq %rcx, %rax1633; CHECK-O0-NEXT: movq %rax, (%rdi)1634; CHECK-O0-NEXT: retq1635;1636; CHECK-O3-LABEL: rmw_fold_urem1:1637; CHECK-O3: # %bb.0:1638; CHECK-O3-NEXT: movq (%rdi), %rdx1639; CHECK-O3-NEXT: movabsq $-8608480567731124087, %rax # imm = 0x88888888888888891640; CHECK-O3-NEXT: mulxq %rax, %rax, %rax1641; CHECK-O3-NEXT: shrq $3, %rax1642; CHECK-O3-NEXT: leaq (%rax,%rax,4), %rax1643; CHECK-O3-NEXT: leaq (%rax,%rax,2), %rax1644; CHECK-O3-NEXT: subq %rax, %rdx1645; CHECK-O3-NEXT: movq %rdx, (%rdi)1646; CHECK-O3-NEXT: retq1647 %prev = load atomic i64, ptr %p unordered, align 81648 %val = urem i64 %prev, 151649 store atomic i64 %val, ptr %p unordered, align 81650 ret void1651}1652 1653; Legal, as expected1654define void @rmw_fold_urem2(ptr %p, i64 %v) {1655; CHECK-O0-LABEL: rmw_fold_urem2:1656; CHECK-O0: # %bb.0:1657; CHECK-O0-NEXT: movq (%rdi), %rax1658; CHECK-O0-NEXT: xorl %ecx, %ecx1659; CHECK-O0-NEXT: movl %ecx, %edx1660; CHECK-O0-NEXT: divq %rsi1661; CHECK-O0-NEXT: movq %rdx, (%rdi)1662; CHECK-O0-NEXT: retq1663;1664; CHECK-O3-LABEL: rmw_fold_urem2:1665; CHECK-O3: # %bb.0:1666; CHECK-O3-NEXT: movq (%rdi), %rax1667; CHECK-O3-NEXT: movq %rax, %rcx1668; CHECK-O3-NEXT: orq %rsi, %rcx1669; CHECK-O3-NEXT: shrq $32, %rcx1670; CHECK-O3-NEXT: je .LBB80_11671; CHECK-O3-NEXT: # %bb.2:1672; CHECK-O3-NEXT: xorl %edx, %edx1673; CHECK-O3-NEXT: divq %rsi1674; CHECK-O3-NEXT: movq %rdx, (%rdi)1675; CHECK-O3-NEXT: retq1676; CHECK-O3-NEXT: .LBB80_1:1677; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax1678; CHECK-O3-NEXT: xorl %edx, %edx1679; CHECK-O3-NEXT: divl %esi1680; CHECK-O3-NEXT: # kill: def $edx killed $edx def $rdx1681; CHECK-O3-NEXT: movq %rdx, (%rdi)1682; CHECK-O3-NEXT: retq1683 %prev = load atomic i64, ptr %p unordered, align 81684 %val = urem i64 %prev, %v1685 store atomic i64 %val, ptr %p unordered, align 81686 ret void1687}1688 1689; Legal to fold (TODO)1690define void @rmw_fold_shl1(ptr %p, i64 %v) {1691; CHECK-LABEL: rmw_fold_shl1:1692; CHECK: # %bb.0:1693; CHECK-NEXT: movq (%rdi), %rax1694; CHECK-NEXT: shlq $15, %rax1695; CHECK-NEXT: movq %rax, (%rdi)1696; CHECK-NEXT: retq1697 %prev = load atomic i64, ptr %p unordered, align 81698 %val = shl i64 %prev, 151699 store atomic i64 %val, ptr %p unordered, align 81700 ret void1701}1702 1703; Legal to fold (TODO)1704define void @rmw_fold_shl2(ptr %p, i64 %v) {1705; CHECK-O0-LABEL: rmw_fold_shl2:1706; CHECK-O0: # %bb.0:1707; CHECK-O0-NEXT: movq (%rdi), %rax1708; CHECK-O0-NEXT: movb %sil, %dl1709; CHECK-O0-NEXT: # implicit-def: $rcx1710; CHECK-O0-NEXT: movb %dl, %cl1711; CHECK-O0-NEXT: shlxq %rcx, %rax, %rax1712; CHECK-O0-NEXT: movq %rax, (%rdi)1713; CHECK-O0-NEXT: retq1714;1715; CHECK-O3-LABEL: rmw_fold_shl2:1716; CHECK-O3: # %bb.0:1717; CHECK-O3-NEXT: shlxq %rsi, (%rdi), %rax1718; CHECK-O3-NEXT: movq %rax, (%rdi)1719; CHECK-O3-NEXT: retq1720 %prev = load atomic i64, ptr %p unordered, align 81721 %val = shl i64 %prev, %v1722 store atomic i64 %val, ptr %p unordered, align 81723 ret void1724}1725 1726; Legal to fold (TODO)1727define void @rmw_fold_lshr1(ptr %p, i64 %v) {1728; CHECK-LABEL: rmw_fold_lshr1:1729; CHECK: # %bb.0:1730; CHECK-NEXT: movq (%rdi), %rax1731; CHECK-NEXT: shrq $15, %rax1732; CHECK-NEXT: movq %rax, (%rdi)1733; CHECK-NEXT: retq1734 %prev = load atomic i64, ptr %p unordered, align 81735 %val = lshr i64 %prev, 151736 store atomic i64 %val, ptr %p unordered, align 81737 ret void1738}1739 1740; Legal to fold (TODO)1741define void @rmw_fold_lshr2(ptr %p, i64 %v) {1742; CHECK-O0-LABEL: rmw_fold_lshr2:1743; CHECK-O0: # %bb.0:1744; CHECK-O0-NEXT: movq (%rdi), %rax1745; CHECK-O0-NEXT: movb %sil, %dl1746; CHECK-O0-NEXT: # implicit-def: $rcx1747; CHECK-O0-NEXT: movb %dl, %cl1748; CHECK-O0-NEXT: shrxq %rcx, %rax, %rax1749; CHECK-O0-NEXT: movq %rax, (%rdi)1750; CHECK-O0-NEXT: retq1751;1752; CHECK-O3-LABEL: rmw_fold_lshr2:1753; CHECK-O3: # %bb.0:1754; CHECK-O3-NEXT: shrxq %rsi, (%rdi), %rax1755; CHECK-O3-NEXT: movq %rax, (%rdi)1756; CHECK-O3-NEXT: retq1757 %prev = load atomic i64, ptr %p unordered, align 81758 %val = lshr i64 %prev, %v1759 store atomic i64 %val, ptr %p unordered, align 81760 ret void1761}1762 1763; Legal to fold (TODO)1764define void @rmw_fold_ashr1(ptr %p, i64 %v) {1765; CHECK-LABEL: rmw_fold_ashr1:1766; CHECK: # %bb.0:1767; CHECK-NEXT: movq (%rdi), %rax1768; CHECK-NEXT: sarq $15, %rax1769; CHECK-NEXT: movq %rax, (%rdi)1770; CHECK-NEXT: retq1771 %prev = load atomic i64, ptr %p unordered, align 81772 %val = ashr i64 %prev, 151773 store atomic i64 %val, ptr %p unordered, align 81774 ret void1775}1776 1777; Legal to fold (TODO)1778define void @rmw_fold_ashr2(ptr %p, i64 %v) {1779; CHECK-O0-LABEL: rmw_fold_ashr2:1780; CHECK-O0: # %bb.0:1781; CHECK-O0-NEXT: movq (%rdi), %rax1782; CHECK-O0-NEXT: movb %sil, %dl1783; CHECK-O0-NEXT: # implicit-def: $rcx1784; CHECK-O0-NEXT: movb %dl, %cl1785; CHECK-O0-NEXT: sarxq %rcx, %rax, %rax1786; CHECK-O0-NEXT: movq %rax, (%rdi)1787; CHECK-O0-NEXT: retq1788;1789; CHECK-O3-LABEL: rmw_fold_ashr2:1790; CHECK-O3: # %bb.0:1791; CHECK-O3-NEXT: sarxq %rsi, (%rdi), %rax1792; CHECK-O3-NEXT: movq %rax, (%rdi)1793; CHECK-O3-NEXT: retq1794 %prev = load atomic i64, ptr %p unordered, align 81795 %val = ashr i64 %prev, %v1796 store atomic i64 %val, ptr %p unordered, align 81797 ret void1798}1799 1800; Legal, as expected1801define void @rmw_fold_and1(ptr %p, i64 %v) {1802; CHECK-O0-LABEL: rmw_fold_and1:1803; CHECK-O0: # %bb.0:1804; CHECK-O0-NEXT: movq (%rdi), %rax1805; CHECK-O0-NEXT: # kill: def $eax killed $eax killed $rax1806; CHECK-O0-NEXT: andl $15, %eax1807; CHECK-O0-NEXT: # kill: def $rax killed $eax1808; CHECK-O0-NEXT: movq %rax, (%rdi)1809; CHECK-O0-NEXT: retq1810;1811; CHECK-O3-LABEL: rmw_fold_and1:1812; CHECK-O3: # %bb.0:1813; CHECK-O3-NEXT: andq $15, (%rdi)1814; CHECK-O3-NEXT: retq1815 %prev = load atomic i64, ptr %p unordered, align 81816 %val = and i64 %prev, 151817 store atomic i64 %val, ptr %p unordered, align 81818 ret void1819}1820 1821; Legal, as expected1822define void @rmw_fold_and2(ptr %p, i64 %v) {1823; CHECK-O0-LABEL: rmw_fold_and2:1824; CHECK-O0: # %bb.0:1825; CHECK-O0-NEXT: movq (%rdi), %rax1826; CHECK-O0-NEXT: andq %rsi, %rax1827; CHECK-O0-NEXT: movq %rax, (%rdi)1828; CHECK-O0-NEXT: retq1829;1830; CHECK-O3-LABEL: rmw_fold_and2:1831; CHECK-O3: # %bb.0:1832; CHECK-O3-NEXT: andq %rsi, (%rdi)1833; CHECK-O3-NEXT: retq1834 %prev = load atomic i64, ptr %p unordered, align 81835 %val = and i64 %prev, %v1836 store atomic i64 %val, ptr %p unordered, align 81837 ret void1838}1839 1840; Legal, as expected1841define void @rmw_fold_or1(ptr %p, i64 %v) {1842; CHECK-O0-LABEL: rmw_fold_or1:1843; CHECK-O0: # %bb.0:1844; CHECK-O0-NEXT: movq (%rdi), %rax1845; CHECK-O0-NEXT: orq $15, %rax1846; CHECK-O0-NEXT: movq %rax, (%rdi)1847; CHECK-O0-NEXT: retq1848;1849; CHECK-O3-LABEL: rmw_fold_or1:1850; CHECK-O3: # %bb.0:1851; CHECK-O3-NEXT: orq $15, (%rdi)1852; CHECK-O3-NEXT: retq1853 %prev = load atomic i64, ptr %p unordered, align 81854 %val = or i64 %prev, 151855 store atomic i64 %val, ptr %p unordered, align 81856 ret void1857}1858 1859; Legal, as expected1860define void @rmw_fold_or2(ptr %p, i64 %v) {1861; CHECK-O0-LABEL: rmw_fold_or2:1862; CHECK-O0: # %bb.0:1863; CHECK-O0-NEXT: movq (%rdi), %rax1864; CHECK-O0-NEXT: orq %rsi, %rax1865; CHECK-O0-NEXT: movq %rax, (%rdi)1866; CHECK-O0-NEXT: retq1867;1868; CHECK-O3-LABEL: rmw_fold_or2:1869; CHECK-O3: # %bb.0:1870; CHECK-O3-NEXT: orq %rsi, (%rdi)1871; CHECK-O3-NEXT: retq1872 %prev = load atomic i64, ptr %p unordered, align 81873 %val = or i64 %prev, %v1874 store atomic i64 %val, ptr %p unordered, align 81875 ret void1876}1877 1878; Legal, as expected1879define void @rmw_fold_xor1(ptr %p, i64 %v) {1880; CHECK-O0-LABEL: rmw_fold_xor1:1881; CHECK-O0: # %bb.0:1882; CHECK-O0-NEXT: movq (%rdi), %rax1883; CHECK-O0-NEXT: xorq $15, %rax1884; CHECK-O0-NEXT: movq %rax, (%rdi)1885; CHECK-O0-NEXT: retq1886;1887; CHECK-O3-LABEL: rmw_fold_xor1:1888; CHECK-O3: # %bb.0:1889; CHECK-O3-NEXT: xorq $15, (%rdi)1890; CHECK-O3-NEXT: retq1891 %prev = load atomic i64, ptr %p unordered, align 81892 %val = xor i64 %prev, 151893 store atomic i64 %val, ptr %p unordered, align 81894 ret void1895}1896 1897; Legal, as expected1898define void @rmw_fold_xor2(ptr %p, i64 %v) {1899; CHECK-O0-LABEL: rmw_fold_xor2:1900; CHECK-O0: # %bb.0:1901; CHECK-O0-NEXT: movq (%rdi), %rax1902; CHECK-O0-NEXT: xorq %rsi, %rax1903; CHECK-O0-NEXT: movq %rax, (%rdi)1904; CHECK-O0-NEXT: retq1905;1906; CHECK-O3-LABEL: rmw_fold_xor2:1907; CHECK-O3: # %bb.0:1908; CHECK-O3-NEXT: xorq %rsi, (%rdi)1909; CHECK-O3-NEXT: retq1910 %prev = load atomic i64, ptr %p unordered, align 81911 %val = xor i64 %prev, %v1912 store atomic i64 %val, ptr %p unordered, align 81913 ret void1914}1915 1916;; The next batch test truncations, in combination w/operations which could1917;; be folded against the memory operation.1918 1919; Legal to reduce the load width (TODO)1920define i32 @fold_trunc(ptr %p) {1921; CHECK-LABEL: fold_trunc:1922; CHECK: # %bb.0:1923; CHECK-NEXT: movq (%rdi), %rax1924; CHECK-NEXT: # kill: def $eax killed $eax killed $rax1925; CHECK-NEXT: retq1926 %v = load atomic i64, ptr %p unordered, align 81927 %ret = trunc i64 %v to i321928 ret i32 %ret1929}1930 1931; Legal to reduce the load width and fold the load (TODO)1932define i32 @fold_trunc_add(ptr %p, i32 %v2) {1933; CHECK-O0-LABEL: fold_trunc_add:1934; CHECK-O0: # %bb.0:1935; CHECK-O0-NEXT: movq (%rdi), %rax1936; CHECK-O0-NEXT: # kill: def $eax killed $eax killed $rax1937; CHECK-O0-NEXT: addl %esi, %eax1938; CHECK-O0-NEXT: retq1939;1940; CHECK-O3-LABEL: fold_trunc_add:1941; CHECK-O3: # %bb.0:1942; CHECK-O3-NEXT: movq (%rdi), %rax1943; CHECK-O3-NEXT: addl %esi, %eax1944; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax1945; CHECK-O3-NEXT: retq1946 %v = load atomic i64, ptr %p unordered, align 81947 %trunc = trunc i64 %v to i321948 %ret = add i32 %trunc, %v21949 ret i32 %ret1950}1951 1952; Legal to reduce the load width and fold the load (TODO)1953define i32 @fold_trunc_and(ptr %p, i32 %v2) {1954; CHECK-O0-LABEL: fold_trunc_and:1955; CHECK-O0: # %bb.0:1956; CHECK-O0-NEXT: movq (%rdi), %rax1957; CHECK-O0-NEXT: # kill: def $eax killed $eax killed $rax1958; CHECK-O0-NEXT: andl %esi, %eax1959; CHECK-O0-NEXT: retq1960;1961; CHECK-O3-LABEL: fold_trunc_and:1962; CHECK-O3: # %bb.0:1963; CHECK-O3-NEXT: movq (%rdi), %rax1964; CHECK-O3-NEXT: andl %esi, %eax1965; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax1966; CHECK-O3-NEXT: retq1967 %v = load atomic i64, ptr %p unordered, align 81968 %trunc = trunc i64 %v to i321969 %ret = and i32 %trunc, %v21970 ret i32 %ret1971}1972 1973; Legal to reduce the load width and fold the load (TODO)1974define i32 @fold_trunc_or(ptr %p, i32 %v2) {1975; CHECK-O0-LABEL: fold_trunc_or:1976; CHECK-O0: # %bb.0:1977; CHECK-O0-NEXT: movq (%rdi), %rax1978; CHECK-O0-NEXT: # kill: def $eax killed $eax killed $rax1979; CHECK-O0-NEXT: orl %esi, %eax1980; CHECK-O0-NEXT: retq1981;1982; CHECK-O3-LABEL: fold_trunc_or:1983; CHECK-O3: # %bb.0:1984; CHECK-O3-NEXT: movq (%rdi), %rax1985; CHECK-O3-NEXT: orl %esi, %eax1986; CHECK-O3-NEXT: # kill: def $eax killed $eax killed $rax1987; CHECK-O3-NEXT: retq1988 %v = load atomic i64, ptr %p unordered, align 81989 %trunc = trunc i64 %v to i321990 %ret = or i32 %trunc, %v21991 ret i32 %ret1992}1993 1994; It's tempting to split the wide load into two smaller byte loads1995; to reduce memory traffic, but this would be illegal for a atomic load1996define i32 @split_load(ptr %p) {1997; CHECK-O0-LABEL: split_load:1998; CHECK-O0: # %bb.0:1999; CHECK-O0-NEXT: movq (%rdi), %rcx2000; CHECK-O0-NEXT: movb %cl, %al2001; CHECK-O0-NEXT: shrq $32, %rcx2002; CHECK-O0-NEXT: # kill: def $cl killed $cl killed $rcx2003; CHECK-O0-NEXT: orb %cl, %al2004; CHECK-O0-NEXT: movzbl %al, %eax2005; CHECK-O0-NEXT: retq2006;2007; CHECK-O3-LABEL: split_load:2008; CHECK-O3: # %bb.0:2009; CHECK-O3-NEXT: movq (%rdi), %rax2010; CHECK-O3-NEXT: movq %rax, %rcx2011; CHECK-O3-NEXT: shrq $32, %rcx2012; CHECK-O3-NEXT: orl %eax, %ecx2013; CHECK-O3-NEXT: movzbl %cl, %eax2014; CHECK-O3-NEXT: retq2015 %v = load atomic i64, ptr %p unordered, align 82016 %b1 = trunc i64 %v to i82017 %v.shift = lshr i64 %v, 322018 %b2 = trunc i64 %v.shift to i82019 %or = or i8 %b1, %b22020 %ret = zext i8 %or to i322021 ret i32 %ret2022}2023 2024;; A collection of simple memory forwarding tests. Nothing particular2025;; interesting semantic wise, just demonstrating obvious missed transforms.2026 2027@Zero = constant i64 02028 2029; TODO: should return constant2030define i64 @constant_folding(ptr %p) {2031; CHECK-LABEL: constant_folding:2032; CHECK: # %bb.0:2033; CHECK-NEXT: movq (%rdi), %rax2034; CHECK-NEXT: retq2035 %v = load atomic i64, ptr %p unordered, align 82036 ret i64 %v2037}2038 2039; Legal to forward and fold (TODO)2040define i64 @load_forwarding(ptr %p) {2041; CHECK-LABEL: load_forwarding:2042; CHECK: # %bb.0:2043; CHECK-NEXT: movq (%rdi), %rax2044; CHECK-NEXT: orq (%rdi), %rax2045; CHECK-NEXT: retq2046 %v = load atomic i64, ptr %p unordered, align 82047 %v2 = load atomic i64, ptr %p unordered, align 82048 %ret = or i64 %v, %v22049 ret i64 %ret2050}2051 2052; Legal to forward (TODO)2053define i64 @store_forward(ptr %p, i64 %v) {2054; CHECK-LABEL: store_forward:2055; CHECK: # %bb.0:2056; CHECK-NEXT: movq %rsi, (%rdi)2057; CHECK-NEXT: movq (%rdi), %rax2058; CHECK-NEXT: retq2059 store atomic i64 %v, ptr %p unordered, align 82060 %ret = load atomic i64, ptr %p unordered, align 82061 ret i64 %ret2062}2063 2064; Legal to kill (TODO)2065define void @dead_writeback(ptr %p) {2066; CHECK-LABEL: dead_writeback:2067; CHECK: # %bb.0:2068; CHECK-NEXT: movq (%rdi), %rax2069; CHECK-NEXT: movq %rax, (%rdi)2070; CHECK-NEXT: retq2071 %v = load atomic i64, ptr %p unordered, align 82072 store atomic i64 %v, ptr %p unordered, align 82073 ret void2074}2075 2076; Legal to kill (TODO)2077define void @dead_store(ptr %p, i64 %v) {2078; CHECK-LABEL: dead_store:2079; CHECK: # %bb.0:2080; CHECK-NEXT: movq $0, (%rdi)2081; CHECK-NEXT: movq %rsi, (%rdi)2082; CHECK-NEXT: retq2083 store atomic i64 0, ptr %p unordered, align 82084 store atomic i64 %v, ptr %p unordered, align 82085 ret void2086}2087 2088;; The next batch of tests ensure that we don't try to fold a load into a2089;; use where the code motion implied for the load is prevented by a fence.2090;; Note: We're checking that the load doesn't get moved below the fence as2091;; part of folding, but is technically legal to lift the add above the fence.2092;; If that were to happen, please rewrite the test to ensure load movement2093;; isn't violated.2094 2095define i64 @nofold_fence(ptr %p) {2096; CHECK-LABEL: nofold_fence:2097; CHECK: # %bb.0:2098; CHECK-NEXT: movq (%rdi), %rax2099; CHECK-NEXT: lock orl $0, -{{[0-9]+}}(%rsp)2100; CHECK-NEXT: addq $15, %rax2101; CHECK-NEXT: retq2102 %v = load atomic i64, ptr %p unordered, align 82103 fence seq_cst2104 %ret = add i64 %v, 152105 ret i64 %ret2106}2107 2108define i64 @nofold_fence_acquire(ptr %p) {2109; CHECK-LABEL: nofold_fence_acquire:2110; CHECK: # %bb.0:2111; CHECK-NEXT: movq (%rdi), %rax2112; CHECK-NEXT: #MEMBARRIER2113; CHECK-NEXT: addq $15, %rax2114; CHECK-NEXT: retq2115 %v = load atomic i64, ptr %p unordered, align 82116 fence acquire2117 %ret = add i64 %v, 152118 ret i64 %ret2119}2120 2121 2122define i64 @nofold_stfence(ptr %p) {2123; CHECK-LABEL: nofold_stfence:2124; CHECK: # %bb.0:2125; CHECK-NEXT: movq (%rdi), %rax2126; CHECK-NEXT: #MEMBARRIER2127; CHECK-NEXT: addq $15, %rax2128; CHECK-NEXT: retq2129 %v = load atomic i64, ptr %p unordered, align 82130 fence syncscope("singlethread") seq_cst2131 %ret = add i64 %v, 152132 ret i64 %ret2133}2134 2135;; Next, test how well we can fold invariant loads.2136 2137@Constant = external dso_local constant i642138 2139define i64 @fold_constant(i64 %arg) {2140; CHECK-O0-LABEL: fold_constant:2141; CHECK-O0: # %bb.0:2142; CHECK-O0-NEXT: movq %rdi, %rax2143; CHECK-O0-NEXT: addq Constant, %rax2144; CHECK-O0-NEXT: retq2145;2146; CHECK-O3-LABEL: fold_constant:2147; CHECK-O3: # %bb.0:2148; CHECK-O3-NEXT: movq %rdi, %rax2149; CHECK-O3-NEXT: addq Constant(%rip), %rax2150; CHECK-O3-NEXT: retq2151 %v = load atomic i64, ptr @Constant unordered, align 82152 %ret = add i64 %v, %arg2153 ret i64 %ret2154}2155 2156define i64 @fold_constant_clobber(ptr %p, i64 %arg) {2157; CHECK-LABEL: fold_constant_clobber:2158; CHECK: # %bb.0:2159; CHECK-NEXT: movq Constant(%rip), %rax2160; CHECK-NEXT: movq $5, (%rdi)2161; CHECK-NEXT: addq %rsi, %rax2162; CHECK-NEXT: retq2163 %v = load atomic i64, ptr @Constant unordered, align 82164 store i64 5, ptr %p2165 %ret = add i64 %v, %arg2166 ret i64 %ret2167}2168 2169define i64 @fold_constant_fence(i64 %arg) {2170; CHECK-LABEL: fold_constant_fence:2171; CHECK: # %bb.0:2172; CHECK-NEXT: movq Constant(%rip), %rax2173; CHECK-NEXT: lock orl $0, -{{[0-9]+}}(%rsp)2174; CHECK-NEXT: addq %rdi, %rax2175; CHECK-NEXT: retq2176 %v = load atomic i64, ptr @Constant unordered, align 82177 fence seq_cst2178 %ret = add i64 %v, %arg2179 ret i64 %ret2180}2181 2182define i64 @fold_invariant_clobber(ptr dereferenceable(8) %p, i64 %arg) {2183; CHECK-LABEL: fold_invariant_clobber:2184; CHECK: # %bb.0:2185; CHECK-NEXT: movq (%rdi), %rax2186; CHECK-NEXT: movq $5, (%rdi)2187; CHECK-NEXT: addq %rsi, %rax2188; CHECK-NEXT: retq2189 %v = load atomic i64, ptr %p unordered, align 8, !invariant.load !{}2190 store i64 5, ptr %p2191 %ret = add i64 %v, %arg2192 ret i64 %ret2193}2194 2195 2196define i64 @fold_invariant_fence(ptr dereferenceable(8) %p, i64 %arg) {2197; CHECK-LABEL: fold_invariant_fence:2198; CHECK: # %bb.0:2199; CHECK-NEXT: movq (%rdi), %rax2200; CHECK-NEXT: lock orl $0, -{{[0-9]+}}(%rsp)2201; CHECK-NEXT: addq %rsi, %rax2202; CHECK-NEXT: retq2203 %v = load atomic i64, ptr %p unordered, align 8, !invariant.load !{}2204 fence seq_cst2205 %ret = add i64 %v, %arg2206 ret i64 %ret2207}2208 2209 2210; Exercise a few cases involving any extend idioms2211 2212define i16 @load_i8_anyext_i16(ptr %ptr) {2213; CHECK-O0-LABEL: load_i8_anyext_i16:2214; CHECK-O0: # %bb.0:2215; CHECK-O0-NEXT: movb (%rdi), %al2216; CHECK-O0-NEXT: movzbl %al, %eax2217; CHECK-O0-NEXT: # kill: def $ax killed $ax killed $eax2218; CHECK-O0-NEXT: retq2219;2220; CHECK-O3-LABEL: load_i8_anyext_i16:2221; CHECK-O3: # %bb.0:2222; CHECK-O3-NEXT: movzbl (%rdi), %eax2223; CHECK-O3-NEXT: # kill: def $ax killed $ax killed $eax2224; CHECK-O3-NEXT: retq2225 %v = load atomic i8, ptr %ptr unordered, align 22226 %vec = insertelement <2 x i8> undef, i8 %v, i32 02227 %res = bitcast <2 x i8> %vec to i162228 ret i16 %res2229}2230 2231define i32 @load_i8_anyext_i32(ptr %ptr) {2232; CHECK-O0-LABEL: load_i8_anyext_i32:2233; CHECK-O0: # %bb.0:2234; CHECK-O0-NEXT: movb (%rdi), %al2235; CHECK-O0-NEXT: movzbl %al, %eax2236; CHECK-O0-NEXT: retq2237;2238; CHECK-O3-LABEL: load_i8_anyext_i32:2239; CHECK-O3: # %bb.0:2240; CHECK-O3-NEXT: movzbl (%rdi), %eax2241; CHECK-O3-NEXT: retq2242 %v = load atomic i8, ptr %ptr unordered, align 42243 %vec = insertelement <4 x i8> undef, i8 %v, i32 02244 %res = bitcast <4 x i8> %vec to i322245 ret i32 %res2246}2247 2248define i32 @load_i16_anyext_i32(ptr %ptr) {2249; CHECK-O0-LABEL: load_i16_anyext_i32:2250; CHECK-O0: # %bb.0:2251; CHECK-O0-NEXT: movw (%rdi), %cx2252; CHECK-O0-NEXT: # implicit-def: $eax2253; CHECK-O0-NEXT: movw %cx, %ax2254; CHECK-O0-NEXT: retq2255;2256; CHECK-O3-LABEL: load_i16_anyext_i32:2257; CHECK-O3: # %bb.0:2258; CHECK-O3-NEXT: movzwl (%rdi), %eax2259; CHECK-O3-NEXT: retq2260 %v = load atomic i16, ptr %ptr unordered, align 42261 %vec = insertelement <2 x i16> undef, i16 %v, i64 02262 %res = bitcast <2 x i16> %vec to i322263 ret i32 %res2264}2265 2266define i64 @load_i16_anyext_i64(ptr %ptr) {2267; CHECK-O0-LABEL: load_i16_anyext_i64:2268; CHECK-O0: # %bb.0:2269; CHECK-O0-NEXT: movw (%rdi), %cx2270; CHECK-O0-NEXT: # implicit-def: $eax2271; CHECK-O0-NEXT: movw %cx, %ax2272; CHECK-O0-NEXT: vmovd %eax, %xmm02273; CHECK-O0-NEXT: vmovq %xmm0, %rax2274; CHECK-O0-NEXT: retq2275;2276; CHECK-O3-LABEL: load_i16_anyext_i64:2277; CHECK-O3: # %bb.0:2278; CHECK-O3-NEXT: movzwl (%rdi), %eax2279; CHECK-O3-NEXT: vmovd %eax, %xmm02280; CHECK-O3-NEXT: vmovq %xmm0, %rax2281; CHECK-O3-NEXT: retq2282 %v = load atomic i16, ptr %ptr unordered, align 82283 %vec = insertelement <4 x i16> undef, i16 %v, i64 02284 %res = bitcast <4 x i16> %vec to i642285 ret i64 %res2286}2287 2288; TODO: Would be legal to combine for legal atomic wider types2289define i16 @load_combine(ptr %p) {2290; CHECK-O0-LABEL: load_combine:2291; CHECK-O0: # %bb.0:2292; CHECK-O0-NEXT: movb (%rdi), %al2293; CHECK-O0-NEXT: movb 1(%rdi), %cl2294; CHECK-O0-NEXT: movzbl %al, %eax2295; CHECK-O0-NEXT: # kill: def $ax killed $ax killed $eax2296; CHECK-O0-NEXT: movzbl %cl, %ecx2297; CHECK-O0-NEXT: # kill: def $cx killed $cx killed $ecx2298; CHECK-O0-NEXT: shlw $8, %cx2299; CHECK-O0-NEXT: orw %cx, %ax2300; CHECK-O0-NEXT: retq2301;2302; CHECK-O3-LABEL: load_combine:2303; CHECK-O3: # %bb.0:2304; CHECK-O3-NEXT: movzbl (%rdi), %ecx2305; CHECK-O3-NEXT: movzbl 1(%rdi), %eax2306; CHECK-O3-NEXT: shll $8, %eax2307; CHECK-O3-NEXT: orl %ecx, %eax2308; CHECK-O3-NEXT: # kill: def $ax killed $ax killed $eax2309; CHECK-O3-NEXT: retq2310 %v1 = load atomic i8, ptr %p unordered, align 22311 %p2 = getelementptr i8, ptr %p, i64 12312 %v2 = load atomic i8, ptr %p2 unordered, align 12313 %v1.ext = zext i8 %v1 to i162314 %v2.ext = zext i8 %v2 to i162315 %v2.sht = shl i16 %v2.ext, 82316 %res = or i16 %v1.ext, %v2.sht2317 ret i16 %res2318}2319 2320define i1 @fold_cmp_over_fence(ptr %p, i32 %v1) {2321; CHECK-O0-LABEL: fold_cmp_over_fence:2322; CHECK-O0: # %bb.0:2323; CHECK-O0-NEXT: movl (%rdi), %eax2324; CHECK-O0-NEXT: lock orl $0, -{{[0-9]+}}(%rsp)2325; CHECK-O0-NEXT: cmpl %eax, %esi2326; CHECK-O0-NEXT: jne .LBB116_22327; CHECK-O0-NEXT: # %bb.1: # %taken2328; CHECK-O0-NEXT: movb $1, %al2329; CHECK-O0-NEXT: retq2330; CHECK-O0-NEXT: .LBB116_2: # %untaken2331; CHECK-O0-NEXT: xorl %eax, %eax2332; CHECK-O0-NEXT: # kill: def $al killed $al killed $eax2333; CHECK-O0-NEXT: retq2334;2335; CHECK-O3-LABEL: fold_cmp_over_fence:2336; CHECK-O3: # %bb.0:2337; CHECK-O3-NEXT: movl (%rdi), %eax2338; CHECK-O3-NEXT: lock orl $0, -{{[0-9]+}}(%rsp)2339; CHECK-O3-NEXT: cmpl %eax, %esi2340; CHECK-O3-NEXT: jne .LBB116_22341; CHECK-O3-NEXT: # %bb.1: # %taken2342; CHECK-O3-NEXT: movb $1, %al2343; CHECK-O3-NEXT: retq2344; CHECK-O3-NEXT: .LBB116_2: # %untaken2345; CHECK-O3-NEXT: xorl %eax, %eax2346; CHECK-O3-NEXT: retq2347 %v2 = load atomic i32, ptr %p unordered, align 42348 fence seq_cst2349 %cmp = icmp eq i32 %v1, %v22350 br i1 %cmp, label %taken, label %untaken2351taken:2352 ret i1 true2353untaken:2354 ret i1 false2355}2356