433 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-apple-macosx10.9 -verify-machineinstrs -mattr=cx16 | FileCheck %s --check-prefixes=CHECK,CHECK-NOAVX3; RUN: llc < %s -mtriple=x86_64-apple-macosx10.9 -verify-machineinstrs -mattr=cx16,avx | FileCheck %s --check-prefixes=CHECK,CHECK-AVX4 5; Codegen of i128 without cx16 is tested in atomic-nocx16.ll6 7@var = global i128 08 9; Due to the scheduling right after isel for cmpxchg and given the10; machine scheduler and copy coalescer do not mess up with physical11; register live-ranges, we end up with a useless copy.12define i128 @val_compare_and_swap(ptr %p, i128 %oldval, i128 %newval) {13; CHECK-LABEL: val_compare_and_swap:14; CHECK: ## %bb.0:15; CHECK-NEXT: pushq %rbx16; CHECK-NEXT: .cfi_def_cfa_offset 1617; CHECK-NEXT: .cfi_offset %rbx, -1618; CHECK-NEXT: movq %rcx, %rbx19; CHECK-NEXT: movq %rsi, %rax20; CHECK-NEXT: movq %r8, %rcx21; CHECK-NEXT: lock cmpxchg16b (%rdi)22; CHECK-NEXT: popq %rbx23; CHECK-NEXT: retq24 %pair = cmpxchg ptr %p, i128 %oldval, i128 %newval acquire acquire25 %val = extractvalue { i128, i1 } %pair, 026 ret i128 %val27}28 29@cmpxchg16b_global = external dso_local global { i128, i128 }, align 1630 31;; Make sure we retain the offset of the global variable.32define i128 @load_global_with_offset() nounwind {33; CHECK-NOAVX-LABEL: load_global_with_offset:34; CHECK-NOAVX: ## %bb.0: ## %entry35; CHECK-NOAVX-NEXT: pushq %rbx36; CHECK-NOAVX-NEXT: xorl %eax, %eax37; CHECK-NOAVX-NEXT: xorl %edx, %edx38; CHECK-NOAVX-NEXT: xorl %ecx, %ecx39; CHECK-NOAVX-NEXT: xorl %ebx, %ebx40; CHECK-NOAVX-NEXT: lock cmpxchg16b _cmpxchg16b_global+16(%rip)41; CHECK-NOAVX-NEXT: popq %rbx42; CHECK-NOAVX-NEXT: retq43;44; CHECK-AVX-LABEL: load_global_with_offset:45; CHECK-AVX: ## %bb.0: ## %entry46; CHECK-AVX-NEXT: vmovdqa _cmpxchg16b_global+16(%rip), %xmm047; CHECK-AVX-NEXT: vmovq %xmm0, %rax48; CHECK-AVX-NEXT: vpextrq $1, %xmm0, %rdx49; CHECK-AVX-NEXT: retq50entry:51 %0 = load atomic i128, ptr getelementptr inbounds ({i128, i128}, ptr @cmpxchg16b_global, i64 0, i32 1) acquire, align 1652 ret i128 %053}54 55define void @fetch_and_nand(ptr %p, i128 %bits) {56; CHECK-LABEL: fetch_and_nand:57; CHECK: ## %bb.0:58; CHECK-NEXT: pushq %rbx59; CHECK-NEXT: .cfi_def_cfa_offset 1660; CHECK-NEXT: .cfi_offset %rbx, -1661; CHECK-NEXT: movq %rdx, %r862; CHECK-NEXT: movq (%rdi), %rax63; CHECK-NEXT: movq 8(%rdi), %rdx64; CHECK-NEXT: .p2align 465; CHECK-NEXT: LBB2_1: ## %atomicrmw.start66; CHECK-NEXT: ## =>This Inner Loop Header: Depth=167; CHECK-NEXT: movq %rdx, %rcx68; CHECK-NEXT: andq %r8, %rcx69; CHECK-NEXT: movq %rax, %rbx70; CHECK-NEXT: andq %rsi, %rbx71; CHECK-NEXT: notq %rbx72; CHECK-NEXT: notq %rcx73; CHECK-NEXT: lock cmpxchg16b (%rdi)74; CHECK-NEXT: jne LBB2_175; CHECK-NEXT: ## %bb.2: ## %atomicrmw.end76; CHECK-NEXT: movq %rax, _var(%rip)77; CHECK-NEXT: movq %rdx, _var+8(%rip)78; CHECK-NEXT: popq %rbx79; CHECK-NEXT: retq80 %val = atomicrmw nand ptr %p, i128 %bits release81 store i128 %val, ptr @var, align 1682 ret void83}84 85define void @fetch_and_or(ptr %p, i128 %bits) {86; CHECK-LABEL: fetch_and_or:87; CHECK: ## %bb.0:88; CHECK-NEXT: pushq %rbx89; CHECK-NEXT: .cfi_def_cfa_offset 1690; CHECK-NEXT: .cfi_offset %rbx, -1691; CHECK-NEXT: movq %rdx, %r892; CHECK-NEXT: movq (%rdi), %rax93; CHECK-NEXT: movq 8(%rdi), %rdx94; CHECK-NEXT: .p2align 495; CHECK-NEXT: LBB3_1: ## %atomicrmw.start96; CHECK-NEXT: ## =>This Inner Loop Header: Depth=197; CHECK-NEXT: movq %rax, %rbx98; CHECK-NEXT: orq %rsi, %rbx99; CHECK-NEXT: movq %rdx, %rcx100; CHECK-NEXT: orq %r8, %rcx101; CHECK-NEXT: lock cmpxchg16b (%rdi)102; CHECK-NEXT: jne LBB3_1103; CHECK-NEXT: ## %bb.2: ## %atomicrmw.end104; CHECK-NEXT: movq %rax, _var(%rip)105; CHECK-NEXT: movq %rdx, _var+8(%rip)106; CHECK-NEXT: popq %rbx107; CHECK-NEXT: retq108 %val = atomicrmw or ptr %p, i128 %bits seq_cst109 store i128 %val, ptr @var, align 16110 ret void111}112 113define void @fetch_and_add(ptr %p, i128 %bits) {114; CHECK-LABEL: fetch_and_add:115; CHECK: ## %bb.0:116; CHECK-NEXT: pushq %rbx117; CHECK-NEXT: .cfi_def_cfa_offset 16118; CHECK-NEXT: .cfi_offset %rbx, -16119; CHECK-NEXT: movq %rdx, %r8120; CHECK-NEXT: movq (%rdi), %rax121; CHECK-NEXT: movq 8(%rdi), %rdx122; CHECK-NEXT: .p2align 4123; CHECK-NEXT: LBB4_1: ## %atomicrmw.start124; CHECK-NEXT: ## =>This Inner Loop Header: Depth=1125; CHECK-NEXT: movq %rax, %rbx126; CHECK-NEXT: addq %rsi, %rbx127; CHECK-NEXT: movq %rdx, %rcx128; CHECK-NEXT: adcq %r8, %rcx129; CHECK-NEXT: lock cmpxchg16b (%rdi)130; CHECK-NEXT: jne LBB4_1131; CHECK-NEXT: ## %bb.2: ## %atomicrmw.end132; CHECK-NEXT: movq %rax, _var(%rip)133; CHECK-NEXT: movq %rdx, _var+8(%rip)134; CHECK-NEXT: popq %rbx135; CHECK-NEXT: retq136 %val = atomicrmw add ptr %p, i128 %bits seq_cst137 store i128 %val, ptr @var, align 16138 ret void139}140 141define void @fetch_and_sub(ptr %p, i128 %bits) {142; CHECK-LABEL: fetch_and_sub:143; CHECK: ## %bb.0:144; CHECK-NEXT: pushq %rbx145; CHECK-NEXT: .cfi_def_cfa_offset 16146; CHECK-NEXT: .cfi_offset %rbx, -16147; CHECK-NEXT: movq %rdx, %r8148; CHECK-NEXT: movq (%rdi), %rax149; CHECK-NEXT: movq 8(%rdi), %rdx150; CHECK-NEXT: .p2align 4151; CHECK-NEXT: LBB5_1: ## %atomicrmw.start152; CHECK-NEXT: ## =>This Inner Loop Header: Depth=1153; CHECK-NEXT: movq %rax, %rbx154; CHECK-NEXT: subq %rsi, %rbx155; CHECK-NEXT: movq %rdx, %rcx156; CHECK-NEXT: sbbq %r8, %rcx157; CHECK-NEXT: lock cmpxchg16b (%rdi)158; CHECK-NEXT: jne LBB5_1159; CHECK-NEXT: ## %bb.2: ## %atomicrmw.end160; CHECK-NEXT: movq %rax, _var(%rip)161; CHECK-NEXT: movq %rdx, _var+8(%rip)162; CHECK-NEXT: popq %rbx163; CHECK-NEXT: retq164 %val = atomicrmw sub ptr %p, i128 %bits seq_cst165 store i128 %val, ptr @var, align 16166 ret void167}168 169define void @fetch_and_min(ptr %p, i128 %bits) {170; CHECK-LABEL: fetch_and_min:171; CHECK: ## %bb.0:172; CHECK-NEXT: pushq %rbx173; CHECK-NEXT: .cfi_def_cfa_offset 16174; CHECK-NEXT: .cfi_offset %rbx, -16175; CHECK-NEXT: movq %rdx, %r8176; CHECK-NEXT: movq (%rdi), %rax177; CHECK-NEXT: movq 8(%rdi), %rdx178; CHECK-NEXT: .p2align 4179; CHECK-NEXT: LBB6_1: ## %atomicrmw.start180; CHECK-NEXT: ## =>This Inner Loop Header: Depth=1181; CHECK-NEXT: cmpq %rax, %rsi182; CHECK-NEXT: movq %r8, %rcx183; CHECK-NEXT: sbbq %rdx, %rcx184; CHECK-NEXT: movq %r8, %rcx185; CHECK-NEXT: cmovgeq %rdx, %rcx186; CHECK-NEXT: movq %rsi, %rbx187; CHECK-NEXT: cmovgeq %rax, %rbx188; CHECK-NEXT: lock cmpxchg16b (%rdi)189; CHECK-NEXT: jne LBB6_1190; CHECK-NEXT: ## %bb.2: ## %atomicrmw.end191; CHECK-NEXT: movq %rax, _var(%rip)192; CHECK-NEXT: movq %rdx, _var+8(%rip)193; CHECK-NEXT: popq %rbx194; CHECK-NEXT: retq195 %val = atomicrmw min ptr %p, i128 %bits seq_cst196 store i128 %val, ptr @var, align 16197 ret void198}199 200define void @fetch_and_max(ptr %p, i128 %bits) {201; CHECK-LABEL: fetch_and_max:202; CHECK: ## %bb.0:203; CHECK-NEXT: pushq %rbx204; CHECK-NEXT: .cfi_def_cfa_offset 16205; CHECK-NEXT: .cfi_offset %rbx, -16206; CHECK-NEXT: movq %rdx, %r8207; CHECK-NEXT: movq (%rdi), %rax208; CHECK-NEXT: movq 8(%rdi), %rdx209; CHECK-NEXT: .p2align 4210; CHECK-NEXT: LBB7_1: ## %atomicrmw.start211; CHECK-NEXT: ## =>This Inner Loop Header: Depth=1212; CHECK-NEXT: cmpq %rax, %rsi213; CHECK-NEXT: movq %r8, %rcx214; CHECK-NEXT: sbbq %rdx, %rcx215; CHECK-NEXT: movq %r8, %rcx216; CHECK-NEXT: cmovlq %rdx, %rcx217; CHECK-NEXT: movq %rsi, %rbx218; CHECK-NEXT: cmovlq %rax, %rbx219; CHECK-NEXT: lock cmpxchg16b (%rdi)220; CHECK-NEXT: jne LBB7_1221; CHECK-NEXT: ## %bb.2: ## %atomicrmw.end222; CHECK-NEXT: movq %rax, _var(%rip)223; CHECK-NEXT: movq %rdx, _var+8(%rip)224; CHECK-NEXT: popq %rbx225; CHECK-NEXT: retq226 %val = atomicrmw max ptr %p, i128 %bits seq_cst227 store i128 %val, ptr @var, align 16228 ret void229}230 231define void @fetch_and_umin(ptr %p, i128 %bits) {232; CHECK-LABEL: fetch_and_umin:233; CHECK: ## %bb.0:234; CHECK-NEXT: pushq %rbx235; CHECK-NEXT: .cfi_def_cfa_offset 16236; CHECK-NEXT: .cfi_offset %rbx, -16237; CHECK-NEXT: movq %rdx, %r8238; CHECK-NEXT: movq (%rdi), %rax239; CHECK-NEXT: movq 8(%rdi), %rdx240; CHECK-NEXT: .p2align 4241; CHECK-NEXT: LBB8_1: ## %atomicrmw.start242; CHECK-NEXT: ## =>This Inner Loop Header: Depth=1243; CHECK-NEXT: cmpq %rax, %rsi244; CHECK-NEXT: movq %r8, %rcx245; CHECK-NEXT: sbbq %rdx, %rcx246; CHECK-NEXT: movq %r8, %rcx247; CHECK-NEXT: cmovaeq %rdx, %rcx248; CHECK-NEXT: movq %rsi, %rbx249; CHECK-NEXT: cmovaeq %rax, %rbx250; CHECK-NEXT: lock cmpxchg16b (%rdi)251; CHECK-NEXT: jne LBB8_1252; CHECK-NEXT: ## %bb.2: ## %atomicrmw.end253; CHECK-NEXT: movq %rax, _var(%rip)254; CHECK-NEXT: movq %rdx, _var+8(%rip)255; CHECK-NEXT: popq %rbx256; CHECK-NEXT: retq257 %val = atomicrmw umin ptr %p, i128 %bits seq_cst258 store i128 %val, ptr @var, align 16259 ret void260}261 262define void @fetch_and_umax(ptr %p, i128 %bits) {263; CHECK-LABEL: fetch_and_umax:264; CHECK: ## %bb.0:265; CHECK-NEXT: pushq %rbx266; CHECK-NEXT: .cfi_def_cfa_offset 16267; CHECK-NEXT: .cfi_offset %rbx, -16268; CHECK-NEXT: movq %rdx, %r8269; CHECK-NEXT: movq (%rdi), %rax270; CHECK-NEXT: movq 8(%rdi), %rdx271; CHECK-NEXT: .p2align 4272; CHECK-NEXT: LBB9_1: ## %atomicrmw.start273; CHECK-NEXT: ## =>This Inner Loop Header: Depth=1274; CHECK-NEXT: cmpq %rax, %rsi275; CHECK-NEXT: movq %r8, %rcx276; CHECK-NEXT: sbbq %rdx, %rcx277; CHECK-NEXT: movq %r8, %rcx278; CHECK-NEXT: cmovbq %rdx, %rcx279; CHECK-NEXT: movq %rsi, %rbx280; CHECK-NEXT: cmovbq %rax, %rbx281; CHECK-NEXT: lock cmpxchg16b (%rdi)282; CHECK-NEXT: jne LBB9_1283; CHECK-NEXT: ## %bb.2: ## %atomicrmw.end284; CHECK-NEXT: movq %rax, _var(%rip)285; CHECK-NEXT: movq %rdx, _var+8(%rip)286; CHECK-NEXT: popq %rbx287; CHECK-NEXT: retq288 %val = atomicrmw umax ptr %p, i128 %bits seq_cst289 store i128 %val, ptr @var, align 16290 ret void291}292 293define i128 @atomic_load_seq_cst(ptr %p) {294; CHECK-NOAVX-LABEL: atomic_load_seq_cst:295; CHECK-NOAVX: ## %bb.0:296; CHECK-NOAVX-NEXT: pushq %rbx297; CHECK-NOAVX-NEXT: .cfi_def_cfa_offset 16298; CHECK-NOAVX-NEXT: .cfi_offset %rbx, -16299; CHECK-NOAVX-NEXT: xorl %eax, %eax300; CHECK-NOAVX-NEXT: xorl %edx, %edx301; CHECK-NOAVX-NEXT: xorl %ecx, %ecx302; CHECK-NOAVX-NEXT: xorl %ebx, %ebx303; CHECK-NOAVX-NEXT: lock cmpxchg16b (%rdi)304; CHECK-NOAVX-NEXT: popq %rbx305; CHECK-NOAVX-NEXT: retq306;307; CHECK-AVX-LABEL: atomic_load_seq_cst:308; CHECK-AVX: ## %bb.0:309; CHECK-AVX-NEXT: vmovdqa (%rdi), %xmm0310; CHECK-AVX-NEXT: vmovq %xmm0, %rax311; CHECK-AVX-NEXT: vpextrq $1, %xmm0, %rdx312; CHECK-AVX-NEXT: retq313 %r = load atomic i128, ptr %p seq_cst, align 16314 ret i128 %r315}316 317define i128 @atomic_load_relaxed(ptr %p) {318; CHECK-NOAVX-LABEL: atomic_load_relaxed:319; CHECK-NOAVX: ## %bb.0:320; CHECK-NOAVX-NEXT: pushq %rbx321; CHECK-NOAVX-NEXT: .cfi_def_cfa_offset 16322; CHECK-NOAVX-NEXT: .cfi_offset %rbx, -16323; CHECK-NOAVX-NEXT: xorl %eax, %eax324; CHECK-NOAVX-NEXT: xorl %edx, %edx325; CHECK-NOAVX-NEXT: xorl %ecx, %ecx326; CHECK-NOAVX-NEXT: xorl %ebx, %ebx327; CHECK-NOAVX-NEXT: lock cmpxchg16b (%rdi)328; CHECK-NOAVX-NEXT: popq %rbx329; CHECK-NOAVX-NEXT: retq330;331; CHECK-AVX-LABEL: atomic_load_relaxed:332; CHECK-AVX: ## %bb.0:333; CHECK-AVX-NEXT: vmovdqa (%rdi), %xmm0334; CHECK-AVX-NEXT: vmovq %xmm0, %rax335; CHECK-AVX-NEXT: vpextrq $1, %xmm0, %rdx336; CHECK-AVX-NEXT: retq337 %r = load atomic i128, ptr %p monotonic, align 16338 ret i128 %r339}340 341define void @atomic_store_seq_cst(ptr %p, i128 %in) {342; CHECK-NOAVX-LABEL: atomic_store_seq_cst:343; CHECK-NOAVX: ## %bb.0:344; CHECK-NOAVX-NEXT: pushq %rbx345; CHECK-NOAVX-NEXT: .cfi_def_cfa_offset 16346; CHECK-NOAVX-NEXT: .cfi_offset %rbx, -16347; CHECK-NOAVX-NEXT: movq %rdx, %rcx348; CHECK-NOAVX-NEXT: movq %rsi, %rbx349; CHECK-NOAVX-NEXT: movq (%rdi), %rax350; CHECK-NOAVX-NEXT: movq 8(%rdi), %rdx351; CHECK-NOAVX-NEXT: .p2align 4352; CHECK-NOAVX-NEXT: LBB12_1: ## %atomicrmw.start353; CHECK-NOAVX-NEXT: ## =>This Inner Loop Header: Depth=1354; CHECK-NOAVX-NEXT: lock cmpxchg16b (%rdi)355; CHECK-NOAVX-NEXT: jne LBB12_1356; CHECK-NOAVX-NEXT: ## %bb.2: ## %atomicrmw.end357; CHECK-NOAVX-NEXT: popq %rbx358; CHECK-NOAVX-NEXT: retq359;360; CHECK-AVX-LABEL: atomic_store_seq_cst:361; CHECK-AVX: ## %bb.0:362; CHECK-AVX-NEXT: vmovq %rdx, %xmm0363; CHECK-AVX-NEXT: vmovq %rsi, %xmm1364; CHECK-AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]365; CHECK-AVX-NEXT: vmovdqa %xmm0, (%rdi)366; CHECK-AVX-NEXT: lock orl $0, -{{[0-9]+}}(%rsp)367; CHECK-AVX-NEXT: retq368 store atomic i128 %in, ptr %p seq_cst, align 16369 ret void370}371 372define void @atomic_store_release(ptr %p, i128 %in) {373; CHECK-NOAVX-LABEL: atomic_store_release:374; CHECK-NOAVX: ## %bb.0:375; CHECK-NOAVX-NEXT: pushq %rbx376; CHECK-NOAVX-NEXT: .cfi_def_cfa_offset 16377; CHECK-NOAVX-NEXT: .cfi_offset %rbx, -16378; CHECK-NOAVX-NEXT: movq %rdx, %rcx379; CHECK-NOAVX-NEXT: movq %rsi, %rbx380; CHECK-NOAVX-NEXT: movq (%rdi), %rax381; CHECK-NOAVX-NEXT: movq 8(%rdi), %rdx382; CHECK-NOAVX-NEXT: .p2align 4383; CHECK-NOAVX-NEXT: LBB13_1: ## %atomicrmw.start384; CHECK-NOAVX-NEXT: ## =>This Inner Loop Header: Depth=1385; CHECK-NOAVX-NEXT: lock cmpxchg16b (%rdi)386; CHECK-NOAVX-NEXT: jne LBB13_1387; CHECK-NOAVX-NEXT: ## %bb.2: ## %atomicrmw.end388; CHECK-NOAVX-NEXT: popq %rbx389; CHECK-NOAVX-NEXT: retq390;391; CHECK-AVX-LABEL: atomic_store_release:392; CHECK-AVX: ## %bb.0:393; CHECK-AVX-NEXT: vmovq %rdx, %xmm0394; CHECK-AVX-NEXT: vmovq %rsi, %xmm1395; CHECK-AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]396; CHECK-AVX-NEXT: vmovdqa %xmm0, (%rdi)397; CHECK-AVX-NEXT: retq398 store atomic i128 %in, ptr %p release, align 16399 ret void400}401 402define void @atomic_store_relaxed(ptr %p, i128 %in) {403; CHECK-NOAVX-LABEL: atomic_store_relaxed:404; CHECK-NOAVX: ## %bb.0:405; CHECK-NOAVX-NEXT: pushq %rbx406; CHECK-NOAVX-NEXT: .cfi_def_cfa_offset 16407; CHECK-NOAVX-NEXT: .cfi_offset %rbx, -16408; CHECK-NOAVX-NEXT: movq %rdx, %rcx409; CHECK-NOAVX-NEXT: movq %rsi, %rbx410; CHECK-NOAVX-NEXT: movq (%rdi), %rax411; CHECK-NOAVX-NEXT: movq 8(%rdi), %rdx412; CHECK-NOAVX-NEXT: .p2align 4413; CHECK-NOAVX-NEXT: LBB14_1: ## %atomicrmw.start414; CHECK-NOAVX-NEXT: ## =>This Inner Loop Header: Depth=1415; CHECK-NOAVX-NEXT: lock cmpxchg16b (%rdi)416; CHECK-NOAVX-NEXT: jne LBB14_1417; CHECK-NOAVX-NEXT: ## %bb.2: ## %atomicrmw.end418; CHECK-NOAVX-NEXT: popq %rbx419; CHECK-NOAVX-NEXT: retq420;421; CHECK-AVX-LABEL: atomic_store_relaxed:422; CHECK-AVX: ## %bb.0:423; CHECK-AVX-NEXT: vmovq %rdx, %xmm0424; CHECK-AVX-NEXT: vmovq %rsi, %xmm1425; CHECK-AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]426; CHECK-AVX-NEXT: vmovdqa %xmm0, (%rdi)427; CHECK-AVX-NEXT: retq428 store atomic i128 %in, ptr %p unordered, align 16429 ret void430}431 432 433