1584 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs -aarch64-enable-sink-fold=true < %s | FileCheck %s --check-prefixes=CHECK,INLINE_ATOMICS3; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs -mattr=+outline-atomics -aarch64-enable-sink-fold=true < %s | FileCheck %s --check-prefixes=CHECK,OUTLINE_ATOMICS4 5@var8 = dso_local global i8 06@var16 = dso_local global i16 07@var32 = dso_local global i32 08@var64 = dso_local global i64 09 10define dso_local i8 @test_atomic_load_add_i8(i8 %offset) nounwind {11; INLINE_ATOMICS-LABEL: test_atomic_load_add_i8:12; INLINE_ATOMICS: // %bb.0:13; INLINE_ATOMICS-NEXT: adrp x9, var814; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var815; INLINE_ATOMICS-NEXT: .LBB0_1: // %atomicrmw.start16; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=117; INLINE_ATOMICS-NEXT: ldaxrb w8, [x9]18; INLINE_ATOMICS-NEXT: add w10, w8, w019; INLINE_ATOMICS-NEXT: stlxrb w11, w10, [x9]20; INLINE_ATOMICS-NEXT: cbnz w11, .LBB0_121; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end22; INLINE_ATOMICS-NEXT: mov w0, w823; INLINE_ATOMICS-NEXT: ret24;25; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i8:26; OUTLINE_ATOMICS: // %bb.0:27; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill28; OUTLINE_ATOMICS-NEXT: adrp x1, var829; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var830; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd1_acq_rel31; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload32; OUTLINE_ATOMICS-NEXT: ret33 %old = atomicrmw add ptr @var8, i8 %offset seq_cst34 ret i8 %old35}36 37define dso_local i16 @test_atomic_load_add_i16(i16 %offset) nounwind {38; INLINE_ATOMICS-LABEL: test_atomic_load_add_i16:39; INLINE_ATOMICS: // %bb.0:40; INLINE_ATOMICS-NEXT: adrp x9, var1641; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var1642; INLINE_ATOMICS-NEXT: .LBB1_1: // %atomicrmw.start43; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=144; INLINE_ATOMICS-NEXT: ldaxrh w8, [x9]45; INLINE_ATOMICS-NEXT: add w10, w8, w046; INLINE_ATOMICS-NEXT: stxrh w11, w10, [x9]47; INLINE_ATOMICS-NEXT: cbnz w11, .LBB1_148; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end49; INLINE_ATOMICS-NEXT: mov w0, w850; INLINE_ATOMICS-NEXT: ret51;52; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i16:53; OUTLINE_ATOMICS: // %bb.0:54; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill55; OUTLINE_ATOMICS-NEXT: adrp x1, var1656; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var1657; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd2_acq58; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload59; OUTLINE_ATOMICS-NEXT: ret60 %old = atomicrmw add ptr @var16, i16 %offset acquire61 ret i16 %old62}63 64define dso_local i32 @test_atomic_load_add_i32(i32 %offset) nounwind {65; INLINE_ATOMICS-LABEL: test_atomic_load_add_i32:66; INLINE_ATOMICS: // %bb.0:67; INLINE_ATOMICS-NEXT: adrp x9, var3268; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var3269; INLINE_ATOMICS-NEXT: .LBB2_1: // %atomicrmw.start70; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=171; INLINE_ATOMICS-NEXT: ldxr w8, [x9]72; INLINE_ATOMICS-NEXT: add w10, w8, w073; INLINE_ATOMICS-NEXT: stlxr w11, w10, [x9]74; INLINE_ATOMICS-NEXT: cbnz w11, .LBB2_175; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end76; INLINE_ATOMICS-NEXT: mov w0, w877; INLINE_ATOMICS-NEXT: ret78;79; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i32:80; OUTLINE_ATOMICS: // %bb.0:81; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill82; OUTLINE_ATOMICS-NEXT: adrp x1, var3283; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var3284; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd4_rel85; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload86; OUTLINE_ATOMICS-NEXT: ret87 %old = atomicrmw add ptr @var32, i32 %offset release88 ret i32 %old89}90 91define dso_local i64 @test_atomic_load_add_i64(i64 %offset) nounwind {92; INLINE_ATOMICS-LABEL: test_atomic_load_add_i64:93; INLINE_ATOMICS: // %bb.0:94; INLINE_ATOMICS-NEXT: adrp x9, var6495; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var6496; INLINE_ATOMICS-NEXT: .LBB3_1: // %atomicrmw.start97; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=198; INLINE_ATOMICS-NEXT: ldxr x8, [x9]99; INLINE_ATOMICS-NEXT: add x10, x8, x0100; INLINE_ATOMICS-NEXT: stxr w11, x10, [x9]101; INLINE_ATOMICS-NEXT: cbnz w11, .LBB3_1102; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end103; INLINE_ATOMICS-NEXT: mov x0, x8104; INLINE_ATOMICS-NEXT: ret105;106; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i64:107; OUTLINE_ATOMICS: // %bb.0:108; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill109; OUTLINE_ATOMICS-NEXT: adrp x1, var64110; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64111; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd8_relax112; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload113; OUTLINE_ATOMICS-NEXT: ret114 %old = atomicrmw add ptr @var64, i64 %offset monotonic115 ret i64 %old116}117 118define dso_local i8 @test_atomic_load_sub_i8(i8 %offset) nounwind {119; INLINE_ATOMICS-LABEL: test_atomic_load_sub_i8:120; INLINE_ATOMICS: // %bb.0:121; INLINE_ATOMICS-NEXT: adrp x9, var8122; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var8123; INLINE_ATOMICS-NEXT: .LBB4_1: // %atomicrmw.start124; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1125; INLINE_ATOMICS-NEXT: ldxrb w8, [x9]126; INLINE_ATOMICS-NEXT: sub w10, w8, w0127; INLINE_ATOMICS-NEXT: stxrb w11, w10, [x9]128; INLINE_ATOMICS-NEXT: cbnz w11, .LBB4_1129; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end130; INLINE_ATOMICS-NEXT: mov w0, w8131; INLINE_ATOMICS-NEXT: ret132;133; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i8:134; OUTLINE_ATOMICS: // %bb.0:135; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill136; OUTLINE_ATOMICS-NEXT: neg w0, w0137; OUTLINE_ATOMICS-NEXT: adrp x1, var8138; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8139; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd1_relax140; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload141; OUTLINE_ATOMICS-NEXT: ret142 %old = atomicrmw sub ptr @var8, i8 %offset monotonic143 ret i8 %old144}145 146define dso_local i16 @test_atomic_load_sub_i16(i16 %offset) nounwind {147; INLINE_ATOMICS-LABEL: test_atomic_load_sub_i16:148; INLINE_ATOMICS: // %bb.0:149; INLINE_ATOMICS-NEXT: adrp x9, var16150; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var16151; INLINE_ATOMICS-NEXT: .LBB5_1: // %atomicrmw.start152; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1153; INLINE_ATOMICS-NEXT: ldxrh w8, [x9]154; INLINE_ATOMICS-NEXT: sub w10, w8, w0155; INLINE_ATOMICS-NEXT: stlxrh w11, w10, [x9]156; INLINE_ATOMICS-NEXT: cbnz w11, .LBB5_1157; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end158; INLINE_ATOMICS-NEXT: mov w0, w8159; INLINE_ATOMICS-NEXT: ret160;161; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i16:162; OUTLINE_ATOMICS: // %bb.0:163; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill164; OUTLINE_ATOMICS-NEXT: neg w0, w0165; OUTLINE_ATOMICS-NEXT: adrp x1, var16166; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16167; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd2_rel168; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload169; OUTLINE_ATOMICS-NEXT: ret170 %old = atomicrmw sub ptr @var16, i16 %offset release171 ret i16 %old172}173 174define dso_local i32 @test_atomic_load_sub_i32(i32 %offset) nounwind {175; INLINE_ATOMICS-LABEL: test_atomic_load_sub_i32:176; INLINE_ATOMICS: // %bb.0:177; INLINE_ATOMICS-NEXT: adrp x9, var32178; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var32179; INLINE_ATOMICS-NEXT: .LBB6_1: // %atomicrmw.start180; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1181; INLINE_ATOMICS-NEXT: ldaxr w8, [x9]182; INLINE_ATOMICS-NEXT: sub w10, w8, w0183; INLINE_ATOMICS-NEXT: stxr w11, w10, [x9]184; INLINE_ATOMICS-NEXT: cbnz w11, .LBB6_1185; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end186; INLINE_ATOMICS-NEXT: mov w0, w8187; INLINE_ATOMICS-NEXT: ret188;189; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i32:190; OUTLINE_ATOMICS: // %bb.0:191; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill192; OUTLINE_ATOMICS-NEXT: neg w0, w0193; OUTLINE_ATOMICS-NEXT: adrp x1, var32194; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32195; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd4_acq196; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload197; OUTLINE_ATOMICS-NEXT: ret198 %old = atomicrmw sub ptr @var32, i32 %offset acquire199 ret i32 %old200}201 202define dso_local i64 @test_atomic_load_sub_i64(i64 %offset) nounwind {203; INLINE_ATOMICS-LABEL: test_atomic_load_sub_i64:204; INLINE_ATOMICS: // %bb.0:205; INLINE_ATOMICS-NEXT: adrp x9, var64206; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var64207; INLINE_ATOMICS-NEXT: .LBB7_1: // %atomicrmw.start208; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1209; INLINE_ATOMICS-NEXT: ldaxr x8, [x9]210; INLINE_ATOMICS-NEXT: sub x10, x8, x0211; INLINE_ATOMICS-NEXT: stlxr w11, x10, [x9]212; INLINE_ATOMICS-NEXT: cbnz w11, .LBB7_1213; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end214; INLINE_ATOMICS-NEXT: mov x0, x8215; INLINE_ATOMICS-NEXT: ret216;217; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i64:218; OUTLINE_ATOMICS: // %bb.0:219; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill220; OUTLINE_ATOMICS-NEXT: neg x0, x0221; OUTLINE_ATOMICS-NEXT: adrp x1, var64222; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64223; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel224; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload225; OUTLINE_ATOMICS-NEXT: ret226 %old = atomicrmw sub ptr @var64, i64 %offset seq_cst227 ret i64 %old228}229 230define dso_local i8 @test_atomic_load_and_i8(i8 %offset) nounwind {231; INLINE_ATOMICS-LABEL: test_atomic_load_and_i8:232; INLINE_ATOMICS: // %bb.0:233; INLINE_ATOMICS-NEXT: adrp x9, var8234; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var8235; INLINE_ATOMICS-NEXT: .LBB8_1: // %atomicrmw.start236; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1237; INLINE_ATOMICS-NEXT: ldxrb w8, [x9]238; INLINE_ATOMICS-NEXT: and w10, w8, w0239; INLINE_ATOMICS-NEXT: stlxrb w11, w10, [x9]240; INLINE_ATOMICS-NEXT: cbnz w11, .LBB8_1241; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end242; INLINE_ATOMICS-NEXT: mov w0, w8243; INLINE_ATOMICS-NEXT: ret244;245; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i8:246; OUTLINE_ATOMICS: // %bb.0:247; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill248; OUTLINE_ATOMICS-NEXT: adrp x1, var8249; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8250; OUTLINE_ATOMICS-NEXT: mvn w0, w0251; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldclr1_rel252; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload253; OUTLINE_ATOMICS-NEXT: ret254 %old = atomicrmw and ptr @var8, i8 %offset release255 ret i8 %old256}257 258define dso_local i16 @test_atomic_load_and_i16(i16 %offset) nounwind {259; INLINE_ATOMICS-LABEL: test_atomic_load_and_i16:260; INLINE_ATOMICS: // %bb.0:261; INLINE_ATOMICS-NEXT: adrp x9, var16262; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var16263; INLINE_ATOMICS-NEXT: .LBB9_1: // %atomicrmw.start264; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1265; INLINE_ATOMICS-NEXT: ldxrh w8, [x9]266; INLINE_ATOMICS-NEXT: and w10, w8, w0267; INLINE_ATOMICS-NEXT: stxrh w11, w10, [x9]268; INLINE_ATOMICS-NEXT: cbnz w11, .LBB9_1269; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end270; INLINE_ATOMICS-NEXT: mov w0, w8271; INLINE_ATOMICS-NEXT: ret272;273; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i16:274; OUTLINE_ATOMICS: // %bb.0:275; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill276; OUTLINE_ATOMICS-NEXT: adrp x1, var16277; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16278; OUTLINE_ATOMICS-NEXT: mvn w0, w0279; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldclr2_relax280; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload281; OUTLINE_ATOMICS-NEXT: ret282 %old = atomicrmw and ptr @var16, i16 %offset monotonic283 ret i16 %old284}285 286define dso_local i32 @test_atomic_load_and_i32(i32 %offset) nounwind {287; INLINE_ATOMICS-LABEL: test_atomic_load_and_i32:288; INLINE_ATOMICS: // %bb.0:289; INLINE_ATOMICS-NEXT: adrp x9, var32290; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var32291; INLINE_ATOMICS-NEXT: .LBB10_1: // %atomicrmw.start292; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1293; INLINE_ATOMICS-NEXT: ldaxr w8, [x9]294; INLINE_ATOMICS-NEXT: and w10, w8, w0295; INLINE_ATOMICS-NEXT: stlxr w11, w10, [x9]296; INLINE_ATOMICS-NEXT: cbnz w11, .LBB10_1297; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end298; INLINE_ATOMICS-NEXT: mov w0, w8299; INLINE_ATOMICS-NEXT: ret300;301; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i32:302; OUTLINE_ATOMICS: // %bb.0:303; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill304; OUTLINE_ATOMICS-NEXT: adrp x1, var32305; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32306; OUTLINE_ATOMICS-NEXT: mvn w0, w0307; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldclr4_acq_rel308; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload309; OUTLINE_ATOMICS-NEXT: ret310 %old = atomicrmw and ptr @var32, i32 %offset seq_cst311 ret i32 %old312}313 314define dso_local i64 @test_atomic_load_and_i64(i64 %offset) nounwind {315; INLINE_ATOMICS-LABEL: test_atomic_load_and_i64:316; INLINE_ATOMICS: // %bb.0:317; INLINE_ATOMICS-NEXT: adrp x9, var64318; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var64319; INLINE_ATOMICS-NEXT: .LBB11_1: // %atomicrmw.start320; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1321; INLINE_ATOMICS-NEXT: ldaxr x8, [x9]322; INLINE_ATOMICS-NEXT: and x10, x8, x0323; INLINE_ATOMICS-NEXT: stxr w11, x10, [x9]324; INLINE_ATOMICS-NEXT: cbnz w11, .LBB11_1325; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end326; INLINE_ATOMICS-NEXT: mov x0, x8327; INLINE_ATOMICS-NEXT: ret328;329; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i64:330; OUTLINE_ATOMICS: // %bb.0:331; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill332; OUTLINE_ATOMICS-NEXT: adrp x1, var64333; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64334; OUTLINE_ATOMICS-NEXT: mvn x0, x0335; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldclr8_acq336; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload337; OUTLINE_ATOMICS-NEXT: ret338 %old = atomicrmw and ptr @var64, i64 %offset acquire339 ret i64 %old340}341 342define dso_local i8 @test_atomic_load_or_i8(i8 %offset) nounwind {343; INLINE_ATOMICS-LABEL: test_atomic_load_or_i8:344; INLINE_ATOMICS: // %bb.0:345; INLINE_ATOMICS-NEXT: adrp x9, var8346; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var8347; INLINE_ATOMICS-NEXT: .LBB12_1: // %atomicrmw.start348; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1349; INLINE_ATOMICS-NEXT: ldaxrb w8, [x9]350; INLINE_ATOMICS-NEXT: orr w10, w8, w0351; INLINE_ATOMICS-NEXT: stlxrb w11, w10, [x9]352; INLINE_ATOMICS-NEXT: cbnz w11, .LBB12_1353; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end354; INLINE_ATOMICS-NEXT: mov w0, w8355; INLINE_ATOMICS-NEXT: ret356;357; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i8:358; OUTLINE_ATOMICS: // %bb.0:359; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill360; OUTLINE_ATOMICS-NEXT: adrp x1, var8361; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8362; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldset1_acq_rel363; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload364; OUTLINE_ATOMICS-NEXT: ret365 %old = atomicrmw or ptr @var8, i8 %offset seq_cst366 ret i8 %old367}368 369define dso_local i16 @test_atomic_load_or_i16(i16 %offset) nounwind {370; INLINE_ATOMICS-LABEL: test_atomic_load_or_i16:371; INLINE_ATOMICS: // %bb.0:372; INLINE_ATOMICS-NEXT: adrp x9, var16373; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var16374; INLINE_ATOMICS-NEXT: .LBB13_1: // %atomicrmw.start375; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1376; INLINE_ATOMICS-NEXT: ldxrh w8, [x9]377; INLINE_ATOMICS-NEXT: orr w10, w8, w0378; INLINE_ATOMICS-NEXT: stxrh w11, w10, [x9]379; INLINE_ATOMICS-NEXT: cbnz w11, .LBB13_1380; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end381; INLINE_ATOMICS-NEXT: mov w0, w8382; INLINE_ATOMICS-NEXT: ret383;384; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i16:385; OUTLINE_ATOMICS: // %bb.0:386; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill387; OUTLINE_ATOMICS-NEXT: adrp x1, var16388; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16389; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldset2_relax390; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload391; OUTLINE_ATOMICS-NEXT: ret392 %old = atomicrmw or ptr @var16, i16 %offset monotonic393 ret i16 %old394}395 396define dso_local i32 @test_atomic_load_or_i32(i32 %offset) nounwind {397; INLINE_ATOMICS-LABEL: test_atomic_load_or_i32:398; INLINE_ATOMICS: // %bb.0:399; INLINE_ATOMICS-NEXT: adrp x9, var32400; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var32401; INLINE_ATOMICS-NEXT: .LBB14_1: // %atomicrmw.start402; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1403; INLINE_ATOMICS-NEXT: ldaxr w8, [x9]404; INLINE_ATOMICS-NEXT: orr w10, w8, w0405; INLINE_ATOMICS-NEXT: stxr w11, w10, [x9]406; INLINE_ATOMICS-NEXT: cbnz w11, .LBB14_1407; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end408; INLINE_ATOMICS-NEXT: mov w0, w8409; INLINE_ATOMICS-NEXT: ret410;411; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i32:412; OUTLINE_ATOMICS: // %bb.0:413; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill414; OUTLINE_ATOMICS-NEXT: adrp x1, var32415; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32416; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldset4_acq417; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload418; OUTLINE_ATOMICS-NEXT: ret419 %old = atomicrmw or ptr @var32, i32 %offset acquire420 ret i32 %old421}422 423define dso_local i64 @test_atomic_load_or_i64(i64 %offset) nounwind {424; INLINE_ATOMICS-LABEL: test_atomic_load_or_i64:425; INLINE_ATOMICS: // %bb.0:426; INLINE_ATOMICS-NEXT: adrp x9, var64427; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var64428; INLINE_ATOMICS-NEXT: .LBB15_1: // %atomicrmw.start429; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1430; INLINE_ATOMICS-NEXT: ldxr x8, [x9]431; INLINE_ATOMICS-NEXT: orr x10, x8, x0432; INLINE_ATOMICS-NEXT: stlxr w11, x10, [x9]433; INLINE_ATOMICS-NEXT: cbnz w11, .LBB15_1434; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end435; INLINE_ATOMICS-NEXT: mov x0, x8436; INLINE_ATOMICS-NEXT: ret437;438; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i64:439; OUTLINE_ATOMICS: // %bb.0:440; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill441; OUTLINE_ATOMICS-NEXT: adrp x1, var64442; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64443; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldset8_rel444; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload445; OUTLINE_ATOMICS-NEXT: ret446 %old = atomicrmw or ptr @var64, i64 %offset release447 ret i64 %old448}449 450define dso_local i8 @test_atomic_load_xor_i8(i8 %offset) nounwind {451; INLINE_ATOMICS-LABEL: test_atomic_load_xor_i8:452; INLINE_ATOMICS: // %bb.0:453; INLINE_ATOMICS-NEXT: adrp x9, var8454; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var8455; INLINE_ATOMICS-NEXT: .LBB16_1: // %atomicrmw.start456; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1457; INLINE_ATOMICS-NEXT: ldaxrb w8, [x9]458; INLINE_ATOMICS-NEXT: eor w10, w8, w0459; INLINE_ATOMICS-NEXT: stxrb w11, w10, [x9]460; INLINE_ATOMICS-NEXT: cbnz w11, .LBB16_1461; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end462; INLINE_ATOMICS-NEXT: mov w0, w8463; INLINE_ATOMICS-NEXT: ret464;465; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i8:466; OUTLINE_ATOMICS: // %bb.0:467; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill468; OUTLINE_ATOMICS-NEXT: adrp x1, var8469; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8470; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldeor1_acq471; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload472; OUTLINE_ATOMICS-NEXT: ret473 %old = atomicrmw xor ptr @var8, i8 %offset acquire474 ret i8 %old475}476 477define dso_local i16 @test_atomic_load_xor_i16(i16 %offset) nounwind {478; INLINE_ATOMICS-LABEL: test_atomic_load_xor_i16:479; INLINE_ATOMICS: // %bb.0:480; INLINE_ATOMICS-NEXT: adrp x9, var16481; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var16482; INLINE_ATOMICS-NEXT: .LBB17_1: // %atomicrmw.start483; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1484; INLINE_ATOMICS-NEXT: ldxrh w8, [x9]485; INLINE_ATOMICS-NEXT: eor w10, w8, w0486; INLINE_ATOMICS-NEXT: stlxrh w11, w10, [x9]487; INLINE_ATOMICS-NEXT: cbnz w11, .LBB17_1488; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end489; INLINE_ATOMICS-NEXT: mov w0, w8490; INLINE_ATOMICS-NEXT: ret491;492; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i16:493; OUTLINE_ATOMICS: // %bb.0:494; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill495; OUTLINE_ATOMICS-NEXT: adrp x1, var16496; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16497; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldeor2_rel498; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload499; OUTLINE_ATOMICS-NEXT: ret500 %old = atomicrmw xor ptr @var16, i16 %offset release501 ret i16 %old502}503 504define dso_local i32 @test_atomic_load_xor_i32(i32 %offset) nounwind {505; INLINE_ATOMICS-LABEL: test_atomic_load_xor_i32:506; INLINE_ATOMICS: // %bb.0:507; INLINE_ATOMICS-NEXT: adrp x9, var32508; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var32509; INLINE_ATOMICS-NEXT: .LBB18_1: // %atomicrmw.start510; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1511; INLINE_ATOMICS-NEXT: ldaxr w8, [x9]512; INLINE_ATOMICS-NEXT: eor w10, w8, w0513; INLINE_ATOMICS-NEXT: stlxr w11, w10, [x9]514; INLINE_ATOMICS-NEXT: cbnz w11, .LBB18_1515; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end516; INLINE_ATOMICS-NEXT: mov w0, w8517; INLINE_ATOMICS-NEXT: ret518;519; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i32:520; OUTLINE_ATOMICS: // %bb.0:521; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill522; OUTLINE_ATOMICS-NEXT: adrp x1, var32523; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32524; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldeor4_acq_rel525; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload526; OUTLINE_ATOMICS-NEXT: ret527 %old = atomicrmw xor ptr @var32, i32 %offset seq_cst528 ret i32 %old529}530 531define dso_local i64 @test_atomic_load_xor_i64(i64 %offset) nounwind {532; INLINE_ATOMICS-LABEL: test_atomic_load_xor_i64:533; INLINE_ATOMICS: // %bb.0:534; INLINE_ATOMICS-NEXT: adrp x9, var64535; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var64536; INLINE_ATOMICS-NEXT: .LBB19_1: // %atomicrmw.start537; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1538; INLINE_ATOMICS-NEXT: ldxr x8, [x9]539; INLINE_ATOMICS-NEXT: eor x10, x8, x0540; INLINE_ATOMICS-NEXT: stxr w11, x10, [x9]541; INLINE_ATOMICS-NEXT: cbnz w11, .LBB19_1542; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end543; INLINE_ATOMICS-NEXT: mov x0, x8544; INLINE_ATOMICS-NEXT: ret545;546; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i64:547; OUTLINE_ATOMICS: // %bb.0:548; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill549; OUTLINE_ATOMICS-NEXT: adrp x1, var64550; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64551; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldeor8_relax552; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload553; OUTLINE_ATOMICS-NEXT: ret554 %old = atomicrmw xor ptr @var64, i64 %offset monotonic555 ret i64 %old556}557 558define dso_local i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind {559; INLINE_ATOMICS-LABEL: test_atomic_load_xchg_i8:560; INLINE_ATOMICS: // %bb.0:561; INLINE_ATOMICS-NEXT: adrp x9, var8562; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var8563; INLINE_ATOMICS-NEXT: .LBB20_1: // %atomicrmw.start564; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1565; INLINE_ATOMICS-NEXT: ldxrb w8, [x9]566; INLINE_ATOMICS-NEXT: stxrb w10, w0, [x9]567; INLINE_ATOMICS-NEXT: cbnz w10, .LBB20_1568; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end569; INLINE_ATOMICS-NEXT: mov w0, w8570; INLINE_ATOMICS-NEXT: ret571;572; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i8:573; OUTLINE_ATOMICS: // %bb.0:574; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill575; OUTLINE_ATOMICS-NEXT: adrp x1, var8576; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8577; OUTLINE_ATOMICS-NEXT: bl __aarch64_swp1_relax578; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload579; OUTLINE_ATOMICS-NEXT: ret580 %old = atomicrmw xchg ptr @var8, i8 %offset monotonic581 ret i8 %old582}583 584define dso_local i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind {585; INLINE_ATOMICS-LABEL: test_atomic_load_xchg_i16:586; INLINE_ATOMICS: // %bb.0:587; INLINE_ATOMICS-NEXT: adrp x9, var16588; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var16589; INLINE_ATOMICS-NEXT: .LBB21_1: // %atomicrmw.start590; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1591; INLINE_ATOMICS-NEXT: ldaxrh w8, [x9]592; INLINE_ATOMICS-NEXT: stlxrh w10, w0, [x9]593; INLINE_ATOMICS-NEXT: cbnz w10, .LBB21_1594; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end595; INLINE_ATOMICS-NEXT: mov w0, w8596; INLINE_ATOMICS-NEXT: ret597;598; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i16:599; OUTLINE_ATOMICS: // %bb.0:600; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill601; OUTLINE_ATOMICS-NEXT: adrp x1, var16602; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16603; OUTLINE_ATOMICS-NEXT: bl __aarch64_swp2_acq_rel604; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload605; OUTLINE_ATOMICS-NEXT: ret606 %old = atomicrmw xchg ptr @var16, i16 %offset seq_cst607 ret i16 %old608}609 610define dso_local i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind {611; INLINE_ATOMICS-LABEL: test_atomic_load_xchg_i32:612; INLINE_ATOMICS: // %bb.0:613; INLINE_ATOMICS-NEXT: mov w8, w0614; INLINE_ATOMICS-NEXT: adrp x9, var32615; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var32616; INLINE_ATOMICS-NEXT: .LBB22_1: // %atomicrmw.start617; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1618; INLINE_ATOMICS-NEXT: ldxr w0, [x9]619; INLINE_ATOMICS-NEXT: stlxr w10, w8, [x9]620; INLINE_ATOMICS-NEXT: cbnz w10, .LBB22_1621; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end622; INLINE_ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0623; INLINE_ATOMICS-NEXT: ret624;625; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i32:626; OUTLINE_ATOMICS: // %bb.0:627; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill628; OUTLINE_ATOMICS-NEXT: adrp x1, var32629; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32630; OUTLINE_ATOMICS-NEXT: bl __aarch64_swp4_rel631; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload632; OUTLINE_ATOMICS-NEXT: ret633 %old = atomicrmw xchg ptr @var32, i32 %offset release634 ret i32 %old635}636 637define dso_local i64 @test_atomic_load_xchg_i64(i64 %offset) nounwind {638; INLINE_ATOMICS-LABEL: test_atomic_load_xchg_i64:639; INLINE_ATOMICS: // %bb.0:640; INLINE_ATOMICS-NEXT: adrp x9, var64641; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var64642; INLINE_ATOMICS-NEXT: .LBB23_1: // %atomicrmw.start643; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1644; INLINE_ATOMICS-NEXT: ldaxr x8, [x9]645; INLINE_ATOMICS-NEXT: stxr w10, x0, [x9]646; INLINE_ATOMICS-NEXT: cbnz w10, .LBB23_1647; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end648; INLINE_ATOMICS-NEXT: mov x0, x8649; INLINE_ATOMICS-NEXT: ret650;651; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i64:652; OUTLINE_ATOMICS: // %bb.0:653; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill654; OUTLINE_ATOMICS-NEXT: adrp x1, var64655; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64656; OUTLINE_ATOMICS-NEXT: bl __aarch64_swp8_acq657; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload658; OUTLINE_ATOMICS-NEXT: ret659 %old = atomicrmw xchg ptr @var64, i64 %offset acquire660 ret i64 %old661}662 663 664define dso_local i8 @test_atomic_load_min_i8(i8 %offset) nounwind {665; CHECK-LABEL: test_atomic_load_min_i8:666; CHECK: // %bb.0:667; CHECK-NEXT: adrp x9, var8668; CHECK-NEXT: add x9, x9, :lo12:var8669; CHECK-NEXT: .LBB24_1: // %atomicrmw.start670; CHECK-NEXT: // =>This Inner Loop Header: Depth=1671; CHECK-NEXT: ldaxrb w10, [x9]672; CHECK-NEXT: sxtb w8, w10673; CHECK-NEXT: cmp w8, w0, sxtb674; CHECK-NEXT: csel w10, w10, w0, le675; CHECK-NEXT: stxrb w11, w10, [x9]676; CHECK-NEXT: cbnz w11, .LBB24_1677; CHECK-NEXT: // %bb.2: // %atomicrmw.end678; CHECK-NEXT: mov w0, w8679; CHECK-NEXT: ret680 %old = atomicrmw min ptr @var8, i8 %offset acquire681 ret i8 %old682}683 684define dso_local i16 @test_atomic_load_min_i16(i16 %offset) nounwind {685; CHECK-LABEL: test_atomic_load_min_i16:686; CHECK: // %bb.0:687; CHECK-NEXT: adrp x9, var16688; CHECK-NEXT: add x9, x9, :lo12:var16689; CHECK-NEXT: .LBB25_1: // %atomicrmw.start690; CHECK-NEXT: // =>This Inner Loop Header: Depth=1691; CHECK-NEXT: ldxrh w10, [x9]692; CHECK-NEXT: sxth w8, w10693; CHECK-NEXT: cmp w8, w0, sxth694; CHECK-NEXT: csel w10, w10, w0, le695; CHECK-NEXT: stlxrh w11, w10, [x9]696; CHECK-NEXT: cbnz w11, .LBB25_1697; CHECK-NEXT: // %bb.2: // %atomicrmw.end698; CHECK-NEXT: mov w0, w8699; CHECK-NEXT: ret700 %old = atomicrmw min ptr @var16, i16 %offset release701 ret i16 %old702}703 704define dso_local i32 @test_atomic_load_min_i32(i32 %offset) nounwind {705; CHECK-LABEL: test_atomic_load_min_i32:706; CHECK: // %bb.0:707; CHECK-NEXT: adrp x9, var32708; CHECK-NEXT: add x9, x9, :lo12:var32709; CHECK-NEXT: .LBB26_1: // %atomicrmw.start710; CHECK-NEXT: // =>This Inner Loop Header: Depth=1711; CHECK-NEXT: ldxr w8, [x9]712; CHECK-NEXT: cmp w8, w0713; CHECK-NEXT: csel w10, w8, w0, le714; CHECK-NEXT: stxr w11, w10, [x9]715; CHECK-NEXT: cbnz w11, .LBB26_1716; CHECK-NEXT: // %bb.2: // %atomicrmw.end717; CHECK-NEXT: mov w0, w8718; CHECK-NEXT: ret719 %old = atomicrmw min ptr @var32, i32 %offset monotonic720 ret i32 %old721}722 723define dso_local i64 @test_atomic_load_min_i64(i64 %offset) nounwind {724; CHECK-LABEL: test_atomic_load_min_i64:725; CHECK: // %bb.0:726; CHECK-NEXT: adrp x9, var64727; CHECK-NEXT: add x9, x9, :lo12:var64728; CHECK-NEXT: .LBB27_1: // %atomicrmw.start729; CHECK-NEXT: // =>This Inner Loop Header: Depth=1730; CHECK-NEXT: ldaxr x8, [x9]731; CHECK-NEXT: cmp x8, x0732; CHECK-NEXT: csel x10, x8, x0, le733; CHECK-NEXT: stlxr w11, x10, [x9]734; CHECK-NEXT: cbnz w11, .LBB27_1735; CHECK-NEXT: // %bb.2: // %atomicrmw.end736; CHECK-NEXT: mov x0, x8737; CHECK-NEXT: ret738 %old = atomicrmw min ptr @var64, i64 %offset seq_cst739 ret i64 %old740}741 742define dso_local i8 @test_atomic_load_max_i8(i8 %offset) nounwind {743; CHECK-LABEL: test_atomic_load_max_i8:744; CHECK: // %bb.0:745; CHECK-NEXT: adrp x9, var8746; CHECK-NEXT: add x9, x9, :lo12:var8747; CHECK-NEXT: .LBB28_1: // %atomicrmw.start748; CHECK-NEXT: // =>This Inner Loop Header: Depth=1749; CHECK-NEXT: ldaxrb w10, [x9]750; CHECK-NEXT: sxtb w8, w10751; CHECK-NEXT: cmp w8, w0, sxtb752; CHECK-NEXT: csel w10, w10, w0, gt753; CHECK-NEXT: stlxrb w11, w10, [x9]754; CHECK-NEXT: cbnz w11, .LBB28_1755; CHECK-NEXT: // %bb.2: // %atomicrmw.end756; CHECK-NEXT: mov w0, w8757; CHECK-NEXT: ret758 %old = atomicrmw max ptr @var8, i8 %offset seq_cst759 ret i8 %old760}761 762define dso_local i16 @test_atomic_load_max_i16(i16 %offset) nounwind {763; CHECK-LABEL: test_atomic_load_max_i16:764; CHECK: // %bb.0:765; CHECK-NEXT: adrp x9, var16766; CHECK-NEXT: add x9, x9, :lo12:var16767; CHECK-NEXT: .LBB29_1: // %atomicrmw.start768; CHECK-NEXT: // =>This Inner Loop Header: Depth=1769; CHECK-NEXT: ldaxrh w10, [x9]770; CHECK-NEXT: sxth w8, w10771; CHECK-NEXT: cmp w8, w0, sxth772; CHECK-NEXT: csel w10, w10, w0, gt773; CHECK-NEXT: stxrh w11, w10, [x9]774; CHECK-NEXT: cbnz w11, .LBB29_1775; CHECK-NEXT: // %bb.2: // %atomicrmw.end776; CHECK-NEXT: mov w0, w8777; CHECK-NEXT: ret778 %old = atomicrmw max ptr @var16, i16 %offset acquire779 ret i16 %old780}781 782define dso_local i32 @test_atomic_load_max_i32(i32 %offset) nounwind {783; CHECK-LABEL: test_atomic_load_max_i32:784; CHECK: // %bb.0:785; CHECK-NEXT: adrp x9, var32786; CHECK-NEXT: add x9, x9, :lo12:var32787; CHECK-NEXT: .LBB30_1: // %atomicrmw.start788; CHECK-NEXT: // =>This Inner Loop Header: Depth=1789; CHECK-NEXT: ldxr w8, [x9]790; CHECK-NEXT: cmp w8, w0791; CHECK-NEXT: csel w10, w8, w0, gt792; CHECK-NEXT: stlxr w11, w10, [x9]793; CHECK-NEXT: cbnz w11, .LBB30_1794; CHECK-NEXT: // %bb.2: // %atomicrmw.end795; CHECK-NEXT: mov w0, w8796; CHECK-NEXT: ret797 %old = atomicrmw max ptr @var32, i32 %offset release798 ret i32 %old799}800 801define dso_local i64 @test_atomic_load_max_i64(i64 %offset) nounwind {802; CHECK-LABEL: test_atomic_load_max_i64:803; CHECK: // %bb.0:804; CHECK-NEXT: adrp x9, var64805; CHECK-NEXT: add x9, x9, :lo12:var64806; CHECK-NEXT: .LBB31_1: // %atomicrmw.start807; CHECK-NEXT: // =>This Inner Loop Header: Depth=1808; CHECK-NEXT: ldxr x8, [x9]809; CHECK-NEXT: cmp x8, x0810; CHECK-NEXT: csel x10, x8, x0, gt811; CHECK-NEXT: stxr w11, x10, [x9]812; CHECK-NEXT: cbnz w11, .LBB31_1813; CHECK-NEXT: // %bb.2: // %atomicrmw.end814; CHECK-NEXT: mov x0, x8815; CHECK-NEXT: ret816 %old = atomicrmw max ptr @var64, i64 %offset monotonic817 ret i64 %old818}819 820define dso_local i8 @test_atomic_load_umin_i8(i8 %offset) nounwind {821; CHECK-LABEL: test_atomic_load_umin_i8:822; CHECK: // %bb.0:823; CHECK-NEXT: adrp x8, var8824; CHECK-NEXT: add x8, x8, :lo12:var8825; CHECK-NEXT: and w9, w0, #0xff826; CHECK-NEXT: .LBB32_1: // %atomicrmw.start827; CHECK-NEXT: // =>This Inner Loop Header: Depth=1828; CHECK-NEXT: ldxrb w0, [x8]829; CHECK-NEXT: cmp w0, w9830; CHECK-NEXT: csel w10, w0, w9, ls831; CHECK-NEXT: stxrb w11, w10, [x8]832; CHECK-NEXT: cbnz w11, .LBB32_1833; CHECK-NEXT: // %bb.2: // %atomicrmw.end834; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0835; CHECK-NEXT: ret836 %old = atomicrmw umin ptr @var8, i8 %offset monotonic837 ret i8 %old838}839 840define dso_local i16 @test_atomic_load_umin_i16(i16 %offset) nounwind {841; CHECK-LABEL: test_atomic_load_umin_i16:842; CHECK: // %bb.0:843; CHECK-NEXT: adrp x8, var16844; CHECK-NEXT: add x8, x8, :lo12:var16845; CHECK-NEXT: and w9, w0, #0xffff846; CHECK-NEXT: .LBB33_1: // %atomicrmw.start847; CHECK-NEXT: // =>This Inner Loop Header: Depth=1848; CHECK-NEXT: ldaxrh w0, [x8]849; CHECK-NEXT: cmp w0, w9850; CHECK-NEXT: csel w10, w0, w9, ls851; CHECK-NEXT: stxrh w11, w10, [x8]852; CHECK-NEXT: cbnz w11, .LBB33_1853; CHECK-NEXT: // %bb.2: // %atomicrmw.end854; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0855; CHECK-NEXT: ret856 %old = atomicrmw umin ptr @var16, i16 %offset acquire857 ret i16 %old858}859 860define dso_local i32 @test_atomic_load_umin_i32(i32 %offset) nounwind {861; CHECK-LABEL: test_atomic_load_umin_i32:862; CHECK: // %bb.0:863; CHECK-NEXT: adrp x9, var32864; CHECK-NEXT: add x9, x9, :lo12:var32865; CHECK-NEXT: .LBB34_1: // %atomicrmw.start866; CHECK-NEXT: // =>This Inner Loop Header: Depth=1867; CHECK-NEXT: ldaxr w8, [x9]868; CHECK-NEXT: cmp w8, w0869; CHECK-NEXT: csel w10, w8, w0, ls870; CHECK-NEXT: stlxr w11, w10, [x9]871; CHECK-NEXT: cbnz w11, .LBB34_1872; CHECK-NEXT: // %bb.2: // %atomicrmw.end873; CHECK-NEXT: mov w0, w8874; CHECK-NEXT: ret875 %old = atomicrmw umin ptr @var32, i32 %offset seq_cst876 ret i32 %old877}878 879define dso_local i64 @test_atomic_load_umin_i64(i64 %offset) nounwind {880; CHECK-LABEL: test_atomic_load_umin_i64:881; CHECK: // %bb.0:882; CHECK-NEXT: adrp x9, var64883; CHECK-NEXT: add x9, x9, :lo12:var64884; CHECK-NEXT: .LBB35_1: // %atomicrmw.start885; CHECK-NEXT: // =>This Inner Loop Header: Depth=1886; CHECK-NEXT: ldaxr x8, [x9]887; CHECK-NEXT: cmp x8, x0888; CHECK-NEXT: csel x10, x8, x0, ls889; CHECK-NEXT: stlxr w11, x10, [x9]890; CHECK-NEXT: cbnz w11, .LBB35_1891; CHECK-NEXT: // %bb.2: // %atomicrmw.end892; CHECK-NEXT: mov x0, x8893; CHECK-NEXT: ret894 %old = atomicrmw umin ptr @var64, i64 %offset acq_rel895 ret i64 %old896}897 898define dso_local i8 @test_atomic_load_umax_i8(i8 %offset) nounwind {899; CHECK-LABEL: test_atomic_load_umax_i8:900; CHECK: // %bb.0:901; CHECK-NEXT: adrp x8, var8902; CHECK-NEXT: add x8, x8, :lo12:var8903; CHECK-NEXT: and w9, w0, #0xff904; CHECK-NEXT: .LBB36_1: // %atomicrmw.start905; CHECK-NEXT: // =>This Inner Loop Header: Depth=1906; CHECK-NEXT: ldaxrb w0, [x8]907; CHECK-NEXT: cmp w0, w9908; CHECK-NEXT: csel w10, w0, w9, hi909; CHECK-NEXT: stlxrb w11, w10, [x8]910; CHECK-NEXT: cbnz w11, .LBB36_1911; CHECK-NEXT: // %bb.2: // %atomicrmw.end912; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0913; CHECK-NEXT: ret914 %old = atomicrmw umax ptr @var8, i8 %offset acq_rel915 ret i8 %old916}917 918define dso_local i16 @test_atomic_load_umax_i16(i16 %offset) nounwind {919; CHECK-LABEL: test_atomic_load_umax_i16:920; CHECK: // %bb.0:921; CHECK-NEXT: adrp x8, var16922; CHECK-NEXT: add x8, x8, :lo12:var16923; CHECK-NEXT: and w9, w0, #0xffff924; CHECK-NEXT: .LBB37_1: // %atomicrmw.start925; CHECK-NEXT: // =>This Inner Loop Header: Depth=1926; CHECK-NEXT: ldxrh w0, [x8]927; CHECK-NEXT: cmp w0, w9928; CHECK-NEXT: csel w10, w0, w9, hi929; CHECK-NEXT: stxrh w11, w10, [x8]930; CHECK-NEXT: cbnz w11, .LBB37_1931; CHECK-NEXT: // %bb.2: // %atomicrmw.end932; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0933; CHECK-NEXT: ret934 %old = atomicrmw umax ptr @var16, i16 %offset monotonic935 ret i16 %old936}937 938define dso_local i32 @test_atomic_load_umax_i32(i32 %offset) nounwind {939; CHECK-LABEL: test_atomic_load_umax_i32:940; CHECK: // %bb.0:941; CHECK-NEXT: adrp x9, var32942; CHECK-NEXT: add x9, x9, :lo12:var32943; CHECK-NEXT: .LBB38_1: // %atomicrmw.start944; CHECK-NEXT: // =>This Inner Loop Header: Depth=1945; CHECK-NEXT: ldaxr w8, [x9]946; CHECK-NEXT: cmp w8, w0947; CHECK-NEXT: csel w10, w8, w0, hi948; CHECK-NEXT: stlxr w11, w10, [x9]949; CHECK-NEXT: cbnz w11, .LBB38_1950; CHECK-NEXT: // %bb.2: // %atomicrmw.end951; CHECK-NEXT: mov w0, w8952; CHECK-NEXT: ret953 %old = atomicrmw umax ptr @var32, i32 %offset seq_cst954 ret i32 %old955}956 957define dso_local i64 @test_atomic_load_umax_i64(i64 %offset) nounwind {958; CHECK-LABEL: test_atomic_load_umax_i64:959; CHECK: // %bb.0:960; CHECK-NEXT: adrp x9, var64961; CHECK-NEXT: add x9, x9, :lo12:var64962; CHECK-NEXT: .LBB39_1: // %atomicrmw.start963; CHECK-NEXT: // =>This Inner Loop Header: Depth=1964; CHECK-NEXT: ldxr x8, [x9]965; CHECK-NEXT: cmp x8, x0966; CHECK-NEXT: csel x10, x8, x0, hi967; CHECK-NEXT: stlxr w11, x10, [x9]968; CHECK-NEXT: cbnz w11, .LBB39_1969; CHECK-NEXT: // %bb.2: // %atomicrmw.end970; CHECK-NEXT: mov x0, x8971; CHECK-NEXT: ret972 %old = atomicrmw umax ptr @var64, i64 %offset release973 ret i64 %old974}975 976define dso_local i8 @test_atomic_load_uinc_wrap_i8(i8 %offset) nounwind {977; CHECK-LABEL: test_atomic_load_uinc_wrap_i8:978; CHECK: // %bb.0:979; CHECK-NEXT: adrp x9, var8980; CHECK-NEXT: add x9, x9, :lo12:var8981; CHECK-NEXT: .LBB40_1: // %atomicrmw.start982; CHECK-NEXT: // =>This Inner Loop Header: Depth=1983; CHECK-NEXT: ldxrb w8, [x9]984; CHECK-NEXT: cmp w8, w0, uxtb985; CHECK-NEXT: csinc w10, wzr, w8, hs986; CHECK-NEXT: stlxrb w11, w10, [x9]987; CHECK-NEXT: cbnz w11, .LBB40_1988; CHECK-NEXT: // %bb.2: // %atomicrmw.end989; CHECK-NEXT: mov w0, w8990; CHECK-NEXT: ret991 %old = atomicrmw uinc_wrap ptr @var8, i8 %offset release992 ret i8 %old993}994 995define dso_local i16 @test_atomic_load_uinc_wrap_i16(i16 %offset) nounwind {996; CHECK-LABEL: test_atomic_load_uinc_wrap_i16:997; CHECK: // %bb.0:998; CHECK-NEXT: adrp x9, var16999; CHECK-NEXT: add x9, x9, :lo12:var161000; CHECK-NEXT: .LBB41_1: // %atomicrmw.start1001; CHECK-NEXT: // =>This Inner Loop Header: Depth=11002; CHECK-NEXT: ldxrh w8, [x9]1003; CHECK-NEXT: cmp w8, w0, uxth1004; CHECK-NEXT: csinc w10, wzr, w8, hs1005; CHECK-NEXT: stxrh w11, w10, [x9]1006; CHECK-NEXT: cbnz w11, .LBB41_11007; CHECK-NEXT: // %bb.2: // %atomicrmw.end1008; CHECK-NEXT: mov w0, w81009; CHECK-NEXT: ret1010 %old = atomicrmw uinc_wrap ptr @var16, i16 %offset monotonic1011 ret i16 %old1012}1013 1014define dso_local i32 @test_atomic_load_uinc_wrap_i32(i32 %offset) nounwind {1015; CHECK-LABEL: test_atomic_load_uinc_wrap_i32:1016; CHECK: // %bb.0:1017; CHECK-NEXT: adrp x9, var321018; CHECK-NEXT: add x9, x9, :lo12:var321019; CHECK-NEXT: .LBB42_1: // %atomicrmw.start1020; CHECK-NEXT: // =>This Inner Loop Header: Depth=11021; CHECK-NEXT: ldaxr w8, [x9]1022; CHECK-NEXT: cmp w8, w01023; CHECK-NEXT: csinc w10, wzr, w8, hs1024; CHECK-NEXT: stlxr w11, w10, [x9]1025; CHECK-NEXT: cbnz w11, .LBB42_11026; CHECK-NEXT: // %bb.2: // %atomicrmw.end1027; CHECK-NEXT: mov w0, w81028; CHECK-NEXT: ret1029 %old = atomicrmw uinc_wrap ptr @var32, i32 %offset seq_cst1030 ret i32 %old1031}1032 1033define dso_local i64 @test_atomic_load_uinc_wrap_i64(i64 %offset) nounwind {1034; CHECK-LABEL: test_atomic_load_uinc_wrap_i64:1035; CHECK: // %bb.0:1036; CHECK-NEXT: adrp x9, var641037; CHECK-NEXT: add x9, x9, :lo12:var641038; CHECK-NEXT: .LBB43_1: // %atomicrmw.start1039; CHECK-NEXT: // =>This Inner Loop Header: Depth=11040; CHECK-NEXT: ldaxr x8, [x9]1041; CHECK-NEXT: cmp x8, x01042; CHECK-NEXT: csinc x10, xzr, x8, hs1043; CHECK-NEXT: stxr w11, x10, [x9]1044; CHECK-NEXT: cbnz w11, .LBB43_11045; CHECK-NEXT: // %bb.2: // %atomicrmw.end1046; CHECK-NEXT: mov x0, x81047; CHECK-NEXT: ret1048 %old = atomicrmw uinc_wrap ptr @var64, i64 %offset acquire1049 ret i64 %old1050}1051 1052define dso_local i8 @test_atomic_load_udec_wrap_i8(i8 %offset) nounwind {1053; CHECK-LABEL: test_atomic_load_udec_wrap_i8:1054; CHECK: // %bb.0:1055; CHECK-NEXT: adrp x9, var81056; CHECK-NEXT: add x9, x9, :lo12:var81057; CHECK-NEXT: .LBB44_1: // %atomicrmw.start1058; CHECK-NEXT: // =>This Inner Loop Header: Depth=11059; CHECK-NEXT: ldxrb w8, [x9]1060; CHECK-NEXT: cmp w8, w0, uxtb1061; CHECK-NEXT: sub w10, w8, #11062; CHECK-NEXT: ccmp w8, #0, #4, ls1063; CHECK-NEXT: csel w10, w0, w10, eq1064; CHECK-NEXT: stlxrb w11, w10, [x9]1065; CHECK-NEXT: cbnz w11, .LBB44_11066; CHECK-NEXT: // %bb.2: // %atomicrmw.end1067; CHECK-NEXT: mov w0, w81068; CHECK-NEXT: ret1069 %old = atomicrmw udec_wrap ptr @var8, i8 %offset release1070 ret i8 %old1071}1072 1073define dso_local i16 @test_atomic_load_udec_wrap_i16(i16 %offset) nounwind {1074; CHECK-LABEL: test_atomic_load_udec_wrap_i16:1075; CHECK: // %bb.0:1076; CHECK-NEXT: adrp x9, var161077; CHECK-NEXT: add x9, x9, :lo12:var161078; CHECK-NEXT: .LBB45_1: // %atomicrmw.start1079; CHECK-NEXT: // =>This Inner Loop Header: Depth=11080; CHECK-NEXT: ldxrh w8, [x9]1081; CHECK-NEXT: cmp w8, w0, uxth1082; CHECK-NEXT: sub w10, w8, #11083; CHECK-NEXT: ccmp w8, #0, #4, ls1084; CHECK-NEXT: csel w10, w0, w10, eq1085; CHECK-NEXT: stxrh w11, w10, [x9]1086; CHECK-NEXT: cbnz w11, .LBB45_11087; CHECK-NEXT: // %bb.2: // %atomicrmw.end1088; CHECK-NEXT: mov w0, w81089; CHECK-NEXT: ret1090 %old = atomicrmw udec_wrap ptr @var16, i16 %offset monotonic1091 ret i16 %old1092}1093 1094define dso_local i32 @test_atomic_load_udec_wrap_i32(i32 %offset) nounwind {1095; CHECK-LABEL: test_atomic_load_udec_wrap_i32:1096; CHECK: // %bb.0:1097; CHECK-NEXT: adrp x9, var321098; CHECK-NEXT: add x9, x9, :lo12:var321099; CHECK-NEXT: .LBB46_1: // %atomicrmw.start1100; CHECK-NEXT: // =>This Inner Loop Header: Depth=11101; CHECK-NEXT: ldaxr w8, [x9]1102; CHECK-NEXT: cmp w8, w01103; CHECK-NEXT: sub w10, w8, #11104; CHECK-NEXT: ccmp w8, #0, #4, ls1105; CHECK-NEXT: csel w10, w0, w10, eq1106; CHECK-NEXT: stlxr w11, w10, [x9]1107; CHECK-NEXT: cbnz w11, .LBB46_11108; CHECK-NEXT: // %bb.2: // %atomicrmw.end1109; CHECK-NEXT: mov w0, w81110; CHECK-NEXT: ret1111 %old = atomicrmw udec_wrap ptr @var32, i32 %offset seq_cst1112 ret i32 %old1113}1114 1115define dso_local i64 @test_atomic_load_udec_wrap_i64(i64 %offset) nounwind {1116; CHECK-LABEL: test_atomic_load_udec_wrap_i64:1117; CHECK: // %bb.0:1118; CHECK-NEXT: adrp x9, var641119; CHECK-NEXT: add x9, x9, :lo12:var641120; CHECK-NEXT: .LBB47_1: // %atomicrmw.start1121; CHECK-NEXT: // =>This Inner Loop Header: Depth=11122; CHECK-NEXT: ldaxr x8, [x9]1123; CHECK-NEXT: cmp x8, x01124; CHECK-NEXT: sub x10, x8, #11125; CHECK-NEXT: ccmp x8, #0, #4, ls1126; CHECK-NEXT: csel x10, x0, x10, eq1127; CHECK-NEXT: stxr w11, x10, [x9]1128; CHECK-NEXT: cbnz w11, .LBB47_11129; CHECK-NEXT: // %bb.2: // %atomicrmw.end1130; CHECK-NEXT: mov x0, x81131; CHECK-NEXT: ret1132 %old = atomicrmw udec_wrap ptr @var64, i64 %offset acquire1133 ret i64 %old1134}1135 1136define dso_local i8 @test_atomic_load_usub_cond_i8(i8 %offset) nounwind {1137; CHECK-LABEL: test_atomic_load_usub_cond_i8:1138; CHECK: // %bb.0:1139; CHECK-NEXT: adrp x9, var81140; CHECK-NEXT: add x9, x9, :lo12:var81141; CHECK-NEXT: .LBB48_1: // %atomicrmw.start1142; CHECK-NEXT: // =>This Inner Loop Header: Depth=11143; CHECK-NEXT: ldxrb w8, [x9]1144; CHECK-NEXT: sub w10, w8, w01145; CHECK-NEXT: cmp w8, w0, uxtb1146; CHECK-NEXT: csel w10, w10, w8, hs1147; CHECK-NEXT: stlxrb w11, w10, [x9]1148; CHECK-NEXT: cbnz w11, .LBB48_11149; CHECK-NEXT: // %bb.2: // %atomicrmw.end1150; CHECK-NEXT: mov w0, w81151; CHECK-NEXT: ret1152 %old = atomicrmw usub_cond ptr @var8, i8 %offset release1153 ret i8 %old1154}1155 1156define dso_local i16 @test_atomic_load_usub_cond_i16(i16 %offset) nounwind {1157; CHECK-LABEL: test_atomic_load_usub_cond_i16:1158; CHECK: // %bb.0:1159; CHECK-NEXT: adrp x9, var161160; CHECK-NEXT: add x9, x9, :lo12:var161161; CHECK-NEXT: .LBB49_1: // %atomicrmw.start1162; CHECK-NEXT: // =>This Inner Loop Header: Depth=11163; CHECK-NEXT: ldxrh w8, [x9]1164; CHECK-NEXT: sub w10, w8, w01165; CHECK-NEXT: cmp w8, w0, uxth1166; CHECK-NEXT: csel w10, w10, w8, hs1167; CHECK-NEXT: stxrh w11, w10, [x9]1168; CHECK-NEXT: cbnz w11, .LBB49_11169; CHECK-NEXT: // %bb.2: // %atomicrmw.end1170; CHECK-NEXT: mov w0, w81171; CHECK-NEXT: ret1172 %old = atomicrmw usub_cond ptr @var16, i16 %offset monotonic1173 ret i16 %old1174}1175 1176define dso_local i32 @test_atomic_load_usub_cond_i32(i32 %offset) nounwind {1177; CHECK-LABEL: test_atomic_load_usub_cond_i32:1178; CHECK: // %bb.0:1179; CHECK-NEXT: adrp x9, var321180; CHECK-NEXT: add x9, x9, :lo12:var321181; CHECK-NEXT: .LBB50_1: // %atomicrmw.start1182; CHECK-NEXT: // =>This Inner Loop Header: Depth=11183; CHECK-NEXT: ldaxr w8, [x9]1184; CHECK-NEXT: subs w10, w8, w01185; CHECK-NEXT: csel w10, w10, w8, hs1186; CHECK-NEXT: stlxr w11, w10, [x9]1187; CHECK-NEXT: cbnz w11, .LBB50_11188; CHECK-NEXT: // %bb.2: // %atomicrmw.end1189; CHECK-NEXT: mov w0, w81190; CHECK-NEXT: ret1191 %old = atomicrmw usub_cond ptr @var32, i32 %offset seq_cst1192 ret i32 %old1193}1194 1195define dso_local i64 @test_atomic_load_usub_cond_i64(i64 %offset) nounwind {1196; CHECK-LABEL: test_atomic_load_usub_cond_i64:1197; CHECK: // %bb.0:1198; CHECK-NEXT: adrp x9, var641199; CHECK-NEXT: add x9, x9, :lo12:var641200; CHECK-NEXT: .LBB51_1: // %atomicrmw.start1201; CHECK-NEXT: // =>This Inner Loop Header: Depth=11202; CHECK-NEXT: ldaxr x8, [x9]1203; CHECK-NEXT: subs x10, x8, x01204; CHECK-NEXT: csel x10, x10, x8, hs1205; CHECK-NEXT: stxr w11, x10, [x9]1206; CHECK-NEXT: cbnz w11, .LBB51_11207; CHECK-NEXT: // %bb.2: // %atomicrmw.end1208; CHECK-NEXT: mov x0, x81209; CHECK-NEXT: ret1210 %old = atomicrmw usub_cond ptr @var64, i64 %offset acquire1211 ret i64 %old1212}1213 1214define dso_local i8 @test_atomic_load_usub_sat_i8(i8 %offset) nounwind {1215; CHECK-LABEL: test_atomic_load_usub_sat_i8:1216; CHECK: // %bb.0:1217; CHECK-NEXT: adrp x9, var81218; CHECK-NEXT: add x9, x9, :lo12:var81219; CHECK-NEXT: .LBB52_1: // %atomicrmw.start1220; CHECK-NEXT: // =>This Inner Loop Header: Depth=11221; CHECK-NEXT: ldxrb w8, [x9]1222; CHECK-NEXT: subs w10, w8, w0, uxtb1223; CHECK-NEXT: csel w10, wzr, w10, lo1224; CHECK-NEXT: stlxrb w11, w10, [x9]1225; CHECK-NEXT: cbnz w11, .LBB52_11226; CHECK-NEXT: // %bb.2: // %atomicrmw.end1227; CHECK-NEXT: mov w0, w81228; CHECK-NEXT: ret1229 %old = atomicrmw usub_sat ptr @var8, i8 %offset release1230 ret i8 %old1231}1232 1233define dso_local i16 @test_atomic_load_usub_sat_i16(i16 %offset) nounwind {1234; CHECK-LABEL: test_atomic_load_usub_sat_i16:1235; CHECK: // %bb.0:1236; CHECK-NEXT: adrp x9, var161237; CHECK-NEXT: add x9, x9, :lo12:var161238; CHECK-NEXT: .LBB53_1: // %atomicrmw.start1239; CHECK-NEXT: // =>This Inner Loop Header: Depth=11240; CHECK-NEXT: ldxrh w8, [x9]1241; CHECK-NEXT: subs w10, w8, w0, uxth1242; CHECK-NEXT: csel w10, wzr, w10, lo1243; CHECK-NEXT: stxrh w11, w10, [x9]1244; CHECK-NEXT: cbnz w11, .LBB53_11245; CHECK-NEXT: // %bb.2: // %atomicrmw.end1246; CHECK-NEXT: mov w0, w81247; CHECK-NEXT: ret1248 %old = atomicrmw usub_sat ptr @var16, i16 %offset monotonic1249 ret i16 %old1250}1251 1252define dso_local i32 @test_atomic_load_usub_sat_i32(i32 %offset) nounwind {1253; CHECK-LABEL: test_atomic_load_usub_sat_i32:1254; CHECK: // %bb.0:1255; CHECK-NEXT: adrp x9, var321256; CHECK-NEXT: add x9, x9, :lo12:var321257; CHECK-NEXT: .LBB54_1: // %atomicrmw.start1258; CHECK-NEXT: // =>This Inner Loop Header: Depth=11259; CHECK-NEXT: ldaxr w8, [x9]1260; CHECK-NEXT: subs w10, w8, w01261; CHECK-NEXT: csel w10, wzr, w10, lo1262; CHECK-NEXT: stlxr w11, w10, [x9]1263; CHECK-NEXT: cbnz w11, .LBB54_11264; CHECK-NEXT: // %bb.2: // %atomicrmw.end1265; CHECK-NEXT: mov w0, w81266; CHECK-NEXT: ret1267 %old = atomicrmw usub_sat ptr @var32, i32 %offset seq_cst1268 ret i32 %old1269}1270 1271define dso_local i64 @test_atomic_load_usub_sat_i64(i64 %offset) nounwind {1272; CHECK-LABEL: test_atomic_load_usub_sat_i64:1273; CHECK: // %bb.0:1274; CHECK-NEXT: adrp x9, var641275; CHECK-NEXT: add x9, x9, :lo12:var641276; CHECK-NEXT: .LBB55_1: // %atomicrmw.start1277; CHECK-NEXT: // =>This Inner Loop Header: Depth=11278; CHECK-NEXT: ldaxr x8, [x9]1279; CHECK-NEXT: subs x10, x8, x01280; CHECK-NEXT: csel x10, xzr, x10, lo1281; CHECK-NEXT: stxr w11, x10, [x9]1282; CHECK-NEXT: cbnz w11, .LBB55_11283; CHECK-NEXT: // %bb.2: // %atomicrmw.end1284; CHECK-NEXT: mov x0, x81285; CHECK-NEXT: ret1286 %old = atomicrmw usub_sat ptr @var64, i64 %offset acquire1287 ret i64 %old1288}1289 1290define dso_local i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {1291; INLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i8:1292; INLINE_ATOMICS: // %bb.0:1293; INLINE_ATOMICS-NEXT: and w8, w0, #0xff1294; INLINE_ATOMICS-NEXT: adrp x9, var81295; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var81296; INLINE_ATOMICS-NEXT: .LBB56_1: // %cmpxchg.start1297; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=11298; INLINE_ATOMICS-NEXT: ldaxrb w0, [x9]1299; INLINE_ATOMICS-NEXT: cmp w0, w81300; INLINE_ATOMICS-NEXT: b.ne .LBB56_41301; INLINE_ATOMICS-NEXT: // %bb.2: // %cmpxchg.trystore1302; INLINE_ATOMICS-NEXT: // in Loop: Header=BB56_1 Depth=11303; INLINE_ATOMICS-NEXT: stxrb w10, w1, [x9]1304; INLINE_ATOMICS-NEXT: cbnz w10, .LBB56_11305; INLINE_ATOMICS-NEXT: // %bb.3: // %cmpxchg.end1306; INLINE_ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x01307; INLINE_ATOMICS-NEXT: ret1308; INLINE_ATOMICS-NEXT: .LBB56_4: // %cmpxchg.nostore1309; INLINE_ATOMICS-NEXT: clrex1310; INLINE_ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x01311; INLINE_ATOMICS-NEXT: ret1312;1313; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i8:1314; OUTLINE_ATOMICS: // %bb.0:1315; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill1316; OUTLINE_ATOMICS-NEXT: adrp x2, var81317; OUTLINE_ATOMICS-NEXT: add x2, x2, :lo12:var81318; OUTLINE_ATOMICS-NEXT: bl __aarch64_cas1_acq1319; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload1320; OUTLINE_ATOMICS-NEXT: ret1321 %pair = cmpxchg ptr @var8, i8 %wanted, i8 %new acquire acquire1322 %old = extractvalue { i8, i1 } %pair, 01323 ret i8 %old1324}1325 1326define dso_local i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind {1327; INLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i16:1328; INLINE_ATOMICS: // %bb.0:1329; INLINE_ATOMICS-NEXT: and w8, w0, #0xffff1330; INLINE_ATOMICS-NEXT: adrp x9, var161331; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var161332; INLINE_ATOMICS-NEXT: .LBB57_1: // %cmpxchg.start1333; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=11334; INLINE_ATOMICS-NEXT: ldaxrh w0, [x9]1335; INLINE_ATOMICS-NEXT: cmp w0, w81336; INLINE_ATOMICS-NEXT: b.ne .LBB57_41337; INLINE_ATOMICS-NEXT: // %bb.2: // %cmpxchg.trystore1338; INLINE_ATOMICS-NEXT: // in Loop: Header=BB57_1 Depth=11339; INLINE_ATOMICS-NEXT: stlxrh w10, w1, [x9]1340; INLINE_ATOMICS-NEXT: cbnz w10, .LBB57_11341; INLINE_ATOMICS-NEXT: // %bb.3: // %cmpxchg.end1342; INLINE_ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x01343; INLINE_ATOMICS-NEXT: ret1344; INLINE_ATOMICS-NEXT: .LBB57_4: // %cmpxchg.nostore1345; INLINE_ATOMICS-NEXT: clrex1346; INLINE_ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x01347; INLINE_ATOMICS-NEXT: ret1348;1349; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i16:1350; OUTLINE_ATOMICS: // %bb.0:1351; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill1352; OUTLINE_ATOMICS-NEXT: adrp x2, var161353; OUTLINE_ATOMICS-NEXT: add x2, x2, :lo12:var161354; OUTLINE_ATOMICS-NEXT: bl __aarch64_cas2_acq_rel1355; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload1356; OUTLINE_ATOMICS-NEXT: ret1357 %pair = cmpxchg ptr @var16, i16 %wanted, i16 %new seq_cst seq_cst1358 %old = extractvalue { i16, i1 } %pair, 01359 ret i16 %old1360}1361 1362define dso_local i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind {1363; INLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i32:1364; INLINE_ATOMICS: // %bb.0:1365; INLINE_ATOMICS-NEXT: mov w8, w01366; INLINE_ATOMICS-NEXT: adrp x9, var321367; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var321368; INLINE_ATOMICS-NEXT: .LBB58_1: // %cmpxchg.start1369; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=11370; INLINE_ATOMICS-NEXT: ldxr w0, [x9]1371; INLINE_ATOMICS-NEXT: cmp w0, w81372; INLINE_ATOMICS-NEXT: b.ne .LBB58_41373; INLINE_ATOMICS-NEXT: // %bb.2: // %cmpxchg.trystore1374; INLINE_ATOMICS-NEXT: // in Loop: Header=BB58_1 Depth=11375; INLINE_ATOMICS-NEXT: stlxr w10, w1, [x9]1376; INLINE_ATOMICS-NEXT: cbnz w10, .LBB58_11377; INLINE_ATOMICS-NEXT: // %bb.3: // %cmpxchg.end1378; INLINE_ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x01379; INLINE_ATOMICS-NEXT: ret1380; INLINE_ATOMICS-NEXT: .LBB58_4: // %cmpxchg.nostore1381; INLINE_ATOMICS-NEXT: clrex1382; INLINE_ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x01383; INLINE_ATOMICS-NEXT: ret1384;1385; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i32:1386; OUTLINE_ATOMICS: // %bb.0:1387; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill1388; OUTLINE_ATOMICS-NEXT: adrp x2, var321389; OUTLINE_ATOMICS-NEXT: add x2, x2, :lo12:var321390; OUTLINE_ATOMICS-NEXT: bl __aarch64_cas4_rel1391; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload1392; OUTLINE_ATOMICS-NEXT: ret1393 %pair = cmpxchg ptr @var32, i32 %wanted, i32 %new release monotonic1394 %old = extractvalue { i32, i1 } %pair, 01395 ret i32 %old1396}1397 1398define dso_local void @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {1399; INLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i64:1400; INLINE_ATOMICS: // %bb.0:1401; INLINE_ATOMICS-NEXT: adrp x9, var641402; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var641403; INLINE_ATOMICS-NEXT: .LBB59_1: // %cmpxchg.start1404; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=11405; INLINE_ATOMICS-NEXT: ldxr x8, [x9]1406; INLINE_ATOMICS-NEXT: cmp x8, x01407; INLINE_ATOMICS-NEXT: b.ne .LBB59_41408; INLINE_ATOMICS-NEXT: // %bb.2: // %cmpxchg.trystore1409; INLINE_ATOMICS-NEXT: // in Loop: Header=BB59_1 Depth=11410; INLINE_ATOMICS-NEXT: stxr w10, x1, [x9]1411; INLINE_ATOMICS-NEXT: cbnz w10, .LBB59_11412; INLINE_ATOMICS-NEXT: .LBB59_3: // %cmpxchg.end1413; INLINE_ATOMICS-NEXT: adrp x9, var641414; INLINE_ATOMICS-NEXT: str x8, [x9, :lo12:var64]1415; INLINE_ATOMICS-NEXT: ret1416; INLINE_ATOMICS-NEXT: .LBB59_4: // %cmpxchg.nostore1417; INLINE_ATOMICS-NEXT: clrex1418; INLINE_ATOMICS-NEXT: b .LBB59_31419;1420; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i64:1421; OUTLINE_ATOMICS: // %bb.0:1422; OUTLINE_ATOMICS-NEXT: stp x30, x19, [sp, #-16]! // 16-byte Folded Spill1423; OUTLINE_ATOMICS-NEXT: adrp x19, var641424; OUTLINE_ATOMICS-NEXT: add x19, x19, :lo12:var641425; OUTLINE_ATOMICS-NEXT: mov x2, x191426; OUTLINE_ATOMICS-NEXT: bl __aarch64_cas8_relax1427; OUTLINE_ATOMICS-NEXT: str x0, [x19]1428; OUTLINE_ATOMICS-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload1429; OUTLINE_ATOMICS-NEXT: ret1430 %pair = cmpxchg ptr @var64, i64 %wanted, i64 %new monotonic monotonic1431 %old = extractvalue { i64, i1 } %pair, 01432 store i64 %old, ptr @var641433 ret void1434}1435 1436define dso_local i8 @test_atomic_load_monotonic_i8() nounwind {1437; CHECK-LABEL: test_atomic_load_monotonic_i8:1438; CHECK: // %bb.0:1439; CHECK-NEXT: adrp x8, var81440; CHECK-NEXT: ldrb w0, [x8, :lo12:var8]1441; CHECK-NEXT: ret1442 %val = load atomic i8, ptr @var8 monotonic, align 11443 ret i8 %val1444}1445 1446define dso_local i8 @test_atomic_load_monotonic_regoff_i8(i64 %base, i64 %off) nounwind {1447; CHECK-LABEL: test_atomic_load_monotonic_regoff_i8:1448; CHECK: // %bb.0:1449; CHECK-NEXT: ldrb w0, [x0, x1]1450; CHECK-NEXT: ret1451 %addr_int = add i64 %base, %off1452 %addr = inttoptr i64 %addr_int to ptr1453 %val = load atomic i8, ptr %addr monotonic, align 11454 ret i8 %val1455}1456 1457define dso_local i8 @test_atomic_load_acquire_i8() nounwind {1458; CHECK-LABEL: test_atomic_load_acquire_i8:1459; CHECK: // %bb.0:1460; CHECK-NEXT: adrp x8, var81461; CHECK-NEXT: add x8, x8, :lo12:var81462; CHECK-NEXT: ldarb w0, [x8]1463; CHECK-NEXT: ret1464 %val = load atomic i8, ptr @var8 acquire, align 11465 ret i8 %val1466}1467 1468define dso_local i8 @test_atomic_load_seq_cst_i8() nounwind {1469; CHECK-LABEL: test_atomic_load_seq_cst_i8:1470; CHECK: // %bb.0:1471; CHECK-NEXT: adrp x8, var81472; CHECK-NEXT: add x8, x8, :lo12:var81473; CHECK-NEXT: ldarb w0, [x8]1474; CHECK-NEXT: ret1475 %val = load atomic i8, ptr @var8 seq_cst, align 11476 ret i8 %val1477}1478 1479define dso_local i16 @test_atomic_load_monotonic_i16() nounwind {1480; CHECK-LABEL: test_atomic_load_monotonic_i16:1481; CHECK: // %bb.0:1482; CHECK-NEXT: adrp x8, var161483; CHECK-NEXT: ldrh w0, [x8, :lo12:var16]1484; CHECK-NEXT: ret1485 %val = load atomic i16, ptr @var16 monotonic, align 21486 ret i16 %val1487}1488 1489define dso_local i32 @test_atomic_load_monotonic_regoff_i32(i64 %base, i64 %off) nounwind {1490; CHECK-LABEL: test_atomic_load_monotonic_regoff_i32:1491; CHECK: // %bb.0:1492; CHECK-NEXT: ldr w0, [x0, x1]1493; CHECK-NEXT: ret1494 %addr_int = add i64 %base, %off1495 %addr = inttoptr i64 %addr_int to ptr1496 %val = load atomic i32, ptr %addr monotonic, align 41497 ret i32 %val1498}1499 1500define dso_local i64 @test_atomic_load_seq_cst_i64() nounwind {1501; CHECK-LABEL: test_atomic_load_seq_cst_i64:1502; CHECK: // %bb.0:1503; CHECK-NEXT: adrp x8, var641504; CHECK-NEXT: add x8, x8, :lo12:var641505; CHECK-NEXT: ldar x0, [x8]1506; CHECK-NEXT: ret1507 %val = load atomic i64, ptr @var64 seq_cst, align 81508 ret i64 %val1509}1510 1511define dso_local void @test_atomic_store_monotonic_i8(i8 %val) nounwind {1512; CHECK-LABEL: test_atomic_store_monotonic_i8:1513; CHECK: // %bb.0:1514; CHECK-NEXT: adrp x8, var81515; CHECK-NEXT: strb w0, [x8, :lo12:var8]1516; CHECK-NEXT: ret1517 store atomic i8 %val, ptr @var8 monotonic, align 11518 ret void1519}1520 1521define dso_local void @test_atomic_store_monotonic_regoff_i8(i64 %base, i64 %off, i8 %val) nounwind {1522; CHECK-LABEL: test_atomic_store_monotonic_regoff_i8:1523; CHECK: // %bb.0:1524; CHECK-NEXT: strb w2, [x0, x1]1525; CHECK-NEXT: ret1526 %addr_int = add i64 %base, %off1527 %addr = inttoptr i64 %addr_int to ptr1528 store atomic i8 %val, ptr %addr monotonic, align 11529 ret void1530}1531define dso_local void @test_atomic_store_release_i8(i8 %val) nounwind {1532; CHECK-LABEL: test_atomic_store_release_i8:1533; CHECK: // %bb.0:1534; CHECK-NEXT: adrp x8, var81535; CHECK-NEXT: add x8, x8, :lo12:var81536; CHECK-NEXT: stlrb w0, [x8]1537; CHECK-NEXT: ret1538 store atomic i8 %val, ptr @var8 release, align 11539 ret void1540}1541 1542define dso_local void @test_atomic_store_seq_cst_i8(i8 %val) nounwind {1543; CHECK-LABEL: test_atomic_store_seq_cst_i8:1544; CHECK: // %bb.0:1545; CHECK-NEXT: adrp x8, var81546; CHECK-NEXT: add x8, x8, :lo12:var81547; CHECK-NEXT: stlrb w0, [x8]1548; CHECK-NEXT: ret1549 store atomic i8 %val, ptr @var8 seq_cst, align 11550 ret void1551}1552 1553define dso_local void @test_atomic_store_monotonic_i16(i16 %val) nounwind {1554; CHECK-LABEL: test_atomic_store_monotonic_i16:1555; CHECK: // %bb.0:1556; CHECK-NEXT: adrp x8, var161557; CHECK-NEXT: strh w0, [x8, :lo12:var16]1558; CHECK-NEXT: ret1559 store atomic i16 %val, ptr @var16 monotonic, align 21560 ret void1561}1562 1563define dso_local void @test_atomic_store_monotonic_regoff_i32(i64 %base, i64 %off, i32 %val) nounwind {1564; CHECK-LABEL: test_atomic_store_monotonic_regoff_i32:1565; CHECK: // %bb.0:1566; CHECK-NEXT: str w2, [x0, x1]1567; CHECK-NEXT: ret1568 %addr_int = add i64 %base, %off1569 %addr = inttoptr i64 %addr_int to ptr1570 store atomic i32 %val, ptr %addr monotonic, align 41571 ret void1572}1573 1574define dso_local void @test_atomic_store_release_i64(i64 %val) nounwind {1575; CHECK-LABEL: test_atomic_store_release_i64:1576; CHECK: // %bb.0:1577; CHECK-NEXT: adrp x8, var641578; CHECK-NEXT: add x8, x8, :lo12:var641579; CHECK-NEXT: stlr x0, [x8]1580; CHECK-NEXT: ret1581 store atomic i64 %val, ptr @var64 release, align 81582 ret void1583}1584