674 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -verify-machineinstrs -mtriple=aarch64-- -O0 -fast-isel=0 -global-isel=false %s -o - | FileCheck %s -check-prefix=NOLSE3; RUN: llc -verify-machineinstrs -mtriple=aarch64-- -mattr=+lse -O0 -fast-isel=0 -global-isel=false %s -o - | FileCheck %s -check-prefix=LSE4 5; Ensure there's no stack spill in between ldxr/stxr pairs.6 7define i8 @test_rmw_add_8(ptr %dst) {8; NOLSE-LABEL: test_rmw_add_8:9; NOLSE: // %bb.0: // %entry10; NOLSE-NEXT: sub sp, sp, #3211; NOLSE-NEXT: .cfi_def_cfa_offset 3212; NOLSE-NEXT: str x0, [sp, #16] // 8-byte Spill13; NOLSE-NEXT: ldrb w8, [x0]14; NOLSE-NEXT: str w8, [sp, #28] // 4-byte Spill15; NOLSE-NEXT: b .LBB0_116; NOLSE-NEXT: .LBB0_1: // %atomicrmw.start17; NOLSE-NEXT: // =>This Loop Header: Depth=118; NOLSE-NEXT: // Child Loop BB0_2 Depth 219; NOLSE-NEXT: ldr w9, [sp, #28] // 4-byte Reload20; NOLSE-NEXT: ldr x11, [sp, #16] // 8-byte Reload21; NOLSE-NEXT: add w12, w9, #122; NOLSE-NEXT: .LBB0_2: // %atomicrmw.start23; NOLSE-NEXT: // Parent Loop BB0_1 Depth=124; NOLSE-NEXT: // => This Inner Loop Header: Depth=225; NOLSE-NEXT: ldaxrb w8, [x11]26; NOLSE-NEXT: cmp w8, w9, uxtb27; NOLSE-NEXT: b.ne .LBB0_428; NOLSE-NEXT: // %bb.3: // %atomicrmw.start29; NOLSE-NEXT: // in Loop: Header=BB0_2 Depth=230; NOLSE-NEXT: stlxrb w10, w12, [x11]31; NOLSE-NEXT: cbnz w10, .LBB0_232; NOLSE-NEXT: .LBB0_4: // %atomicrmw.start33; NOLSE-NEXT: // in Loop: Header=BB0_1 Depth=134; NOLSE-NEXT: subs w9, w8, w9, uxtb35; NOLSE-NEXT: str w8, [sp, #12] // 4-byte Spill36; NOLSE-NEXT: str w8, [sp, #28] // 4-byte Spill37; NOLSE-NEXT: b.ne .LBB0_138; NOLSE-NEXT: b .LBB0_539; NOLSE-NEXT: .LBB0_5: // %atomicrmw.end40; NOLSE-NEXT: ldr w0, [sp, #12] // 4-byte Reload41; NOLSE-NEXT: add sp, sp, #3242; NOLSE-NEXT: ret43;44; LSE-LABEL: test_rmw_add_8:45; LSE: // %bb.0: // %entry46; LSE-NEXT: mov w8, #1 // =0x147; LSE-NEXT: ldaddalb w8, w0, [x0]48; LSE-NEXT: ret49entry:50 %res = atomicrmw add ptr %dst, i8 1 seq_cst51 ret i8 %res52}53 54define i16 @test_rmw_add_16(ptr %dst) {55; NOLSE-LABEL: test_rmw_add_16:56; NOLSE: // %bb.0: // %entry57; NOLSE-NEXT: sub sp, sp, #3258; NOLSE-NEXT: .cfi_def_cfa_offset 3259; NOLSE-NEXT: str x0, [sp, #16] // 8-byte Spill60; NOLSE-NEXT: ldrh w8, [x0]61; NOLSE-NEXT: str w8, [sp, #28] // 4-byte Spill62; NOLSE-NEXT: b .LBB1_163; NOLSE-NEXT: .LBB1_1: // %atomicrmw.start64; NOLSE-NEXT: // =>This Loop Header: Depth=165; NOLSE-NEXT: // Child Loop BB1_2 Depth 266; NOLSE-NEXT: ldr w9, [sp, #28] // 4-byte Reload67; NOLSE-NEXT: ldr x11, [sp, #16] // 8-byte Reload68; NOLSE-NEXT: add w12, w9, #169; NOLSE-NEXT: .LBB1_2: // %atomicrmw.start70; NOLSE-NEXT: // Parent Loop BB1_1 Depth=171; NOLSE-NEXT: // => This Inner Loop Header: Depth=272; NOLSE-NEXT: ldaxrh w8, [x11]73; NOLSE-NEXT: cmp w8, w9, uxth74; NOLSE-NEXT: b.ne .LBB1_475; NOLSE-NEXT: // %bb.3: // %atomicrmw.start76; NOLSE-NEXT: // in Loop: Header=BB1_2 Depth=277; NOLSE-NEXT: stlxrh w10, w12, [x11]78; NOLSE-NEXT: cbnz w10, .LBB1_279; NOLSE-NEXT: .LBB1_4: // %atomicrmw.start80; NOLSE-NEXT: // in Loop: Header=BB1_1 Depth=181; NOLSE-NEXT: subs w9, w8, w9, uxth82; NOLSE-NEXT: str w8, [sp, #12] // 4-byte Spill83; NOLSE-NEXT: str w8, [sp, #28] // 4-byte Spill84; NOLSE-NEXT: b.ne .LBB1_185; NOLSE-NEXT: b .LBB1_586; NOLSE-NEXT: .LBB1_5: // %atomicrmw.end87; NOLSE-NEXT: ldr w0, [sp, #12] // 4-byte Reload88; NOLSE-NEXT: add sp, sp, #3289; NOLSE-NEXT: ret90;91; LSE-LABEL: test_rmw_add_16:92; LSE: // %bb.0: // %entry93; LSE-NEXT: mov w8, #1 // =0x194; LSE-NEXT: ldaddalh w8, w0, [x0]95; LSE-NEXT: ret96entry:97 %res = atomicrmw add ptr %dst, i16 1 seq_cst98 ret i16 %res99}100 101define i32 @test_rmw_add_32(ptr %dst) {102; NOLSE-LABEL: test_rmw_add_32:103; NOLSE: // %bb.0: // %entry104; NOLSE-NEXT: sub sp, sp, #32105; NOLSE-NEXT: .cfi_def_cfa_offset 32106; NOLSE-NEXT: str x0, [sp, #16] // 8-byte Spill107; NOLSE-NEXT: ldr w8, [x0]108; NOLSE-NEXT: str w8, [sp, #28] // 4-byte Spill109; NOLSE-NEXT: b .LBB2_1110; NOLSE-NEXT: .LBB2_1: // %atomicrmw.start111; NOLSE-NEXT: // =>This Loop Header: Depth=1112; NOLSE-NEXT: // Child Loop BB2_2 Depth 2113; NOLSE-NEXT: ldr w9, [sp, #28] // 4-byte Reload114; NOLSE-NEXT: ldr x11, [sp, #16] // 8-byte Reload115; NOLSE-NEXT: add w12, w9, #1116; NOLSE-NEXT: .LBB2_2: // %atomicrmw.start117; NOLSE-NEXT: // Parent Loop BB2_1 Depth=1118; NOLSE-NEXT: // => This Inner Loop Header: Depth=2119; NOLSE-NEXT: ldaxr w8, [x11]120; NOLSE-NEXT: cmp w8, w9121; NOLSE-NEXT: b.ne .LBB2_4122; NOLSE-NEXT: // %bb.3: // %atomicrmw.start123; NOLSE-NEXT: // in Loop: Header=BB2_2 Depth=2124; NOLSE-NEXT: stlxr w10, w12, [x11]125; NOLSE-NEXT: cbnz w10, .LBB2_2126; NOLSE-NEXT: .LBB2_4: // %atomicrmw.start127; NOLSE-NEXT: // in Loop: Header=BB2_1 Depth=1128; NOLSE-NEXT: subs w9, w8, w9129; NOLSE-NEXT: str w8, [sp, #12] // 4-byte Spill130; NOLSE-NEXT: str w8, [sp, #28] // 4-byte Spill131; NOLSE-NEXT: b.ne .LBB2_1132; NOLSE-NEXT: b .LBB2_5133; NOLSE-NEXT: .LBB2_5: // %atomicrmw.end134; NOLSE-NEXT: ldr w0, [sp, #12] // 4-byte Reload135; NOLSE-NEXT: add sp, sp, #32136; NOLSE-NEXT: ret137;138; LSE-LABEL: test_rmw_add_32:139; LSE: // %bb.0: // %entry140; LSE-NEXT: mov w8, #1 // =0x1141; LSE-NEXT: ldaddal w8, w0, [x0]142; LSE-NEXT: ret143entry:144 %res = atomicrmw add ptr %dst, i32 1 seq_cst145 ret i32 %res146}147 148define i64 @test_rmw_add_64(ptr %dst) {149; NOLSE-LABEL: test_rmw_add_64:150; NOLSE: // %bb.0: // %entry151; NOLSE-NEXT: sub sp, sp, #32152; NOLSE-NEXT: .cfi_def_cfa_offset 32153; NOLSE-NEXT: str x0, [sp, #16] // 8-byte Spill154; NOLSE-NEXT: ldr x8, [x0]155; NOLSE-NEXT: str x8, [sp, #24] // 8-byte Spill156; NOLSE-NEXT: b .LBB3_1157; NOLSE-NEXT: .LBB3_1: // %atomicrmw.start158; NOLSE-NEXT: // =>This Loop Header: Depth=1159; NOLSE-NEXT: // Child Loop BB3_2 Depth 2160; NOLSE-NEXT: ldr x9, [sp, #24] // 8-byte Reload161; NOLSE-NEXT: ldr x11, [sp, #16] // 8-byte Reload162; NOLSE-NEXT: add x12, x9, #1163; NOLSE-NEXT: .LBB3_2: // %atomicrmw.start164; NOLSE-NEXT: // Parent Loop BB3_1 Depth=1165; NOLSE-NEXT: // => This Inner Loop Header: Depth=2166; NOLSE-NEXT: ldaxr x8, [x11]167; NOLSE-NEXT: cmp x8, x9168; NOLSE-NEXT: b.ne .LBB3_4169; NOLSE-NEXT: // %bb.3: // %atomicrmw.start170; NOLSE-NEXT: // in Loop: Header=BB3_2 Depth=2171; NOLSE-NEXT: stlxr w10, x12, [x11]172; NOLSE-NEXT: cbnz w10, .LBB3_2173; NOLSE-NEXT: .LBB3_4: // %atomicrmw.start174; NOLSE-NEXT: // in Loop: Header=BB3_1 Depth=1175; NOLSE-NEXT: subs x9, x8, x9176; NOLSE-NEXT: str x8, [sp, #8] // 8-byte Spill177; NOLSE-NEXT: str x8, [sp, #24] // 8-byte Spill178; NOLSE-NEXT: b.ne .LBB3_1179; NOLSE-NEXT: b .LBB3_5180; NOLSE-NEXT: .LBB3_5: // %atomicrmw.end181; NOLSE-NEXT: ldr x0, [sp, #8] // 8-byte Reload182; NOLSE-NEXT: add sp, sp, #32183; NOLSE-NEXT: ret184;185; LSE-LABEL: test_rmw_add_64:186; LSE: // %bb.0: // %entry187; LSE-NEXT: mov w8, #1 // =0x1188; LSE-NEXT: // kill: def $x8 killed $w8189; LSE-NEXT: ldaddal x8, x0, [x0]190; LSE-NEXT: ret191entry:192 %res = atomicrmw add ptr %dst, i64 1 seq_cst193 ret i64 %res194}195 196define i128 @test_rmw_add_128(ptr %dst) {197; NOLSE-LABEL: test_rmw_add_128:198; NOLSE: // %bb.0: // %entry199; NOLSE-NEXT: sub sp, sp, #48200; NOLSE-NEXT: .cfi_def_cfa_offset 48201; NOLSE-NEXT: str x0, [sp, #24] // 8-byte Spill202; NOLSE-NEXT: ldr x8, [x0, #8]203; NOLSE-NEXT: ldr x9, [x0]204; NOLSE-NEXT: str x9, [sp, #32] // 8-byte Spill205; NOLSE-NEXT: str x8, [sp, #40] // 8-byte Spill206; NOLSE-NEXT: b .LBB4_1207; NOLSE-NEXT: .LBB4_1: // %atomicrmw.start208; NOLSE-NEXT: // =>This Loop Header: Depth=1209; NOLSE-NEXT: // Child Loop BB4_2 Depth 2210; NOLSE-NEXT: ldr x13, [sp, #40] // 8-byte Reload211; NOLSE-NEXT: ldr x11, [sp, #32] // 8-byte Reload212; NOLSE-NEXT: ldr x9, [sp, #24] // 8-byte Reload213; NOLSE-NEXT: adds x14, x11, #1214; NOLSE-NEXT: cinc x15, x13, hs215; NOLSE-NEXT: .LBB4_2: // %atomicrmw.start216; NOLSE-NEXT: // Parent Loop BB4_1 Depth=1217; NOLSE-NEXT: // => This Inner Loop Header: Depth=2218; NOLSE-NEXT: ldaxp x10, x12, [x9]219; NOLSE-NEXT: cmp x10, x11220; NOLSE-NEXT: cset w8, ne221; NOLSE-NEXT: cmp x12, x13222; NOLSE-NEXT: cinc w8, w8, ne223; NOLSE-NEXT: cbnz w8, .LBB4_4224; NOLSE-NEXT: // %bb.3: // %atomicrmw.start225; NOLSE-NEXT: // in Loop: Header=BB4_2 Depth=2226; NOLSE-NEXT: stlxp w8, x14, x15, [x9]227; NOLSE-NEXT: cbnz w8, .LBB4_2228; NOLSE-NEXT: b .LBB4_5229; NOLSE-NEXT: .LBB4_4: // %atomicrmw.start230; NOLSE-NEXT: // in Loop: Header=BB4_2 Depth=2231; NOLSE-NEXT: stlxp w8, x10, x12, [x9]232; NOLSE-NEXT: cbnz w8, .LBB4_2233; NOLSE-NEXT: .LBB4_5: // %atomicrmw.start234; NOLSE-NEXT: // in Loop: Header=BB4_1 Depth=1235; NOLSE-NEXT: mov x8, x12236; NOLSE-NEXT: str x8, [sp, #8] // 8-byte Spill237; NOLSE-NEXT: mov x9, x10238; NOLSE-NEXT: str x9, [sp, #16] // 8-byte Spill239; NOLSE-NEXT: subs x12, x12, x13240; NOLSE-NEXT: ccmp x10, x11, #0, eq241; NOLSE-NEXT: str x9, [sp, #32] // 8-byte Spill242; NOLSE-NEXT: str x8, [sp, #40] // 8-byte Spill243; NOLSE-NEXT: b.ne .LBB4_1244; NOLSE-NEXT: b .LBB4_6245; NOLSE-NEXT: .LBB4_6: // %atomicrmw.end246; NOLSE-NEXT: ldr x1, [sp, #8] // 8-byte Reload247; NOLSE-NEXT: ldr x0, [sp, #16] // 8-byte Reload248; NOLSE-NEXT: add sp, sp, #48249; NOLSE-NEXT: ret250;251; LSE-LABEL: test_rmw_add_128:252; LSE: // %bb.0: // %entry253; LSE-NEXT: sub sp, sp, #48254; LSE-NEXT: .cfi_def_cfa_offset 48255; LSE-NEXT: str x0, [sp, #24] // 8-byte Spill256; LSE-NEXT: ldr x8, [x0, #8]257; LSE-NEXT: ldr x9, [x0]258; LSE-NEXT: str x9, [sp, #32] // 8-byte Spill259; LSE-NEXT: str x8, [sp, #40] // 8-byte Spill260; LSE-NEXT: b .LBB4_1261; LSE-NEXT: .LBB4_1: // %atomicrmw.start262; LSE-NEXT: // =>This Inner Loop Header: Depth=1263; LSE-NEXT: ldr x11, [sp, #40] // 8-byte Reload264; LSE-NEXT: ldr x10, [sp, #32] // 8-byte Reload265; LSE-NEXT: ldr x8, [sp, #24] // 8-byte Reload266; LSE-NEXT: mov x0, x10267; LSE-NEXT: mov x1, x11268; LSE-NEXT: adds x2, x10, #1269; LSE-NEXT: cinc x9, x11, hs270; LSE-NEXT: // kill: def $x2 killed $x2 def $x2_x3271; LSE-NEXT: mov x3, x9272; LSE-NEXT: caspal x0, x1, x2, x3, [x8]273; LSE-NEXT: mov x9, x0274; LSE-NEXT: str x9, [sp, #8] // 8-byte Spill275; LSE-NEXT: mov x8, x1276; LSE-NEXT: str x8, [sp, #16] // 8-byte Spill277; LSE-NEXT: subs x11, x8, x11278; LSE-NEXT: ccmp x9, x10, #0, eq279; LSE-NEXT: str x9, [sp, #32] // 8-byte Spill280; LSE-NEXT: str x8, [sp, #40] // 8-byte Spill281; LSE-NEXT: b.ne .LBB4_1282; LSE-NEXT: b .LBB4_2283; LSE-NEXT: .LBB4_2: // %atomicrmw.end284; LSE-NEXT: ldr x1, [sp, #16] // 8-byte Reload285; LSE-NEXT: ldr x0, [sp, #8] // 8-byte Reload286; LSE-NEXT: add sp, sp, #48287; LSE-NEXT: ret288entry:289 %res = atomicrmw add ptr %dst, i128 1 seq_cst290 ret i128 %res291}292define i8 @test_rmw_nand_8(ptr %dst) {293; NOLSE-LABEL: test_rmw_nand_8:294; NOLSE: // %bb.0: // %entry295; NOLSE-NEXT: sub sp, sp, #32296; NOLSE-NEXT: .cfi_def_cfa_offset 32297; NOLSE-NEXT: str x0, [sp, #16] // 8-byte Spill298; NOLSE-NEXT: ldrb w8, [x0]299; NOLSE-NEXT: str w8, [sp, #28] // 4-byte Spill300; NOLSE-NEXT: b .LBB5_1301; NOLSE-NEXT: .LBB5_1: // %atomicrmw.start302; NOLSE-NEXT: // =>This Loop Header: Depth=1303; NOLSE-NEXT: // Child Loop BB5_2 Depth 2304; NOLSE-NEXT: ldr w9, [sp, #28] // 4-byte Reload305; NOLSE-NEXT: ldr x11, [sp, #16] // 8-byte Reload306; NOLSE-NEXT: mvn w8, w9307; NOLSE-NEXT: orr w12, w8, #0xfffffffe308; NOLSE-NEXT: .LBB5_2: // %atomicrmw.start309; NOLSE-NEXT: // Parent Loop BB5_1 Depth=1310; NOLSE-NEXT: // => This Inner Loop Header: Depth=2311; NOLSE-NEXT: ldaxrb w8, [x11]312; NOLSE-NEXT: cmp w8, w9, uxtb313; NOLSE-NEXT: b.ne .LBB5_4314; NOLSE-NEXT: // %bb.3: // %atomicrmw.start315; NOLSE-NEXT: // in Loop: Header=BB5_2 Depth=2316; NOLSE-NEXT: stlxrb w10, w12, [x11]317; NOLSE-NEXT: cbnz w10, .LBB5_2318; NOLSE-NEXT: .LBB5_4: // %atomicrmw.start319; NOLSE-NEXT: // in Loop: Header=BB5_1 Depth=1320; NOLSE-NEXT: subs w9, w8, w9, uxtb321; NOLSE-NEXT: str w8, [sp, #12] // 4-byte Spill322; NOLSE-NEXT: str w8, [sp, #28] // 4-byte Spill323; NOLSE-NEXT: b.ne .LBB5_1324; NOLSE-NEXT: b .LBB5_5325; NOLSE-NEXT: .LBB5_5: // %atomicrmw.end326; NOLSE-NEXT: ldr w0, [sp, #12] // 4-byte Reload327; NOLSE-NEXT: add sp, sp, #32328; NOLSE-NEXT: ret329;330; LSE-LABEL: test_rmw_nand_8:331; LSE: // %bb.0: // %entry332; LSE-NEXT: sub sp, sp, #32333; LSE-NEXT: .cfi_def_cfa_offset 32334; LSE-NEXT: str x0, [sp, #16] // 8-byte Spill335; LSE-NEXT: ldrb w8, [x0]336; LSE-NEXT: str w8, [sp, #28] // 4-byte Spill337; LSE-NEXT: b .LBB5_1338; LSE-NEXT: .LBB5_1: // %atomicrmw.start339; LSE-NEXT: // =>This Inner Loop Header: Depth=1340; LSE-NEXT: ldr w9, [sp, #28] // 4-byte Reload341; LSE-NEXT: ldr x11, [sp, #16] // 8-byte Reload342; LSE-NEXT: mvn w8, w9343; LSE-NEXT: orr w10, w8, #0xfffffffe344; LSE-NEXT: mov w8, w9345; LSE-NEXT: casalb w8, w10, [x11]346; LSE-NEXT: subs w9, w8, w9, uxtb347; LSE-NEXT: str w8, [sp, #12] // 4-byte Spill348; LSE-NEXT: str w8, [sp, #28] // 4-byte Spill349; LSE-NEXT: b.ne .LBB5_1350; LSE-NEXT: b .LBB5_2351; LSE-NEXT: .LBB5_2: // %atomicrmw.end352; LSE-NEXT: ldr w0, [sp, #12] // 4-byte Reload353; LSE-NEXT: add sp, sp, #32354; LSE-NEXT: ret355entry:356 %res = atomicrmw nand ptr %dst, i8 1 seq_cst357 ret i8 %res358}359 360define i16 @test_rmw_nand_16(ptr %dst) {361; NOLSE-LABEL: test_rmw_nand_16:362; NOLSE: // %bb.0: // %entry363; NOLSE-NEXT: sub sp, sp, #32364; NOLSE-NEXT: .cfi_def_cfa_offset 32365; NOLSE-NEXT: str x0, [sp, #16] // 8-byte Spill366; NOLSE-NEXT: ldrh w8, [x0]367; NOLSE-NEXT: str w8, [sp, #28] // 4-byte Spill368; NOLSE-NEXT: b .LBB6_1369; NOLSE-NEXT: .LBB6_1: // %atomicrmw.start370; NOLSE-NEXT: // =>This Loop Header: Depth=1371; NOLSE-NEXT: // Child Loop BB6_2 Depth 2372; NOLSE-NEXT: ldr w9, [sp, #28] // 4-byte Reload373; NOLSE-NEXT: ldr x11, [sp, #16] // 8-byte Reload374; NOLSE-NEXT: mvn w8, w9375; NOLSE-NEXT: orr w12, w8, #0xfffffffe376; NOLSE-NEXT: .LBB6_2: // %atomicrmw.start377; NOLSE-NEXT: // Parent Loop BB6_1 Depth=1378; NOLSE-NEXT: // => This Inner Loop Header: Depth=2379; NOLSE-NEXT: ldaxrh w8, [x11]380; NOLSE-NEXT: cmp w8, w9, uxth381; NOLSE-NEXT: b.ne .LBB6_4382; NOLSE-NEXT: // %bb.3: // %atomicrmw.start383; NOLSE-NEXT: // in Loop: Header=BB6_2 Depth=2384; NOLSE-NEXT: stlxrh w10, w12, [x11]385; NOLSE-NEXT: cbnz w10, .LBB6_2386; NOLSE-NEXT: .LBB6_4: // %atomicrmw.start387; NOLSE-NEXT: // in Loop: Header=BB6_1 Depth=1388; NOLSE-NEXT: subs w9, w8, w9, uxth389; NOLSE-NEXT: str w8, [sp, #12] // 4-byte Spill390; NOLSE-NEXT: str w8, [sp, #28] // 4-byte Spill391; NOLSE-NEXT: b.ne .LBB6_1392; NOLSE-NEXT: b .LBB6_5393; NOLSE-NEXT: .LBB6_5: // %atomicrmw.end394; NOLSE-NEXT: ldr w0, [sp, #12] // 4-byte Reload395; NOLSE-NEXT: add sp, sp, #32396; NOLSE-NEXT: ret397;398; LSE-LABEL: test_rmw_nand_16:399; LSE: // %bb.0: // %entry400; LSE-NEXT: sub sp, sp, #32401; LSE-NEXT: .cfi_def_cfa_offset 32402; LSE-NEXT: str x0, [sp, #16] // 8-byte Spill403; LSE-NEXT: ldrh w8, [x0]404; LSE-NEXT: str w8, [sp, #28] // 4-byte Spill405; LSE-NEXT: b .LBB6_1406; LSE-NEXT: .LBB6_1: // %atomicrmw.start407; LSE-NEXT: // =>This Inner Loop Header: Depth=1408; LSE-NEXT: ldr w9, [sp, #28] // 4-byte Reload409; LSE-NEXT: ldr x11, [sp, #16] // 8-byte Reload410; LSE-NEXT: mvn w8, w9411; LSE-NEXT: orr w10, w8, #0xfffffffe412; LSE-NEXT: mov w8, w9413; LSE-NEXT: casalh w8, w10, [x11]414; LSE-NEXT: subs w9, w8, w9, uxth415; LSE-NEXT: str w8, [sp, #12] // 4-byte Spill416; LSE-NEXT: str w8, [sp, #28] // 4-byte Spill417; LSE-NEXT: b.ne .LBB6_1418; LSE-NEXT: b .LBB6_2419; LSE-NEXT: .LBB6_2: // %atomicrmw.end420; LSE-NEXT: ldr w0, [sp, #12] // 4-byte Reload421; LSE-NEXT: add sp, sp, #32422; LSE-NEXT: ret423entry:424 %res = atomicrmw nand ptr %dst, i16 1 seq_cst425 ret i16 %res426}427 428define i32 @test_rmw_nand_32(ptr %dst) {429; NOLSE-LABEL: test_rmw_nand_32:430; NOLSE: // %bb.0: // %entry431; NOLSE-NEXT: sub sp, sp, #32432; NOLSE-NEXT: .cfi_def_cfa_offset 32433; NOLSE-NEXT: str x0, [sp, #16] // 8-byte Spill434; NOLSE-NEXT: ldr w8, [x0]435; NOLSE-NEXT: str w8, [sp, #28] // 4-byte Spill436; NOLSE-NEXT: b .LBB7_1437; NOLSE-NEXT: .LBB7_1: // %atomicrmw.start438; NOLSE-NEXT: // =>This Loop Header: Depth=1439; NOLSE-NEXT: // Child Loop BB7_2 Depth 2440; NOLSE-NEXT: ldr w9, [sp, #28] // 4-byte Reload441; NOLSE-NEXT: ldr x11, [sp, #16] // 8-byte Reload442; NOLSE-NEXT: mvn w8, w9443; NOLSE-NEXT: orr w12, w8, #0xfffffffe444; NOLSE-NEXT: .LBB7_2: // %atomicrmw.start445; NOLSE-NEXT: // Parent Loop BB7_1 Depth=1446; NOLSE-NEXT: // => This Inner Loop Header: Depth=2447; NOLSE-NEXT: ldaxr w8, [x11]448; NOLSE-NEXT: cmp w8, w9449; NOLSE-NEXT: b.ne .LBB7_4450; NOLSE-NEXT: // %bb.3: // %atomicrmw.start451; NOLSE-NEXT: // in Loop: Header=BB7_2 Depth=2452; NOLSE-NEXT: stlxr w10, w12, [x11]453; NOLSE-NEXT: cbnz w10, .LBB7_2454; NOLSE-NEXT: .LBB7_4: // %atomicrmw.start455; NOLSE-NEXT: // in Loop: Header=BB7_1 Depth=1456; NOLSE-NEXT: subs w9, w8, w9457; NOLSE-NEXT: str w8, [sp, #12] // 4-byte Spill458; NOLSE-NEXT: str w8, [sp, #28] // 4-byte Spill459; NOLSE-NEXT: b.ne .LBB7_1460; NOLSE-NEXT: b .LBB7_5461; NOLSE-NEXT: .LBB7_5: // %atomicrmw.end462; NOLSE-NEXT: ldr w0, [sp, #12] // 4-byte Reload463; NOLSE-NEXT: add sp, sp, #32464; NOLSE-NEXT: ret465;466; LSE-LABEL: test_rmw_nand_32:467; LSE: // %bb.0: // %entry468; LSE-NEXT: sub sp, sp, #32469; LSE-NEXT: .cfi_def_cfa_offset 32470; LSE-NEXT: str x0, [sp, #16] // 8-byte Spill471; LSE-NEXT: ldr w8, [x0]472; LSE-NEXT: str w8, [sp, #28] // 4-byte Spill473; LSE-NEXT: b .LBB7_1474; LSE-NEXT: .LBB7_1: // %atomicrmw.start475; LSE-NEXT: // =>This Inner Loop Header: Depth=1476; LSE-NEXT: ldr w9, [sp, #28] // 4-byte Reload477; LSE-NEXT: ldr x11, [sp, #16] // 8-byte Reload478; LSE-NEXT: mvn w8, w9479; LSE-NEXT: orr w10, w8, #0xfffffffe480; LSE-NEXT: mov w8, w9481; LSE-NEXT: casal w8, w10, [x11]482; LSE-NEXT: subs w9, w8, w9483; LSE-NEXT: str w8, [sp, #12] // 4-byte Spill484; LSE-NEXT: str w8, [sp, #28] // 4-byte Spill485; LSE-NEXT: b.ne .LBB7_1486; LSE-NEXT: b .LBB7_2487; LSE-NEXT: .LBB7_2: // %atomicrmw.end488; LSE-NEXT: ldr w0, [sp, #12] // 4-byte Reload489; LSE-NEXT: add sp, sp, #32490; LSE-NEXT: ret491entry:492 %res = atomicrmw nand ptr %dst, i32 1 seq_cst493 ret i32 %res494}495 496define i64 @test_rmw_nand_64(ptr %dst) {497; NOLSE-LABEL: test_rmw_nand_64:498; NOLSE: // %bb.0: // %entry499; NOLSE-NEXT: sub sp, sp, #32500; NOLSE-NEXT: .cfi_def_cfa_offset 32501; NOLSE-NEXT: str x0, [sp, #16] // 8-byte Spill502; NOLSE-NEXT: ldr x8, [x0]503; NOLSE-NEXT: str x8, [sp, #24] // 8-byte Spill504; NOLSE-NEXT: b .LBB8_1505; NOLSE-NEXT: .LBB8_1: // %atomicrmw.start506; NOLSE-NEXT: // =>This Loop Header: Depth=1507; NOLSE-NEXT: // Child Loop BB8_2 Depth 2508; NOLSE-NEXT: ldr x9, [sp, #24] // 8-byte Reload509; NOLSE-NEXT: ldr x11, [sp, #16] // 8-byte Reload510; NOLSE-NEXT: mov w8, w9511; NOLSE-NEXT: mvn w10, w8512; NOLSE-NEXT: // implicit-def: $x8513; NOLSE-NEXT: mov w8, w10514; NOLSE-NEXT: orr x12, x8, #0xfffffffffffffffe515; NOLSE-NEXT: .LBB8_2: // %atomicrmw.start516; NOLSE-NEXT: // Parent Loop BB8_1 Depth=1517; NOLSE-NEXT: // => This Inner Loop Header: Depth=2518; NOLSE-NEXT: ldaxr x8, [x11]519; NOLSE-NEXT: cmp x8, x9520; NOLSE-NEXT: b.ne .LBB8_4521; NOLSE-NEXT: // %bb.3: // %atomicrmw.start522; NOLSE-NEXT: // in Loop: Header=BB8_2 Depth=2523; NOLSE-NEXT: stlxr w10, x12, [x11]524; NOLSE-NEXT: cbnz w10, .LBB8_2525; NOLSE-NEXT: .LBB8_4: // %atomicrmw.start526; NOLSE-NEXT: // in Loop: Header=BB8_1 Depth=1527; NOLSE-NEXT: subs x9, x8, x9528; NOLSE-NEXT: str x8, [sp, #8] // 8-byte Spill529; NOLSE-NEXT: str x8, [sp, #24] // 8-byte Spill530; NOLSE-NEXT: b.ne .LBB8_1531; NOLSE-NEXT: b .LBB8_5532; NOLSE-NEXT: .LBB8_5: // %atomicrmw.end533; NOLSE-NEXT: ldr x0, [sp, #8] // 8-byte Reload534; NOLSE-NEXT: add sp, sp, #32535; NOLSE-NEXT: ret536;537; LSE-LABEL: test_rmw_nand_64:538; LSE: // %bb.0: // %entry539; LSE-NEXT: sub sp, sp, #32540; LSE-NEXT: .cfi_def_cfa_offset 32541; LSE-NEXT: str x0, [sp, #16] // 8-byte Spill542; LSE-NEXT: ldr x8, [x0]543; LSE-NEXT: str x8, [sp, #24] // 8-byte Spill544; LSE-NEXT: b .LBB8_1545; LSE-NEXT: .LBB8_1: // %atomicrmw.start546; LSE-NEXT: // =>This Inner Loop Header: Depth=1547; LSE-NEXT: ldr x9, [sp, #24] // 8-byte Reload548; LSE-NEXT: ldr x11, [sp, #16] // 8-byte Reload549; LSE-NEXT: mov w8, w9550; LSE-NEXT: mvn w10, w8551; LSE-NEXT: // implicit-def: $x8552; LSE-NEXT: mov w8, w10553; LSE-NEXT: orr x10, x8, #0xfffffffffffffffe554; LSE-NEXT: mov x8, x9555; LSE-NEXT: casal x8, x10, [x11]556; LSE-NEXT: subs x9, x8, x9557; LSE-NEXT: str x8, [sp, #8] // 8-byte Spill558; LSE-NEXT: str x8, [sp, #24] // 8-byte Spill559; LSE-NEXT: b.ne .LBB8_1560; LSE-NEXT: b .LBB8_2561; LSE-NEXT: .LBB8_2: // %atomicrmw.end562; LSE-NEXT: ldr x0, [sp, #8] // 8-byte Reload563; LSE-NEXT: add sp, sp, #32564; LSE-NEXT: ret565entry:566 %res = atomicrmw nand ptr %dst, i64 1 seq_cst567 ret i64 %res568}569 570define i128 @test_rmw_nand_128(ptr %dst) {571; NOLSE-LABEL: test_rmw_nand_128:572; NOLSE: // %bb.0: // %entry573; NOLSE-NEXT: sub sp, sp, #48574; NOLSE-NEXT: .cfi_def_cfa_offset 48575; NOLSE-NEXT: str x0, [sp, #24] // 8-byte Spill576; NOLSE-NEXT: ldr x8, [x0, #8]577; NOLSE-NEXT: ldr x9, [x0]578; NOLSE-NEXT: str x9, [sp, #32] // 8-byte Spill579; NOLSE-NEXT: str x8, [sp, #40] // 8-byte Spill580; NOLSE-NEXT: b .LBB9_1581; NOLSE-NEXT: .LBB9_1: // %atomicrmw.start582; NOLSE-NEXT: // =>This Loop Header: Depth=1583; NOLSE-NEXT: // Child Loop BB9_2 Depth 2584; NOLSE-NEXT: ldr x13, [sp, #40] // 8-byte Reload585; NOLSE-NEXT: ldr x11, [sp, #32] // 8-byte Reload586; NOLSE-NEXT: ldr x9, [sp, #24] // 8-byte Reload587; NOLSE-NEXT: mov w8, w11588; NOLSE-NEXT: mvn w10, w8589; NOLSE-NEXT: // implicit-def: $x8590; NOLSE-NEXT: mov w8, w10591; NOLSE-NEXT: orr x14, x8, #0xfffffffffffffffe592; NOLSE-NEXT: mov x15, #-1 // =0xffffffffffffffff593; NOLSE-NEXT: .LBB9_2: // %atomicrmw.start594; NOLSE-NEXT: // Parent Loop BB9_1 Depth=1595; NOLSE-NEXT: // => This Inner Loop Header: Depth=2596; NOLSE-NEXT: ldaxp x10, x12, [x9]597; NOLSE-NEXT: cmp x10, x11598; NOLSE-NEXT: cset w8, ne599; NOLSE-NEXT: cmp x12, x13600; NOLSE-NEXT: cinc w8, w8, ne601; NOLSE-NEXT: cbnz w8, .LBB9_4602; NOLSE-NEXT: // %bb.3: // %atomicrmw.start603; NOLSE-NEXT: // in Loop: Header=BB9_2 Depth=2604; NOLSE-NEXT: stlxp w8, x14, x15, [x9]605; NOLSE-NEXT: cbnz w8, .LBB9_2606; NOLSE-NEXT: b .LBB9_5607; NOLSE-NEXT: .LBB9_4: // %atomicrmw.start608; NOLSE-NEXT: // in Loop: Header=BB9_2 Depth=2609; NOLSE-NEXT: stlxp w8, x10, x12, [x9]610; NOLSE-NEXT: cbnz w8, .LBB9_2611; NOLSE-NEXT: .LBB9_5: // %atomicrmw.start612; NOLSE-NEXT: // in Loop: Header=BB9_1 Depth=1613; NOLSE-NEXT: mov x8, x12614; NOLSE-NEXT: str x8, [sp, #8] // 8-byte Spill615; NOLSE-NEXT: mov x9, x10616; NOLSE-NEXT: str x9, [sp, #16] // 8-byte Spill617; NOLSE-NEXT: subs x12, x12, x13618; NOLSE-NEXT: ccmp x10, x11, #0, eq619; NOLSE-NEXT: str x9, [sp, #32] // 8-byte Spill620; NOLSE-NEXT: str x8, [sp, #40] // 8-byte Spill621; NOLSE-NEXT: b.ne .LBB9_1622; NOLSE-NEXT: b .LBB9_6623; NOLSE-NEXT: .LBB9_6: // %atomicrmw.end624; NOLSE-NEXT: ldr x1, [sp, #8] // 8-byte Reload625; NOLSE-NEXT: ldr x0, [sp, #16] // 8-byte Reload626; NOLSE-NEXT: add sp, sp, #48627; NOLSE-NEXT: ret628;629; LSE-LABEL: test_rmw_nand_128:630; LSE: // %bb.0: // %entry631; LSE-NEXT: sub sp, sp, #48632; LSE-NEXT: .cfi_def_cfa_offset 48633; LSE-NEXT: str x0, [sp, #24] // 8-byte Spill634; LSE-NEXT: ldr x8, [x0, #8]635; LSE-NEXT: ldr x9, [x0]636; LSE-NEXT: str x9, [sp, #32] // 8-byte Spill637; LSE-NEXT: str x8, [sp, #40] // 8-byte Spill638; LSE-NEXT: b .LBB9_1639; LSE-NEXT: .LBB9_1: // %atomicrmw.start640; LSE-NEXT: // =>This Inner Loop Header: Depth=1641; LSE-NEXT: ldr x11, [sp, #40] // 8-byte Reload642; LSE-NEXT: ldr x10, [sp, #32] // 8-byte Reload643; LSE-NEXT: ldr x8, [sp, #24] // 8-byte Reload644; LSE-NEXT: mov x0, x10645; LSE-NEXT: mov x1, x11646; LSE-NEXT: mov w9, w10647; LSE-NEXT: mvn w12, w9648; LSE-NEXT: // implicit-def: $x9649; LSE-NEXT: mov w9, w12650; LSE-NEXT: orr x2, x9, #0xfffffffffffffffe651; LSE-NEXT: mov x9, #-1 // =0xffffffffffffffff652; LSE-NEXT: // kill: def $x2 killed $x2 def $x2_x3653; LSE-NEXT: mov x3, x9654; LSE-NEXT: caspal x0, x1, x2, x3, [x8]655; LSE-NEXT: mov x9, x0656; LSE-NEXT: str x9, [sp, #8] // 8-byte Spill657; LSE-NEXT: mov x8, x1658; LSE-NEXT: str x8, [sp, #16] // 8-byte Spill659; LSE-NEXT: subs x11, x8, x11660; LSE-NEXT: ccmp x9, x10, #0, eq661; LSE-NEXT: str x9, [sp, #32] // 8-byte Spill662; LSE-NEXT: str x8, [sp, #40] // 8-byte Spill663; LSE-NEXT: b.ne .LBB9_1664; LSE-NEXT: b .LBB9_2665; LSE-NEXT: .LBB9_2: // %atomicrmw.end666; LSE-NEXT: ldr x1, [sp, #16] // 8-byte Reload667; LSE-NEXT: ldr x0, [sp, #8] // 8-byte Reload668; LSE-NEXT: add sp, sp, #48669; LSE-NEXT: ret670entry:671 %res = atomicrmw nand ptr %dst, i128 1 seq_cst672 ret i128 %res673}674