16498 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -mcpu=bonaire < %s | FileCheck -check-prefix=GCN1 %s3; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=GCN2 %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s5 6define amdgpu_kernel void @atomic_add_i64_offset(ptr %out, i64 %in) {7; GCN1-LABEL: atomic_add_i64_offset:8; GCN1: ; %bb.0: ; %entry9; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD010; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD111; GCN1-NEXT: s_mov_b32 s14, -112; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x913; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d14; GCN1-NEXT: s_mov_b32 s15, 0xe8f00015; GCN1-NEXT: s_add_u32 s12, s12, s1116; GCN1-NEXT: s_addc_u32 s13, s13, 017; GCN1-NEXT: s_waitcnt lgkmcnt(0)18; GCN1-NEXT: s_add_u32 s0, s0, 3219; GCN1-NEXT: s_addc_u32 s1, s1, 020; GCN1-NEXT: s_cmp_eq_u32 s1, s421; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 022; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]23; GCN1-NEXT: s_mov_b64 s[4:5], -124; GCN1-NEXT: s_cbranch_vccnz .LBB0_325; GCN1-NEXT: ; %bb.1: ; %Flow26; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]27; GCN1-NEXT: s_cbranch_vccz .LBB0_428; GCN1-NEXT: .LBB0_2: ; %atomicrmw.phi29; GCN1-NEXT: s_endpgm30; GCN1-NEXT: .LBB0_3: ; %atomicrmw.global31; GCN1-NEXT: v_mov_b32_e32 v0, s032; GCN1-NEXT: v_mov_b32_e32 v2, s233; GCN1-NEXT: v_mov_b32_e32 v1, s134; GCN1-NEXT: v_mov_b32_e32 v3, s335; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]36; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)37; GCN1-NEXT: buffer_wbinvl1_vol38; GCN1-NEXT: s_cbranch_execnz .LBB0_239; GCN1-NEXT: .LBB0_4: ; %atomicrmw.private40; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 041; GCN1-NEXT: v_mov_b32_e32 v4, s342; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec43; GCN1-NEXT: s_cselect_b32 s0, s0, -144; GCN1-NEXT: v_mov_b32_e32 v0, s045; GCN1-NEXT: s_add_i32 s0, s0, 446; GCN1-NEXT: buffer_load_dword v1, v0, s[12:15], 0 offen47; GCN1-NEXT: v_mov_b32_e32 v2, s048; GCN1-NEXT: buffer_load_dword v3, v2, s[12:15], 0 offen49; GCN1-NEXT: s_waitcnt vmcnt(1)50; GCN1-NEXT: v_add_i32_e32 v1, vcc, s2, v151; GCN1-NEXT: s_waitcnt vmcnt(0)52; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc53; GCN1-NEXT: buffer_store_dword v1, v0, s[12:15], 0 offen54; GCN1-NEXT: buffer_store_dword v3, v2, s[12:15], 0 offen55; GCN1-NEXT: s_endpgm56;57; GCN2-LABEL: atomic_add_i64_offset:58; GCN2: ; %bb.0: ; %entry59; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD060; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD161; GCN2-NEXT: s_mov_b32 s90, -162; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2463; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf464; GCN2-NEXT: s_mov_b32 s91, 0xe8000065; GCN2-NEXT: s_add_u32 s88, s88, s1166; GCN2-NEXT: s_addc_u32 s89, s89, 067; GCN2-NEXT: s_waitcnt lgkmcnt(0)68; GCN2-NEXT: s_add_u32 s0, s0, 3269; GCN2-NEXT: s_addc_u32 s1, s1, 070; GCN2-NEXT: s_cmp_eq_u32 s1, s471; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 072; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]73; GCN2-NEXT: s_mov_b64 s[4:5], -174; GCN2-NEXT: s_cbranch_vccnz .LBB0_375; GCN2-NEXT: ; %bb.1: ; %Flow76; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]77; GCN2-NEXT: s_cbranch_vccz .LBB0_478; GCN2-NEXT: .LBB0_2: ; %atomicrmw.phi79; GCN2-NEXT: s_endpgm80; GCN2-NEXT: .LBB0_3: ; %atomicrmw.global81; GCN2-NEXT: v_mov_b32_e32 v0, s082; GCN2-NEXT: v_mov_b32_e32 v2, s283; GCN2-NEXT: v_mov_b32_e32 v1, s184; GCN2-NEXT: v_mov_b32_e32 v3, s385; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]86; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)87; GCN2-NEXT: buffer_wbinvl1_vol88; GCN2-NEXT: s_cbranch_execnz .LBB0_289; GCN2-NEXT: .LBB0_4: ; %atomicrmw.private90; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 091; GCN2-NEXT: s_cselect_b32 s0, s0, -192; GCN2-NEXT: v_mov_b32_e32 v0, s093; GCN2-NEXT: s_add_i32 s0, s0, 494; GCN2-NEXT: buffer_load_dword v1, v0, s[88:91], 0 offen95; GCN2-NEXT: v_mov_b32_e32 v2, s096; GCN2-NEXT: buffer_load_dword v3, v2, s[88:91], 0 offen97; GCN2-NEXT: v_mov_b32_e32 v4, s398; GCN2-NEXT: s_waitcnt vmcnt(1)99; GCN2-NEXT: v_add_u32_e32 v1, vcc, s2, v1100; GCN2-NEXT: s_waitcnt vmcnt(0)101; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc102; GCN2-NEXT: buffer_store_dword v1, v0, s[88:91], 0 offen103; GCN2-NEXT: buffer_store_dword v3, v2, s[88:91], 0 offen104; GCN2-NEXT: s_endpgm105;106; GFX12-LABEL: atomic_add_i64_offset:107; GFX12: ; %bb.0: ; %entry108; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24109; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base110; GFX12-NEXT: s_wait_kmcnt 0x0111; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 32112; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)113; GFX12-NEXT: s_cmp_eq_u32 s1, s7114; GFX12-NEXT: s_cselect_b32 s4, -1, 0115; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4116; GFX12-NEXT: s_mov_b32 s4, -1117; GFX12-NEXT: s_cbranch_vccnz .LBB0_3118; GFX12-NEXT: ; %bb.1: ; %Flow119; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4120; GFX12-NEXT: s_cbranch_vccz .LBB0_4121; GFX12-NEXT: .LBB0_2: ; %atomicrmw.phi122; GFX12-NEXT: s_endpgm123; GFX12-NEXT: .LBB0_3: ; %atomicrmw.global124; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1125; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3126; GFX12-NEXT: flat_atomic_add_u64 v[0:1], v[2:3] scope:SCOPE_DEV127; GFX12-NEXT: s_wait_storecnt_dscnt 0x0128; GFX12-NEXT: global_inv scope:SCOPE_DEV129; GFX12-NEXT: s_cbranch_execnz .LBB0_2130; GFX12-NEXT: .LBB0_4: ; %atomicrmw.private131; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 0132; GFX12-NEXT: s_cselect_b32 s0, s0, -1133; GFX12-NEXT: scratch_load_b64 v[0:1], off, s0134; GFX12-NEXT: s_wait_loadcnt 0x0135; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, s2136; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)137; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo138; GFX12-NEXT: scratch_store_b64 off, v[0:1], s0139; GFX12-NEXT: s_endpgm140entry:141 %gep = getelementptr inbounds i64, ptr %out, i64 4142 %tmp0 = atomicrmw volatile add ptr %gep, i64 %in syncscope("agent") seq_cst143 ret void144}145 146define amdgpu_kernel void @atomic_add_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {147; GCN1-LABEL: atomic_add_i64_ret_offset:148; GCN1: ; %bb.0: ; %entry149; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0150; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1151; GCN1-NEXT: s_mov_b32 s14, -1152; GCN1-NEXT: s_mov_b32 s15, 0xe8f000153; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9154; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f155; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd156; GCN1-NEXT: s_add_u32 s12, s12, s11157; GCN1-NEXT: s_addc_u32 s13, s13, 0158; GCN1-NEXT: s_waitcnt lgkmcnt(0)159; GCN1-NEXT: s_add_u32 s0, s0, 32160; GCN1-NEXT: s_addc_u32 s1, s1, 0161; GCN1-NEXT: s_cmp_eq_u32 s1, s6162; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0163; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]164; GCN1-NEXT: s_cbranch_vccz .LBB1_2165; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global166; GCN1-NEXT: v_mov_b32_e32 v0, s0167; GCN1-NEXT: v_mov_b32_e32 v2, s4168; GCN1-NEXT: v_mov_b32_e32 v1, s1169; GCN1-NEXT: v_mov_b32_e32 v3, s5170; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc171; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)172; GCN1-NEXT: buffer_wbinvl1_vol173; GCN1-NEXT: s_cbranch_execz .LBB1_3174; GCN1-NEXT: s_branch .LBB1_4175; GCN1-NEXT: .LBB1_2:176; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1177; GCN1-NEXT: .LBB1_3: ; %atomicrmw.private178; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 0179; GCN1-NEXT: v_mov_b32_e32 v4, s5180; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec181; GCN1-NEXT: s_cselect_b32 s0, s0, -1182; GCN1-NEXT: v_mov_b32_e32 v2, s0183; GCN1-NEXT: s_add_i32 s0, s0, 4184; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen185; GCN1-NEXT: v_mov_b32_e32 v3, s0186; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen187; GCN1-NEXT: s_waitcnt vmcnt(1)188; GCN1-NEXT: v_add_i32_e32 v5, vcc, s4, v0189; GCN1-NEXT: s_waitcnt vmcnt(0)190; GCN1-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc191; GCN1-NEXT: buffer_store_dword v5, v2, s[12:15], 0 offen192; GCN1-NEXT: buffer_store_dword v4, v3, s[12:15], 0 offen193; GCN1-NEXT: .LBB1_4: ; %atomicrmw.end194; GCN1-NEXT: v_mov_b32_e32 v2, s2195; GCN1-NEXT: v_mov_b32_e32 v3, s3196; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]197; GCN1-NEXT: s_endpgm198;199; GCN2-LABEL: atomic_add_i64_ret_offset:200; GCN2: ; %bb.0: ; %entry201; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0202; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1203; GCN2-NEXT: s_mov_b32 s90, -1204; GCN2-NEXT: s_mov_b32 s91, 0xe80000205; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24206; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc207; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34208; GCN2-NEXT: s_add_u32 s88, s88, s11209; GCN2-NEXT: s_addc_u32 s89, s89, 0210; GCN2-NEXT: s_waitcnt lgkmcnt(0)211; GCN2-NEXT: s_add_u32 s0, s0, 32212; GCN2-NEXT: s_addc_u32 s1, s1, 0213; GCN2-NEXT: s_cmp_eq_u32 s1, s6214; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0215; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]216; GCN2-NEXT: s_cbranch_vccz .LBB1_2217; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global218; GCN2-NEXT: v_mov_b32_e32 v0, s0219; GCN2-NEXT: v_mov_b32_e32 v2, s4220; GCN2-NEXT: v_mov_b32_e32 v1, s1221; GCN2-NEXT: v_mov_b32_e32 v3, s5222; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc223; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)224; GCN2-NEXT: buffer_wbinvl1_vol225; GCN2-NEXT: s_cbranch_execz .LBB1_3226; GCN2-NEXT: s_branch .LBB1_4227; GCN2-NEXT: .LBB1_2:228; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1229; GCN2-NEXT: .LBB1_3: ; %atomicrmw.private230; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0231; GCN2-NEXT: s_cselect_b32 s0, s0, -1232; GCN2-NEXT: v_mov_b32_e32 v2, s0233; GCN2-NEXT: s_add_i32 s0, s0, 4234; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen235; GCN2-NEXT: v_mov_b32_e32 v3, s0236; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen237; GCN2-NEXT: v_mov_b32_e32 v4, s5238; GCN2-NEXT: s_waitcnt vmcnt(1)239; GCN2-NEXT: v_add_u32_e32 v5, vcc, s4, v0240; GCN2-NEXT: s_waitcnt vmcnt(0)241; GCN2-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc242; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen243; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen244; GCN2-NEXT: .LBB1_4: ; %atomicrmw.end245; GCN2-NEXT: v_mov_b32_e32 v2, s2246; GCN2-NEXT: v_mov_b32_e32 v3, s3247; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]248; GCN2-NEXT: s_endpgm249;250; GFX12-LABEL: atomic_add_i64_ret_offset:251; GFX12: ; %bb.0: ; %entry252; GFX12-NEXT: s_clause 0x1253; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24254; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x34255; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base256; GFX12-NEXT: s_wait_kmcnt 0x0257; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 32258; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)259; GFX12-NEXT: s_cmp_eq_u32 s1, s7260; GFX12-NEXT: s_cselect_b32 s6, -1, 0261; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6262; GFX12-NEXT: s_cbranch_vccz .LBB1_2263; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global264; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1265; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5266; GFX12-NEXT: flat_atomic_add_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV267; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0268; GFX12-NEXT: global_inv scope:SCOPE_DEV269; GFX12-NEXT: s_cbranch_execz .LBB1_3270; GFX12-NEXT: s_branch .LBB1_4271; GFX12-NEXT: .LBB1_2:272; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1273; GFX12-NEXT: .LBB1_3: ; %atomicrmw.private274; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 0275; GFX12-NEXT: s_cselect_b32 s0, s0, -1276; GFX12-NEXT: scratch_load_b64 v[0:1], off, s0277; GFX12-NEXT: s_wait_loadcnt 0x0278; GFX12-NEXT: v_add_co_u32 v2, vcc_lo, v0, s4279; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)280; GFX12-NEXT: v_add_co_ci_u32_e64 v3, null, s5, v1, vcc_lo281; GFX12-NEXT: scratch_store_b64 off, v[2:3], s0282; GFX12-NEXT: .LBB1_4: ; %atomicrmw.end283; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3284; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]285; GFX12-NEXT: s_endpgm286entry:287 %gep = getelementptr inbounds i64, ptr %out, i64 4288 %tmp0 = atomicrmw volatile add ptr %gep, i64 %in syncscope("agent") seq_cst289 store i64 %tmp0, ptr %out2290 ret void291}292 293define amdgpu_kernel void @atomic_add_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {294; GCN1-LABEL: atomic_add_i64_addr64_offset:295; GCN1: ; %bb.0: ; %entry296; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0297; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1298; GCN1-NEXT: s_mov_b32 s14, -1299; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd300; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9301; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f302; GCN1-NEXT: s_mov_b32 s15, 0xe8f000303; GCN1-NEXT: s_add_u32 s12, s12, s11304; GCN1-NEXT: s_addc_u32 s13, s13, 0305; GCN1-NEXT: s_waitcnt lgkmcnt(0)306; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 3307; GCN1-NEXT: s_add_u32 s0, s0, s4308; GCN1-NEXT: s_addc_u32 s1, s1, s5309; GCN1-NEXT: s_add_u32 s0, s0, 32310; GCN1-NEXT: s_addc_u32 s1, s1, 0311; GCN1-NEXT: s_cmp_eq_u32 s1, s8312; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0313; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]314; GCN1-NEXT: s_mov_b64 s[4:5], -1315; GCN1-NEXT: s_cbranch_vccnz .LBB2_3316; GCN1-NEXT: ; %bb.1: ; %Flow317; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]318; GCN1-NEXT: s_cbranch_vccz .LBB2_4319; GCN1-NEXT: .LBB2_2: ; %atomicrmw.phi320; GCN1-NEXT: s_endpgm321; GCN1-NEXT: .LBB2_3: ; %atomicrmw.global322; GCN1-NEXT: v_mov_b32_e32 v0, s0323; GCN1-NEXT: v_mov_b32_e32 v2, s2324; GCN1-NEXT: v_mov_b32_e32 v1, s1325; GCN1-NEXT: v_mov_b32_e32 v3, s3326; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]327; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)328; GCN1-NEXT: buffer_wbinvl1_vol329; GCN1-NEXT: s_cbranch_execnz .LBB2_2330; GCN1-NEXT: .LBB2_4: ; %atomicrmw.private331; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0332; GCN1-NEXT: v_mov_b32_e32 v4, s3333; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec334; GCN1-NEXT: s_cselect_b32 s0, s0, -1335; GCN1-NEXT: v_mov_b32_e32 v0, s0336; GCN1-NEXT: s_add_i32 s0, s0, 4337; GCN1-NEXT: buffer_load_dword v1, v0, s[12:15], 0 offen338; GCN1-NEXT: v_mov_b32_e32 v2, s0339; GCN1-NEXT: buffer_load_dword v3, v2, s[12:15], 0 offen340; GCN1-NEXT: s_waitcnt vmcnt(1)341; GCN1-NEXT: v_add_i32_e32 v1, vcc, s2, v1342; GCN1-NEXT: s_waitcnt vmcnt(0)343; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc344; GCN1-NEXT: buffer_store_dword v1, v0, s[12:15], 0 offen345; GCN1-NEXT: buffer_store_dword v3, v2, s[12:15], 0 offen346; GCN1-NEXT: s_endpgm347;348; GCN2-LABEL: atomic_add_i64_addr64_offset:349; GCN2: ; %bb.0: ; %entry350; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0351; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1352; GCN2-NEXT: s_mov_b32 s90, -1353; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34354; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24355; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc356; GCN2-NEXT: s_mov_b32 s91, 0xe80000357; GCN2-NEXT: s_add_u32 s88, s88, s11358; GCN2-NEXT: s_addc_u32 s89, s89, 0359; GCN2-NEXT: s_waitcnt lgkmcnt(0)360; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3361; GCN2-NEXT: s_add_u32 s0, s0, s4362; GCN2-NEXT: s_addc_u32 s1, s1, s5363; GCN2-NEXT: s_add_u32 s0, s0, 32364; GCN2-NEXT: s_addc_u32 s1, s1, 0365; GCN2-NEXT: s_cmp_eq_u32 s1, s8366; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0367; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]368; GCN2-NEXT: s_mov_b64 s[4:5], -1369; GCN2-NEXT: s_cbranch_vccnz .LBB2_3370; GCN2-NEXT: ; %bb.1: ; %Flow371; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]372; GCN2-NEXT: s_cbranch_vccz .LBB2_4373; GCN2-NEXT: .LBB2_2: ; %atomicrmw.phi374; GCN2-NEXT: s_endpgm375; GCN2-NEXT: .LBB2_3: ; %atomicrmw.global376; GCN2-NEXT: v_mov_b32_e32 v0, s0377; GCN2-NEXT: v_mov_b32_e32 v2, s2378; GCN2-NEXT: v_mov_b32_e32 v1, s1379; GCN2-NEXT: v_mov_b32_e32 v3, s3380; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]381; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)382; GCN2-NEXT: buffer_wbinvl1_vol383; GCN2-NEXT: s_cbranch_execnz .LBB2_2384; GCN2-NEXT: .LBB2_4: ; %atomicrmw.private385; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0386; GCN2-NEXT: s_cselect_b32 s0, s0, -1387; GCN2-NEXT: v_mov_b32_e32 v0, s0388; GCN2-NEXT: s_add_i32 s0, s0, 4389; GCN2-NEXT: buffer_load_dword v1, v0, s[88:91], 0 offen390; GCN2-NEXT: v_mov_b32_e32 v2, s0391; GCN2-NEXT: buffer_load_dword v3, v2, s[88:91], 0 offen392; GCN2-NEXT: v_mov_b32_e32 v4, s3393; GCN2-NEXT: s_waitcnt vmcnt(1)394; GCN2-NEXT: v_add_u32_e32 v1, vcc, s2, v1395; GCN2-NEXT: s_waitcnt vmcnt(0)396; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc397; GCN2-NEXT: buffer_store_dword v1, v0, s[88:91], 0 offen398; GCN2-NEXT: buffer_store_dword v3, v2, s[88:91], 0 offen399; GCN2-NEXT: s_endpgm400;401; GFX12-LABEL: atomic_add_i64_addr64_offset:402; GFX12: ; %bb.0: ; %entry403; GFX12-NEXT: s_clause 0x1404; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x34405; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24406; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base407; GFX12-NEXT: s_wait_kmcnt 0x0408; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 3409; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)410; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]411; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 32412; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)413; GFX12-NEXT: s_cmp_eq_u32 s1, s7414; GFX12-NEXT: s_cselect_b32 s4, -1, 0415; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4416; GFX12-NEXT: s_mov_b32 s4, -1417; GFX12-NEXT: s_cbranch_vccnz .LBB2_3418; GFX12-NEXT: ; %bb.1: ; %Flow419; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4420; GFX12-NEXT: s_cbranch_vccz .LBB2_4421; GFX12-NEXT: .LBB2_2: ; %atomicrmw.phi422; GFX12-NEXT: s_endpgm423; GFX12-NEXT: .LBB2_3: ; %atomicrmw.global424; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1425; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3426; GFX12-NEXT: flat_atomic_add_u64 v[0:1], v[2:3] scope:SCOPE_DEV427; GFX12-NEXT: s_wait_storecnt_dscnt 0x0428; GFX12-NEXT: global_inv scope:SCOPE_DEV429; GFX12-NEXT: s_cbranch_execnz .LBB2_2430; GFX12-NEXT: .LBB2_4: ; %atomicrmw.private431; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 0432; GFX12-NEXT: s_cselect_b32 s0, s0, -1433; GFX12-NEXT: scratch_load_b64 v[0:1], off, s0434; GFX12-NEXT: s_wait_loadcnt 0x0435; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, s2436; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)437; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo438; GFX12-NEXT: scratch_store_b64 off, v[0:1], s0439; GFX12-NEXT: s_endpgm440entry:441 %ptr = getelementptr inbounds i64, ptr %out, i64 %index442 %gep = getelementptr inbounds i64, ptr %ptr, i64 4443 %tmp0 = atomicrmw volatile add ptr %gep, i64 %in syncscope("agent") seq_cst444 ret void445}446 447define amdgpu_kernel void @atomic_add_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {448; GCN1-LABEL: atomic_add_i64_ret_addr64_offset:449; GCN1: ; %bb.0: ; %entry450; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0451; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1452; GCN1-NEXT: s_mov_b32 s18, -1453; GCN1-NEXT: s_mov_b32 s19, 0xe8f000454; GCN1-NEXT: s_add_u32 s16, s16, s11455; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9456; GCN1-NEXT: s_load_dword s2, s[4:5], 0x41457; GCN1-NEXT: s_addc_u32 s17, s17, 0458; GCN1-NEXT: s_waitcnt lgkmcnt(0)459; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 3460; GCN1-NEXT: s_add_u32 s0, s8, s0461; GCN1-NEXT: s_addc_u32 s1, s9, s1462; GCN1-NEXT: s_add_u32 s0, s0, 32463; GCN1-NEXT: s_addc_u32 s1, s1, 0464; GCN1-NEXT: s_cmp_eq_u32 s1, s2465; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0466; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]467; GCN1-NEXT: s_cbranch_vccz .LBB3_2468; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global469; GCN1-NEXT: v_mov_b32_e32 v0, s0470; GCN1-NEXT: v_mov_b32_e32 v2, s12471; GCN1-NEXT: v_mov_b32_e32 v1, s1472; GCN1-NEXT: v_mov_b32_e32 v3, s13473; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc474; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)475; GCN1-NEXT: buffer_wbinvl1_vol476; GCN1-NEXT: s_cbranch_execz .LBB3_3477; GCN1-NEXT: s_branch .LBB3_4478; GCN1-NEXT: .LBB3_2:479; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1480; GCN1-NEXT: .LBB3_3: ; %atomicrmw.private481; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0482; GCN1-NEXT: v_mov_b32_e32 v4, s13483; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec484; GCN1-NEXT: s_cselect_b32 s0, s0, -1485; GCN1-NEXT: v_mov_b32_e32 v2, s0486; GCN1-NEXT: s_add_i32 s0, s0, 4487; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen488; GCN1-NEXT: v_mov_b32_e32 v3, s0489; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen490; GCN1-NEXT: s_waitcnt vmcnt(1)491; GCN1-NEXT: v_add_i32_e32 v5, vcc, s12, v0492; GCN1-NEXT: s_waitcnt vmcnt(0)493; GCN1-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc494; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen495; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen496; GCN1-NEXT: .LBB3_4: ; %atomicrmw.end497; GCN1-NEXT: v_mov_b32_e32 v2, s10498; GCN1-NEXT: v_mov_b32_e32 v3, s11499; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]500; GCN1-NEXT: s_endpgm501;502; GCN2-LABEL: atomic_add_i64_ret_addr64_offset:503; GCN2: ; %bb.0: ; %entry504; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0505; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1506; GCN2-NEXT: s_mov_b32 s90, -1507; GCN2-NEXT: s_mov_b32 s91, 0xe80000508; GCN2-NEXT: s_add_u32 s88, s88, s11509; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24510; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104511; GCN2-NEXT: s_addc_u32 s89, s89, 0512; GCN2-NEXT: s_waitcnt lgkmcnt(0)513; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3514; GCN2-NEXT: s_add_u32 s0, s8, s0515; GCN2-NEXT: s_addc_u32 s1, s9, s1516; GCN2-NEXT: s_add_u32 s0, s0, 32517; GCN2-NEXT: s_addc_u32 s1, s1, 0518; GCN2-NEXT: s_cmp_eq_u32 s1, s2519; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0520; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]521; GCN2-NEXT: s_cbranch_vccz .LBB3_2522; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global523; GCN2-NEXT: v_mov_b32_e32 v0, s0524; GCN2-NEXT: v_mov_b32_e32 v2, s12525; GCN2-NEXT: v_mov_b32_e32 v1, s1526; GCN2-NEXT: v_mov_b32_e32 v3, s13527; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc528; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)529; GCN2-NEXT: buffer_wbinvl1_vol530; GCN2-NEXT: s_cbranch_execz .LBB3_3531; GCN2-NEXT: s_branch .LBB3_4532; GCN2-NEXT: .LBB3_2:533; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1534; GCN2-NEXT: .LBB3_3: ; %atomicrmw.private535; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0536; GCN2-NEXT: s_cselect_b32 s0, s0, -1537; GCN2-NEXT: v_mov_b32_e32 v2, s0538; GCN2-NEXT: s_add_i32 s0, s0, 4539; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen540; GCN2-NEXT: v_mov_b32_e32 v3, s0541; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen542; GCN2-NEXT: v_mov_b32_e32 v4, s13543; GCN2-NEXT: s_waitcnt vmcnt(1)544; GCN2-NEXT: v_add_u32_e32 v5, vcc, s12, v0545; GCN2-NEXT: s_waitcnt vmcnt(0)546; GCN2-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc547; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen548; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen549; GCN2-NEXT: .LBB3_4: ; %atomicrmw.end550; GCN2-NEXT: v_mov_b32_e32 v2, s10551; GCN2-NEXT: v_mov_b32_e32 v3, s11552; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]553; GCN2-NEXT: s_endpgm554;555; GFX12-LABEL: atomic_add_i64_ret_addr64_offset:556; GFX12: ; %bb.0: ; %entry557; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x24558; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base559; GFX12-NEXT: s_wait_kmcnt 0x0560; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 3561; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)562; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]563; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 32564; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)565; GFX12-NEXT: s_cmp_eq_u32 s1, s9566; GFX12-NEXT: s_cselect_b32 s6, -1, 0567; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6568; GFX12-NEXT: s_cbranch_vccz .LBB3_2569; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global570; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1571; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5572; GFX12-NEXT: flat_atomic_add_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV573; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0574; GFX12-NEXT: global_inv scope:SCOPE_DEV575; GFX12-NEXT: s_cbranch_execz .LBB3_3576; GFX12-NEXT: s_branch .LBB3_4577; GFX12-NEXT: .LBB3_2:578; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1579; GFX12-NEXT: .LBB3_3: ; %atomicrmw.private580; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 0581; GFX12-NEXT: s_cselect_b32 s0, s0, -1582; GFX12-NEXT: scratch_load_b64 v[0:1], off, s0583; GFX12-NEXT: s_wait_loadcnt 0x0584; GFX12-NEXT: v_add_co_u32 v2, vcc_lo, v0, s4585; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)586; GFX12-NEXT: v_add_co_ci_u32_e64 v3, null, s5, v1, vcc_lo587; GFX12-NEXT: scratch_store_b64 off, v[2:3], s0588; GFX12-NEXT: .LBB3_4: ; %atomicrmw.end589; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3590; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]591; GFX12-NEXT: s_endpgm592entry:593 %ptr = getelementptr inbounds i64, ptr %out, i64 %index594 %gep = getelementptr inbounds i64, ptr %ptr, i64 4595 %tmp0 = atomicrmw volatile add ptr %gep, i64 %in syncscope("agent") seq_cst596 store i64 %tmp0, ptr %out2597 ret void598}599 600define amdgpu_kernel void @atomic_add_i64(ptr %out, i64 %in) {601; GCN1-LABEL: atomic_add_i64:602; GCN1: ; %bb.0: ; %entry603; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0604; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1605; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9606; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d607; GCN1-NEXT: s_mov_b32 s14, -1608; GCN1-NEXT: s_mov_b32 s15, 0xe8f000609; GCN1-NEXT: s_add_u32 s12, s12, s11610; GCN1-NEXT: s_addc_u32 s13, s13, 0611; GCN1-NEXT: s_waitcnt lgkmcnt(0)612; GCN1-NEXT: s_cmp_eq_u32 s1, s4613; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0614; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]615; GCN1-NEXT: s_mov_b64 s[4:5], -1616; GCN1-NEXT: s_cbranch_vccnz .LBB4_3617; GCN1-NEXT: ; %bb.1: ; %Flow618; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]619; GCN1-NEXT: s_cbranch_vccz .LBB4_4620; GCN1-NEXT: .LBB4_2: ; %atomicrmw.phi621; GCN1-NEXT: s_endpgm622; GCN1-NEXT: .LBB4_3: ; %atomicrmw.global623; GCN1-NEXT: v_mov_b32_e32 v0, s0624; GCN1-NEXT: v_mov_b32_e32 v2, s2625; GCN1-NEXT: v_mov_b32_e32 v1, s1626; GCN1-NEXT: v_mov_b32_e32 v3, s3627; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]628; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)629; GCN1-NEXT: buffer_wbinvl1_vol630; GCN1-NEXT: s_cbranch_execnz .LBB4_2631; GCN1-NEXT: .LBB4_4: ; %atomicrmw.private632; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0633; GCN1-NEXT: v_mov_b32_e32 v4, s3634; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec635; GCN1-NEXT: s_cselect_b32 s0, s0, -1636; GCN1-NEXT: v_mov_b32_e32 v0, s0637; GCN1-NEXT: s_add_i32 s0, s0, 4638; GCN1-NEXT: buffer_load_dword v1, v0, s[12:15], 0 offen639; GCN1-NEXT: v_mov_b32_e32 v2, s0640; GCN1-NEXT: buffer_load_dword v3, v2, s[12:15], 0 offen641; GCN1-NEXT: s_waitcnt vmcnt(1)642; GCN1-NEXT: v_add_i32_e32 v1, vcc, s2, v1643; GCN1-NEXT: s_waitcnt vmcnt(0)644; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc645; GCN1-NEXT: buffer_store_dword v1, v0, s[12:15], 0 offen646; GCN1-NEXT: buffer_store_dword v3, v2, s[12:15], 0 offen647; GCN1-NEXT: s_endpgm648;649; GCN2-LABEL: atomic_add_i64:650; GCN2: ; %bb.0: ; %entry651; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0652; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1653; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24654; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf4655; GCN2-NEXT: s_mov_b32 s90, -1656; GCN2-NEXT: s_mov_b32 s91, 0xe80000657; GCN2-NEXT: s_add_u32 s88, s88, s11658; GCN2-NEXT: s_addc_u32 s89, s89, 0659; GCN2-NEXT: s_waitcnt lgkmcnt(0)660; GCN2-NEXT: s_cmp_eq_u32 s1, s4661; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0662; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]663; GCN2-NEXT: s_mov_b64 s[4:5], -1664; GCN2-NEXT: s_cbranch_vccnz .LBB4_3665; GCN2-NEXT: ; %bb.1: ; %Flow666; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]667; GCN2-NEXT: s_cbranch_vccz .LBB4_4668; GCN2-NEXT: .LBB4_2: ; %atomicrmw.phi669; GCN2-NEXT: s_endpgm670; GCN2-NEXT: .LBB4_3: ; %atomicrmw.global671; GCN2-NEXT: v_mov_b32_e32 v0, s0672; GCN2-NEXT: v_mov_b32_e32 v2, s2673; GCN2-NEXT: v_mov_b32_e32 v1, s1674; GCN2-NEXT: v_mov_b32_e32 v3, s3675; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]676; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)677; GCN2-NEXT: buffer_wbinvl1_vol678; GCN2-NEXT: s_cbranch_execnz .LBB4_2679; GCN2-NEXT: .LBB4_4: ; %atomicrmw.private680; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0681; GCN2-NEXT: s_cselect_b32 s0, s0, -1682; GCN2-NEXT: v_mov_b32_e32 v0, s0683; GCN2-NEXT: s_add_i32 s0, s0, 4684; GCN2-NEXT: buffer_load_dword v1, v0, s[88:91], 0 offen685; GCN2-NEXT: v_mov_b32_e32 v2, s0686; GCN2-NEXT: buffer_load_dword v3, v2, s[88:91], 0 offen687; GCN2-NEXT: v_mov_b32_e32 v4, s3688; GCN2-NEXT: s_waitcnt vmcnt(1)689; GCN2-NEXT: v_add_u32_e32 v1, vcc, s2, v1690; GCN2-NEXT: s_waitcnt vmcnt(0)691; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc692; GCN2-NEXT: buffer_store_dword v1, v0, s[88:91], 0 offen693; GCN2-NEXT: buffer_store_dword v3, v2, s[88:91], 0 offen694; GCN2-NEXT: s_endpgm695;696; GFX12-LABEL: atomic_add_i64:697; GFX12: ; %bb.0: ; %entry698; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24699; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base700; GFX12-NEXT: s_wait_kmcnt 0x0701; GFX12-NEXT: s_cmp_eq_u32 s1, s7702; GFX12-NEXT: s_cselect_b32 s4, -1, 0703; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)704; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4705; GFX12-NEXT: s_mov_b32 s4, -1706; GFX12-NEXT: s_cbranch_vccnz .LBB4_3707; GFX12-NEXT: ; %bb.1: ; %Flow708; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4709; GFX12-NEXT: s_cbranch_vccz .LBB4_4710; GFX12-NEXT: .LBB4_2: ; %atomicrmw.phi711; GFX12-NEXT: s_endpgm712; GFX12-NEXT: .LBB4_3: ; %atomicrmw.global713; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1714; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3715; GFX12-NEXT: flat_atomic_add_u64 v[0:1], v[2:3] scope:SCOPE_DEV716; GFX12-NEXT: s_wait_storecnt_dscnt 0x0717; GFX12-NEXT: global_inv scope:SCOPE_DEV718; GFX12-NEXT: s_cbranch_execnz .LBB4_2719; GFX12-NEXT: .LBB4_4: ; %atomicrmw.private720; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 0721; GFX12-NEXT: s_cselect_b32 s0, s0, -1722; GFX12-NEXT: scratch_load_b64 v[0:1], off, s0723; GFX12-NEXT: s_wait_loadcnt 0x0724; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, s2725; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)726; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo727; GFX12-NEXT: scratch_store_b64 off, v[0:1], s0728; GFX12-NEXT: s_endpgm729entry:730 %tmp0 = atomicrmw volatile add ptr %out, i64 %in syncscope("agent") seq_cst731 ret void732}733 734define amdgpu_kernel void @atomic_add_i64_ret(ptr %out, ptr %out2, i64 %in) {735; GCN1-LABEL: atomic_add_i64_ret:736; GCN1: ; %bb.0: ; %entry737; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0738; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1739; GCN1-NEXT: s_mov_b32 s14, -1740; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f741; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9742; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd743; GCN1-NEXT: s_mov_b32 s15, 0xe8f000744; GCN1-NEXT: s_add_u32 s12, s12, s11745; GCN1-NEXT: s_addc_u32 s13, s13, 0746; GCN1-NEXT: s_waitcnt lgkmcnt(0)747; GCN1-NEXT: s_cmp_eq_u32 s1, s6748; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0749; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]750; GCN1-NEXT: s_cbranch_vccz .LBB5_2751; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global752; GCN1-NEXT: v_mov_b32_e32 v0, s0753; GCN1-NEXT: v_mov_b32_e32 v2, s4754; GCN1-NEXT: v_mov_b32_e32 v1, s1755; GCN1-NEXT: v_mov_b32_e32 v3, s5756; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc757; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)758; GCN1-NEXT: buffer_wbinvl1_vol759; GCN1-NEXT: s_cbranch_execz .LBB5_3760; GCN1-NEXT: s_branch .LBB5_4761; GCN1-NEXT: .LBB5_2:762; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1763; GCN1-NEXT: .LBB5_3: ; %atomicrmw.private764; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 0765; GCN1-NEXT: v_mov_b32_e32 v4, s5766; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec767; GCN1-NEXT: s_cselect_b32 s0, s0, -1768; GCN1-NEXT: v_mov_b32_e32 v2, s0769; GCN1-NEXT: s_add_i32 s0, s0, 4770; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen771; GCN1-NEXT: v_mov_b32_e32 v3, s0772; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen773; GCN1-NEXT: s_waitcnt vmcnt(1)774; GCN1-NEXT: v_add_i32_e32 v5, vcc, s4, v0775; GCN1-NEXT: s_waitcnt vmcnt(0)776; GCN1-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc777; GCN1-NEXT: buffer_store_dword v5, v2, s[12:15], 0 offen778; GCN1-NEXT: buffer_store_dword v4, v3, s[12:15], 0 offen779; GCN1-NEXT: .LBB5_4: ; %atomicrmw.end780; GCN1-NEXT: v_mov_b32_e32 v2, s2781; GCN1-NEXT: v_mov_b32_e32 v3, s3782; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]783; GCN1-NEXT: s_endpgm784;785; GCN2-LABEL: atomic_add_i64_ret:786; GCN2: ; %bb.0: ; %entry787; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0788; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1789; GCN2-NEXT: s_mov_b32 s90, -1790; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc791; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24792; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34793; GCN2-NEXT: s_mov_b32 s91, 0xe80000794; GCN2-NEXT: s_add_u32 s88, s88, s11795; GCN2-NEXT: s_addc_u32 s89, s89, 0796; GCN2-NEXT: s_waitcnt lgkmcnt(0)797; GCN2-NEXT: s_cmp_eq_u32 s1, s6798; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0799; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]800; GCN2-NEXT: s_cbranch_vccz .LBB5_2801; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global802; GCN2-NEXT: v_mov_b32_e32 v0, s0803; GCN2-NEXT: v_mov_b32_e32 v2, s4804; GCN2-NEXT: v_mov_b32_e32 v1, s1805; GCN2-NEXT: v_mov_b32_e32 v3, s5806; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc807; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)808; GCN2-NEXT: buffer_wbinvl1_vol809; GCN2-NEXT: s_cbranch_execz .LBB5_3810; GCN2-NEXT: s_branch .LBB5_4811; GCN2-NEXT: .LBB5_2:812; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1813; GCN2-NEXT: .LBB5_3: ; %atomicrmw.private814; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0815; GCN2-NEXT: s_cselect_b32 s0, s0, -1816; GCN2-NEXT: v_mov_b32_e32 v2, s0817; GCN2-NEXT: s_add_i32 s0, s0, 4818; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen819; GCN2-NEXT: v_mov_b32_e32 v3, s0820; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen821; GCN2-NEXT: v_mov_b32_e32 v4, s5822; GCN2-NEXT: s_waitcnt vmcnt(1)823; GCN2-NEXT: v_add_u32_e32 v5, vcc, s4, v0824; GCN2-NEXT: s_waitcnt vmcnt(0)825; GCN2-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc826; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen827; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen828; GCN2-NEXT: .LBB5_4: ; %atomicrmw.end829; GCN2-NEXT: v_mov_b32_e32 v2, s2830; GCN2-NEXT: v_mov_b32_e32 v3, s3831; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]832; GCN2-NEXT: s_endpgm833;834; GFX12-LABEL: atomic_add_i64_ret:835; GFX12: ; %bb.0: ; %entry836; GFX12-NEXT: s_clause 0x1837; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24838; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x34839; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base840; GFX12-NEXT: s_wait_kmcnt 0x0841; GFX12-NEXT: s_cmp_eq_u32 s1, s7842; GFX12-NEXT: s_cselect_b32 s6, -1, 0843; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)844; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6845; GFX12-NEXT: s_cbranch_vccz .LBB5_2846; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global847; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1848; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5849; GFX12-NEXT: flat_atomic_add_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV850; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0851; GFX12-NEXT: global_inv scope:SCOPE_DEV852; GFX12-NEXT: s_cbranch_execz .LBB5_3853; GFX12-NEXT: s_branch .LBB5_4854; GFX12-NEXT: .LBB5_2:855; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1856; GFX12-NEXT: .LBB5_3: ; %atomicrmw.private857; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 0858; GFX12-NEXT: s_cselect_b32 s0, s0, -1859; GFX12-NEXT: scratch_load_b64 v[0:1], off, s0860; GFX12-NEXT: s_wait_loadcnt 0x0861; GFX12-NEXT: v_add_co_u32 v2, vcc_lo, v0, s4862; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)863; GFX12-NEXT: v_add_co_ci_u32_e64 v3, null, s5, v1, vcc_lo864; GFX12-NEXT: scratch_store_b64 off, v[2:3], s0865; GFX12-NEXT: .LBB5_4: ; %atomicrmw.end866; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3867; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]868; GFX12-NEXT: s_endpgm869entry:870 %tmp0 = atomicrmw volatile add ptr %out, i64 %in syncscope("agent") seq_cst871 store i64 %tmp0, ptr %out2872 ret void873}874 875define amdgpu_kernel void @atomic_add_i64_addr64(ptr %out, i64 %in, i64 %index) {876; GCN1-LABEL: atomic_add_i64_addr64:877; GCN1: ; %bb.0: ; %entry878; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0879; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1880; GCN1-NEXT: s_mov_b32 s14, -1881; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd882; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9883; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f884; GCN1-NEXT: s_mov_b32 s15, 0xe8f000885; GCN1-NEXT: s_add_u32 s12, s12, s11886; GCN1-NEXT: s_addc_u32 s13, s13, 0887; GCN1-NEXT: s_waitcnt lgkmcnt(0)888; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 3889; GCN1-NEXT: s_add_u32 s0, s0, s4890; GCN1-NEXT: s_addc_u32 s1, s1, s5891; GCN1-NEXT: s_cmp_eq_u32 s1, s8892; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0893; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]894; GCN1-NEXT: s_mov_b64 s[4:5], -1895; GCN1-NEXT: s_cbranch_vccnz .LBB6_3896; GCN1-NEXT: ; %bb.1: ; %Flow897; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]898; GCN1-NEXT: s_cbranch_vccz .LBB6_4899; GCN1-NEXT: .LBB6_2: ; %atomicrmw.phi900; GCN1-NEXT: s_endpgm901; GCN1-NEXT: .LBB6_3: ; %atomicrmw.global902; GCN1-NEXT: v_mov_b32_e32 v0, s0903; GCN1-NEXT: v_mov_b32_e32 v2, s2904; GCN1-NEXT: v_mov_b32_e32 v1, s1905; GCN1-NEXT: v_mov_b32_e32 v3, s3906; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]907; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)908; GCN1-NEXT: buffer_wbinvl1_vol909; GCN1-NEXT: s_cbranch_execnz .LBB6_2910; GCN1-NEXT: .LBB6_4: ; %atomicrmw.private911; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0912; GCN1-NEXT: v_mov_b32_e32 v4, s3913; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec914; GCN1-NEXT: s_cselect_b32 s0, s0, -1915; GCN1-NEXT: v_mov_b32_e32 v0, s0916; GCN1-NEXT: s_add_i32 s0, s0, 4917; GCN1-NEXT: buffer_load_dword v1, v0, s[12:15], 0 offen918; GCN1-NEXT: v_mov_b32_e32 v2, s0919; GCN1-NEXT: buffer_load_dword v3, v2, s[12:15], 0 offen920; GCN1-NEXT: s_waitcnt vmcnt(1)921; GCN1-NEXT: v_add_i32_e32 v1, vcc, s2, v1922; GCN1-NEXT: s_waitcnt vmcnt(0)923; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc924; GCN1-NEXT: buffer_store_dword v1, v0, s[12:15], 0 offen925; GCN1-NEXT: buffer_store_dword v3, v2, s[12:15], 0 offen926; GCN1-NEXT: s_endpgm927;928; GCN2-LABEL: atomic_add_i64_addr64:929; GCN2: ; %bb.0: ; %entry930; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0931; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1932; GCN2-NEXT: s_mov_b32 s90, -1933; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34934; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24935; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc936; GCN2-NEXT: s_mov_b32 s91, 0xe80000937; GCN2-NEXT: s_add_u32 s88, s88, s11938; GCN2-NEXT: s_addc_u32 s89, s89, 0939; GCN2-NEXT: s_waitcnt lgkmcnt(0)940; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3941; GCN2-NEXT: s_add_u32 s0, s0, s4942; GCN2-NEXT: s_addc_u32 s1, s1, s5943; GCN2-NEXT: s_cmp_eq_u32 s1, s8944; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0945; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]946; GCN2-NEXT: s_mov_b64 s[4:5], -1947; GCN2-NEXT: s_cbranch_vccnz .LBB6_3948; GCN2-NEXT: ; %bb.1: ; %Flow949; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]950; GCN2-NEXT: s_cbranch_vccz .LBB6_4951; GCN2-NEXT: .LBB6_2: ; %atomicrmw.phi952; GCN2-NEXT: s_endpgm953; GCN2-NEXT: .LBB6_3: ; %atomicrmw.global954; GCN2-NEXT: v_mov_b32_e32 v0, s0955; GCN2-NEXT: v_mov_b32_e32 v2, s2956; GCN2-NEXT: v_mov_b32_e32 v1, s1957; GCN2-NEXT: v_mov_b32_e32 v3, s3958; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]959; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)960; GCN2-NEXT: buffer_wbinvl1_vol961; GCN2-NEXT: s_cbranch_execnz .LBB6_2962; GCN2-NEXT: .LBB6_4: ; %atomicrmw.private963; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0964; GCN2-NEXT: s_cselect_b32 s0, s0, -1965; GCN2-NEXT: v_mov_b32_e32 v0, s0966; GCN2-NEXT: s_add_i32 s0, s0, 4967; GCN2-NEXT: buffer_load_dword v1, v0, s[88:91], 0 offen968; GCN2-NEXT: v_mov_b32_e32 v2, s0969; GCN2-NEXT: buffer_load_dword v3, v2, s[88:91], 0 offen970; GCN2-NEXT: v_mov_b32_e32 v4, s3971; GCN2-NEXT: s_waitcnt vmcnt(1)972; GCN2-NEXT: v_add_u32_e32 v1, vcc, s2, v1973; GCN2-NEXT: s_waitcnt vmcnt(0)974; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc975; GCN2-NEXT: buffer_store_dword v1, v0, s[88:91], 0 offen976; GCN2-NEXT: buffer_store_dword v3, v2, s[88:91], 0 offen977; GCN2-NEXT: s_endpgm978;979; GFX12-LABEL: atomic_add_i64_addr64:980; GFX12: ; %bb.0: ; %entry981; GFX12-NEXT: s_clause 0x1982; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x34983; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24984; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base985; GFX12-NEXT: s_wait_kmcnt 0x0986; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 3987; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)988; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]989; GFX12-NEXT: s_cmp_eq_u32 s1, s7990; GFX12-NEXT: s_cselect_b32 s4, -1, 0991; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)992; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4993; GFX12-NEXT: s_mov_b32 s4, -1994; GFX12-NEXT: s_cbranch_vccnz .LBB6_3995; GFX12-NEXT: ; %bb.1: ; %Flow996; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4997; GFX12-NEXT: s_cbranch_vccz .LBB6_4998; GFX12-NEXT: .LBB6_2: ; %atomicrmw.phi999; GFX12-NEXT: s_endpgm1000; GFX12-NEXT: .LBB6_3: ; %atomicrmw.global1001; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11002; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s31003; GFX12-NEXT: flat_atomic_add_u64 v[0:1], v[2:3] scope:SCOPE_DEV1004; GFX12-NEXT: s_wait_storecnt_dscnt 0x01005; GFX12-NEXT: global_inv scope:SCOPE_DEV1006; GFX12-NEXT: s_cbranch_execnz .LBB6_21007; GFX12-NEXT: .LBB6_4: ; %atomicrmw.private1008; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 01009; GFX12-NEXT: s_cselect_b32 s0, s0, -11010; GFX12-NEXT: scratch_load_b64 v[0:1], off, s01011; GFX12-NEXT: s_wait_loadcnt 0x01012; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, s21013; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)1014; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo1015; GFX12-NEXT: scratch_store_b64 off, v[0:1], s01016; GFX12-NEXT: s_endpgm1017entry:1018 %ptr = getelementptr inbounds i64, ptr %out, i64 %index1019 %tmp0 = atomicrmw volatile add ptr %ptr, i64 %in syncscope("agent") seq_cst1020 ret void1021}1022 1023define amdgpu_kernel void @atomic_add_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {1024; GCN1-LABEL: atomic_add_i64_ret_addr64:1025; GCN1: ; %bb.0: ; %entry1026; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD01027; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD11028; GCN1-NEXT: s_mov_b32 s18, -11029; GCN1-NEXT: s_mov_b32 s19, 0xe8f0001030; GCN1-NEXT: s_add_u32 s16, s16, s111031; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x91032; GCN1-NEXT: s_load_dword s2, s[4:5], 0x411033; GCN1-NEXT: s_addc_u32 s17, s17, 01034; GCN1-NEXT: s_waitcnt lgkmcnt(0)1035; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 31036; GCN1-NEXT: s_add_u32 s0, s8, s01037; GCN1-NEXT: s_addc_u32 s1, s9, s11038; GCN1-NEXT: s_cmp_eq_u32 s1, s21039; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 01040; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]1041; GCN1-NEXT: s_cbranch_vccz .LBB7_21042; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global1043; GCN1-NEXT: v_mov_b32_e32 v0, s01044; GCN1-NEXT: v_mov_b32_e32 v2, s121045; GCN1-NEXT: v_mov_b32_e32 v1, s11046; GCN1-NEXT: v_mov_b32_e32 v3, s131047; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc1048; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1049; GCN1-NEXT: buffer_wbinvl1_vol1050; GCN1-NEXT: s_cbranch_execz .LBB7_31051; GCN1-NEXT: s_branch .LBB7_41052; GCN1-NEXT: .LBB7_2:1053; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr11054; GCN1-NEXT: .LBB7_3: ; %atomicrmw.private1055; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 01056; GCN1-NEXT: v_mov_b32_e32 v4, s131057; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec1058; GCN1-NEXT: s_cselect_b32 s0, s0, -11059; GCN1-NEXT: v_mov_b32_e32 v2, s01060; GCN1-NEXT: s_add_i32 s0, s0, 41061; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen1062; GCN1-NEXT: v_mov_b32_e32 v3, s01063; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen1064; GCN1-NEXT: s_waitcnt vmcnt(1)1065; GCN1-NEXT: v_add_i32_e32 v5, vcc, s12, v01066; GCN1-NEXT: s_waitcnt vmcnt(0)1067; GCN1-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc1068; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen1069; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen1070; GCN1-NEXT: .LBB7_4: ; %atomicrmw.end1071; GCN1-NEXT: v_mov_b32_e32 v2, s101072; GCN1-NEXT: v_mov_b32_e32 v3, s111073; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1074; GCN1-NEXT: s_endpgm1075;1076; GCN2-LABEL: atomic_add_i64_ret_addr64:1077; GCN2: ; %bb.0: ; %entry1078; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD01079; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD11080; GCN2-NEXT: s_mov_b32 s90, -11081; GCN2-NEXT: s_mov_b32 s91, 0xe800001082; GCN2-NEXT: s_add_u32 s88, s88, s111083; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x241084; GCN2-NEXT: s_load_dword s2, s[4:5], 0x1041085; GCN2-NEXT: s_addc_u32 s89, s89, 01086; GCN2-NEXT: s_waitcnt lgkmcnt(0)1087; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 31088; GCN2-NEXT: s_add_u32 s0, s8, s01089; GCN2-NEXT: s_addc_u32 s1, s9, s11090; GCN2-NEXT: s_cmp_eq_u32 s1, s21091; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 01092; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]1093; GCN2-NEXT: s_cbranch_vccz .LBB7_21094; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global1095; GCN2-NEXT: v_mov_b32_e32 v0, s01096; GCN2-NEXT: v_mov_b32_e32 v2, s121097; GCN2-NEXT: v_mov_b32_e32 v1, s11098; GCN2-NEXT: v_mov_b32_e32 v3, s131099; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc1100; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1101; GCN2-NEXT: buffer_wbinvl1_vol1102; GCN2-NEXT: s_cbranch_execz .LBB7_31103; GCN2-NEXT: s_branch .LBB7_41104; GCN2-NEXT: .LBB7_2:1105; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr11106; GCN2-NEXT: .LBB7_3: ; %atomicrmw.private1107; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 01108; GCN2-NEXT: s_cselect_b32 s0, s0, -11109; GCN2-NEXT: v_mov_b32_e32 v2, s01110; GCN2-NEXT: s_add_i32 s0, s0, 41111; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen1112; GCN2-NEXT: v_mov_b32_e32 v3, s01113; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen1114; GCN2-NEXT: v_mov_b32_e32 v4, s131115; GCN2-NEXT: s_waitcnt vmcnt(1)1116; GCN2-NEXT: v_add_u32_e32 v5, vcc, s12, v01117; GCN2-NEXT: s_waitcnt vmcnt(0)1118; GCN2-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc1119; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen1120; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen1121; GCN2-NEXT: .LBB7_4: ; %atomicrmw.end1122; GCN2-NEXT: v_mov_b32_e32 v2, s101123; GCN2-NEXT: v_mov_b32_e32 v3, s111124; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1125; GCN2-NEXT: s_endpgm1126;1127; GFX12-LABEL: atomic_add_i64_ret_addr64:1128; GFX12: ; %bb.0: ; %entry1129; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x241130; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base1131; GFX12-NEXT: s_wait_kmcnt 0x01132; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 31133; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)1134; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]1135; GFX12-NEXT: s_cmp_eq_u32 s1, s91136; GFX12-NEXT: s_cselect_b32 s6, -1, 01137; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1138; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s61139; GFX12-NEXT: s_cbranch_vccz .LBB7_21140; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global1141; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11142; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s51143; GFX12-NEXT: flat_atomic_add_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV1144; GFX12-NEXT: s_wait_loadcnt_dscnt 0x01145; GFX12-NEXT: global_inv scope:SCOPE_DEV1146; GFX12-NEXT: s_cbranch_execz .LBB7_31147; GFX12-NEXT: s_branch .LBB7_41148; GFX12-NEXT: .LBB7_2:1149; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr11150; GFX12-NEXT: .LBB7_3: ; %atomicrmw.private1151; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 01152; GFX12-NEXT: s_cselect_b32 s0, s0, -11153; GFX12-NEXT: scratch_load_b64 v[0:1], off, s01154; GFX12-NEXT: s_wait_loadcnt 0x01155; GFX12-NEXT: v_add_co_u32 v2, vcc_lo, v0, s41156; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)1157; GFX12-NEXT: v_add_co_ci_u32_e64 v3, null, s5, v1, vcc_lo1158; GFX12-NEXT: scratch_store_b64 off, v[2:3], s01159; GFX12-NEXT: .LBB7_4: ; %atomicrmw.end1160; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s31161; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]1162; GFX12-NEXT: s_endpgm1163entry:1164 %ptr = getelementptr inbounds i64, ptr %out, i64 %index1165 %tmp0 = atomicrmw volatile add ptr %ptr, i64 %in syncscope("agent") seq_cst1166 store i64 %tmp0, ptr %out21167 ret void1168}1169 1170define amdgpu_kernel void @atomic_and_i64_offset(ptr %out, i64 %in) {1171; GCN1-LABEL: atomic_and_i64_offset:1172; GCN1: ; %bb.0: ; %entry1173; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD01174; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD11175; GCN1-NEXT: s_mov_b32 s14, -11176; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91177; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d1178; GCN1-NEXT: s_mov_b32 s15, 0xe8f0001179; GCN1-NEXT: s_add_u32 s12, s12, s111180; GCN1-NEXT: s_addc_u32 s13, s13, 01181; GCN1-NEXT: s_waitcnt lgkmcnt(0)1182; GCN1-NEXT: s_add_u32 s0, s0, 321183; GCN1-NEXT: s_addc_u32 s1, s1, 01184; GCN1-NEXT: s_cmp_eq_u32 s1, s41185; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 01186; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]1187; GCN1-NEXT: s_mov_b64 s[4:5], -11188; GCN1-NEXT: s_cbranch_vccnz .LBB8_31189; GCN1-NEXT: ; %bb.1: ; %Flow1190; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]1191; GCN1-NEXT: s_cbranch_vccz .LBB8_41192; GCN1-NEXT: .LBB8_2: ; %atomicrmw.phi1193; GCN1-NEXT: s_endpgm1194; GCN1-NEXT: .LBB8_3: ; %atomicrmw.global1195; GCN1-NEXT: v_mov_b32_e32 v0, s01196; GCN1-NEXT: v_mov_b32_e32 v2, s21197; GCN1-NEXT: v_mov_b32_e32 v1, s11198; GCN1-NEXT: v_mov_b32_e32 v3, s31199; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]1200; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1201; GCN1-NEXT: buffer_wbinvl1_vol1202; GCN1-NEXT: s_cbranch_execnz .LBB8_21203; GCN1-NEXT: .LBB8_4: ; %atomicrmw.private1204; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 01205; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec1206; GCN1-NEXT: s_cselect_b32 s0, s0, -11207; GCN1-NEXT: v_mov_b32_e32 v0, s01208; GCN1-NEXT: s_add_i32 s0, s0, 41209; GCN1-NEXT: v_mov_b32_e32 v1, s01210; GCN1-NEXT: buffer_load_dword v2, v0, s[12:15], 0 offen1211; GCN1-NEXT: buffer_load_dword v3, v1, s[12:15], 0 offen1212; GCN1-NEXT: s_waitcnt vmcnt(1)1213; GCN1-NEXT: v_and_b32_e32 v2, s2, v21214; GCN1-NEXT: s_waitcnt vmcnt(0)1215; GCN1-NEXT: v_and_b32_e32 v3, s3, v31216; GCN1-NEXT: buffer_store_dword v2, v0, s[12:15], 0 offen1217; GCN1-NEXT: buffer_store_dword v3, v1, s[12:15], 0 offen1218; GCN1-NEXT: s_endpgm1219;1220; GCN2-LABEL: atomic_and_i64_offset:1221; GCN2: ; %bb.0: ; %entry1222; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD01223; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD11224; GCN2-NEXT: s_mov_b32 s90, -11225; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241226; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf41227; GCN2-NEXT: s_mov_b32 s91, 0xe800001228; GCN2-NEXT: s_add_u32 s88, s88, s111229; GCN2-NEXT: s_addc_u32 s89, s89, 01230; GCN2-NEXT: s_waitcnt lgkmcnt(0)1231; GCN2-NEXT: s_add_u32 s0, s0, 321232; GCN2-NEXT: s_addc_u32 s1, s1, 01233; GCN2-NEXT: s_cmp_eq_u32 s1, s41234; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 01235; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]1236; GCN2-NEXT: s_mov_b64 s[4:5], -11237; GCN2-NEXT: s_cbranch_vccnz .LBB8_31238; GCN2-NEXT: ; %bb.1: ; %Flow1239; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]1240; GCN2-NEXT: s_cbranch_vccz .LBB8_41241; GCN2-NEXT: .LBB8_2: ; %atomicrmw.phi1242; GCN2-NEXT: s_endpgm1243; GCN2-NEXT: .LBB8_3: ; %atomicrmw.global1244; GCN2-NEXT: v_mov_b32_e32 v0, s01245; GCN2-NEXT: v_mov_b32_e32 v2, s21246; GCN2-NEXT: v_mov_b32_e32 v1, s11247; GCN2-NEXT: v_mov_b32_e32 v3, s31248; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]1249; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1250; GCN2-NEXT: buffer_wbinvl1_vol1251; GCN2-NEXT: s_cbranch_execnz .LBB8_21252; GCN2-NEXT: .LBB8_4: ; %atomicrmw.private1253; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 01254; GCN2-NEXT: s_cselect_b32 s0, s0, -11255; GCN2-NEXT: v_mov_b32_e32 v0, s01256; GCN2-NEXT: s_add_i32 s0, s0, 41257; GCN2-NEXT: v_mov_b32_e32 v1, s01258; GCN2-NEXT: buffer_load_dword v2, v0, s[88:91], 0 offen1259; GCN2-NEXT: buffer_load_dword v3, v1, s[88:91], 0 offen1260; GCN2-NEXT: s_waitcnt vmcnt(1)1261; GCN2-NEXT: v_and_b32_e32 v2, s2, v21262; GCN2-NEXT: s_waitcnt vmcnt(0)1263; GCN2-NEXT: v_and_b32_e32 v3, s3, v31264; GCN2-NEXT: buffer_store_dword v2, v0, s[88:91], 0 offen1265; GCN2-NEXT: buffer_store_dword v3, v1, s[88:91], 0 offen1266; GCN2-NEXT: s_endpgm1267;1268; GFX12-LABEL: atomic_and_i64_offset:1269; GFX12: ; %bb.0: ; %entry1270; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x241271; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base1272; GFX12-NEXT: s_wait_kmcnt 0x01273; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 321274; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)1275; GFX12-NEXT: s_cmp_eq_u32 s1, s71276; GFX12-NEXT: s_cselect_b32 s4, -1, 01277; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s41278; GFX12-NEXT: s_mov_b32 s4, -11279; GFX12-NEXT: s_cbranch_vccnz .LBB8_31280; GFX12-NEXT: ; %bb.1: ; %Flow1281; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s41282; GFX12-NEXT: s_cbranch_vccz .LBB8_41283; GFX12-NEXT: .LBB8_2: ; %atomicrmw.phi1284; GFX12-NEXT: s_endpgm1285; GFX12-NEXT: .LBB8_3: ; %atomicrmw.global1286; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11287; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s31288; GFX12-NEXT: flat_atomic_and_b64 v[0:1], v[2:3] scope:SCOPE_DEV1289; GFX12-NEXT: s_wait_storecnt_dscnt 0x01290; GFX12-NEXT: global_inv scope:SCOPE_DEV1291; GFX12-NEXT: s_cbranch_execnz .LBB8_21292; GFX12-NEXT: .LBB8_4: ; %atomicrmw.private1293; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 01294; GFX12-NEXT: s_cselect_b32 s0, s0, -11295; GFX12-NEXT: scratch_load_b64 v[0:1], off, s01296; GFX12-NEXT: s_wait_loadcnt 0x01297; GFX12-NEXT: v_and_b32_e32 v1, s3, v11298; GFX12-NEXT: v_and_b32_e32 v0, s2, v01299; GFX12-NEXT: scratch_store_b64 off, v[0:1], s01300; GFX12-NEXT: s_endpgm1301entry:1302 %gep = getelementptr inbounds i64, ptr %out, i64 41303 %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01304 ret void1305}1306 1307define amdgpu_kernel void @atomic_and_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {1308; GCN1-LABEL: atomic_and_i64_ret_offset:1309; GCN1: ; %bb.0: ; %entry1310; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD01311; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD11312; GCN1-NEXT: s_mov_b32 s14, -11313; GCN1-NEXT: s_mov_b32 s15, 0xe8f0001314; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91315; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f1316; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd1317; GCN1-NEXT: s_add_u32 s12, s12, s111318; GCN1-NEXT: s_addc_u32 s13, s13, 01319; GCN1-NEXT: s_waitcnt lgkmcnt(0)1320; GCN1-NEXT: s_add_u32 s0, s0, 321321; GCN1-NEXT: s_addc_u32 s1, s1, 01322; GCN1-NEXT: s_cmp_eq_u32 s1, s61323; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 01324; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]1325; GCN1-NEXT: s_cbranch_vccz .LBB9_21326; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global1327; GCN1-NEXT: v_mov_b32_e32 v0, s01328; GCN1-NEXT: v_mov_b32_e32 v2, s41329; GCN1-NEXT: v_mov_b32_e32 v1, s11330; GCN1-NEXT: v_mov_b32_e32 v3, s51331; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc1332; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1333; GCN1-NEXT: buffer_wbinvl1_vol1334; GCN1-NEXT: s_cbranch_execz .LBB9_31335; GCN1-NEXT: s_branch .LBB9_41336; GCN1-NEXT: .LBB9_2:1337; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr11338; GCN1-NEXT: .LBB9_3: ; %atomicrmw.private1339; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 01340; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec1341; GCN1-NEXT: s_cselect_b32 s0, s0, -11342; GCN1-NEXT: v_mov_b32_e32 v2, s01343; GCN1-NEXT: s_add_i32 s0, s0, 41344; GCN1-NEXT: v_mov_b32_e32 v3, s01345; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen1346; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen1347; GCN1-NEXT: s_waitcnt vmcnt(1)1348; GCN1-NEXT: v_and_b32_e32 v4, s4, v01349; GCN1-NEXT: s_waitcnt vmcnt(0)1350; GCN1-NEXT: v_and_b32_e32 v5, s5, v11351; GCN1-NEXT: buffer_store_dword v4, v2, s[12:15], 0 offen1352; GCN1-NEXT: buffer_store_dword v5, v3, s[12:15], 0 offen1353; GCN1-NEXT: .LBB9_4: ; %atomicrmw.end1354; GCN1-NEXT: v_mov_b32_e32 v2, s21355; GCN1-NEXT: v_mov_b32_e32 v3, s31356; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1357; GCN1-NEXT: s_endpgm1358;1359; GCN2-LABEL: atomic_and_i64_ret_offset:1360; GCN2: ; %bb.0: ; %entry1361; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD01362; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD11363; GCN2-NEXT: s_mov_b32 s90, -11364; GCN2-NEXT: s_mov_b32 s91, 0xe800001365; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241366; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc1367; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341368; GCN2-NEXT: s_add_u32 s88, s88, s111369; GCN2-NEXT: s_addc_u32 s89, s89, 01370; GCN2-NEXT: s_waitcnt lgkmcnt(0)1371; GCN2-NEXT: s_add_u32 s0, s0, 321372; GCN2-NEXT: s_addc_u32 s1, s1, 01373; GCN2-NEXT: s_cmp_eq_u32 s1, s61374; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 01375; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]1376; GCN2-NEXT: s_cbranch_vccz .LBB9_21377; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global1378; GCN2-NEXT: v_mov_b32_e32 v0, s01379; GCN2-NEXT: v_mov_b32_e32 v2, s41380; GCN2-NEXT: v_mov_b32_e32 v1, s11381; GCN2-NEXT: v_mov_b32_e32 v3, s51382; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc1383; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1384; GCN2-NEXT: buffer_wbinvl1_vol1385; GCN2-NEXT: s_cbranch_execz .LBB9_31386; GCN2-NEXT: s_branch .LBB9_41387; GCN2-NEXT: .LBB9_2:1388; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr11389; GCN2-NEXT: .LBB9_3: ; %atomicrmw.private1390; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 01391; GCN2-NEXT: s_cselect_b32 s0, s0, -11392; GCN2-NEXT: v_mov_b32_e32 v2, s01393; GCN2-NEXT: s_add_i32 s0, s0, 41394; GCN2-NEXT: v_mov_b32_e32 v3, s01395; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen1396; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen1397; GCN2-NEXT: s_waitcnt vmcnt(1)1398; GCN2-NEXT: v_and_b32_e32 v4, s4, v01399; GCN2-NEXT: s_waitcnt vmcnt(0)1400; GCN2-NEXT: v_and_b32_e32 v5, s5, v11401; GCN2-NEXT: buffer_store_dword v4, v2, s[88:91], 0 offen1402; GCN2-NEXT: buffer_store_dword v5, v3, s[88:91], 0 offen1403; GCN2-NEXT: .LBB9_4: ; %atomicrmw.end1404; GCN2-NEXT: v_mov_b32_e32 v2, s21405; GCN2-NEXT: v_mov_b32_e32 v3, s31406; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1407; GCN2-NEXT: s_endpgm1408;1409; GFX12-LABEL: atomic_and_i64_ret_offset:1410; GFX12: ; %bb.0: ; %entry1411; GFX12-NEXT: s_clause 0x11412; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x241413; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x341414; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base1415; GFX12-NEXT: s_wait_kmcnt 0x01416; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 321417; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)1418; GFX12-NEXT: s_cmp_eq_u32 s1, s71419; GFX12-NEXT: s_cselect_b32 s6, -1, 01420; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s61421; GFX12-NEXT: s_cbranch_vccz .LBB9_21422; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global1423; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11424; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s51425; GFX12-NEXT: flat_atomic_and_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV1426; GFX12-NEXT: s_wait_loadcnt_dscnt 0x01427; GFX12-NEXT: global_inv scope:SCOPE_DEV1428; GFX12-NEXT: s_cbranch_execz .LBB9_31429; GFX12-NEXT: s_branch .LBB9_41430; GFX12-NEXT: .LBB9_2:1431; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr11432; GFX12-NEXT: .LBB9_3: ; %atomicrmw.private1433; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 01434; GFX12-NEXT: s_cselect_b32 s0, s0, -11435; GFX12-NEXT: scratch_load_b64 v[0:1], off, s01436; GFX12-NEXT: s_wait_loadcnt 0x01437; GFX12-NEXT: v_and_b32_e32 v3, s5, v11438; GFX12-NEXT: v_and_b32_e32 v2, s4, v01439; GFX12-NEXT: scratch_store_b64 off, v[2:3], s01440; GFX12-NEXT: .LBB9_4: ; %atomicrmw.end1441; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s31442; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]1443; GFX12-NEXT: s_endpgm1444entry:1445 %gep = getelementptr inbounds i64, ptr %out, i64 41446 %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01447 store i64 %tmp0, ptr %out21448 ret void1449}1450 1451define amdgpu_kernel void @atomic_and_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {1452; GCN1-LABEL: atomic_and_i64_addr64_offset:1453; GCN1: ; %bb.0: ; %entry1454; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD01455; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD11456; GCN1-NEXT: s_mov_b32 s14, -11457; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd1458; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91459; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f1460; GCN1-NEXT: s_mov_b32 s15, 0xe8f0001461; GCN1-NEXT: s_add_u32 s12, s12, s111462; GCN1-NEXT: s_addc_u32 s13, s13, 01463; GCN1-NEXT: s_waitcnt lgkmcnt(0)1464; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 31465; GCN1-NEXT: s_add_u32 s0, s0, s41466; GCN1-NEXT: s_addc_u32 s1, s1, s51467; GCN1-NEXT: s_add_u32 s0, s0, 321468; GCN1-NEXT: s_addc_u32 s1, s1, 01469; GCN1-NEXT: s_cmp_eq_u32 s1, s81470; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 01471; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]1472; GCN1-NEXT: s_mov_b64 s[4:5], -11473; GCN1-NEXT: s_cbranch_vccnz .LBB10_31474; GCN1-NEXT: ; %bb.1: ; %Flow1475; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]1476; GCN1-NEXT: s_cbranch_vccz .LBB10_41477; GCN1-NEXT: .LBB10_2: ; %atomicrmw.phi1478; GCN1-NEXT: s_endpgm1479; GCN1-NEXT: .LBB10_3: ; %atomicrmw.global1480; GCN1-NEXT: v_mov_b32_e32 v0, s01481; GCN1-NEXT: v_mov_b32_e32 v2, s21482; GCN1-NEXT: v_mov_b32_e32 v1, s11483; GCN1-NEXT: v_mov_b32_e32 v3, s31484; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]1485; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1486; GCN1-NEXT: buffer_wbinvl1_vol1487; GCN1-NEXT: s_cbranch_execnz .LBB10_21488; GCN1-NEXT: .LBB10_4: ; %atomicrmw.private1489; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 01490; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec1491; GCN1-NEXT: s_cselect_b32 s0, s0, -11492; GCN1-NEXT: v_mov_b32_e32 v0, s01493; GCN1-NEXT: s_add_i32 s0, s0, 41494; GCN1-NEXT: v_mov_b32_e32 v1, s01495; GCN1-NEXT: buffer_load_dword v2, v0, s[12:15], 0 offen1496; GCN1-NEXT: buffer_load_dword v3, v1, s[12:15], 0 offen1497; GCN1-NEXT: s_waitcnt vmcnt(1)1498; GCN1-NEXT: v_and_b32_e32 v2, s2, v21499; GCN1-NEXT: s_waitcnt vmcnt(0)1500; GCN1-NEXT: v_and_b32_e32 v3, s3, v31501; GCN1-NEXT: buffer_store_dword v2, v0, s[12:15], 0 offen1502; GCN1-NEXT: buffer_store_dword v3, v1, s[12:15], 0 offen1503; GCN1-NEXT: s_endpgm1504;1505; GCN2-LABEL: atomic_and_i64_addr64_offset:1506; GCN2: ; %bb.0: ; %entry1507; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD01508; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD11509; GCN2-NEXT: s_mov_b32 s90, -11510; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341511; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241512; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc1513; GCN2-NEXT: s_mov_b32 s91, 0xe800001514; GCN2-NEXT: s_add_u32 s88, s88, s111515; GCN2-NEXT: s_addc_u32 s89, s89, 01516; GCN2-NEXT: s_waitcnt lgkmcnt(0)1517; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 31518; GCN2-NEXT: s_add_u32 s0, s0, s41519; GCN2-NEXT: s_addc_u32 s1, s1, s51520; GCN2-NEXT: s_add_u32 s0, s0, 321521; GCN2-NEXT: s_addc_u32 s1, s1, 01522; GCN2-NEXT: s_cmp_eq_u32 s1, s81523; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 01524; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]1525; GCN2-NEXT: s_mov_b64 s[4:5], -11526; GCN2-NEXT: s_cbranch_vccnz .LBB10_31527; GCN2-NEXT: ; %bb.1: ; %Flow1528; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]1529; GCN2-NEXT: s_cbranch_vccz .LBB10_41530; GCN2-NEXT: .LBB10_2: ; %atomicrmw.phi1531; GCN2-NEXT: s_endpgm1532; GCN2-NEXT: .LBB10_3: ; %atomicrmw.global1533; GCN2-NEXT: v_mov_b32_e32 v0, s01534; GCN2-NEXT: v_mov_b32_e32 v2, s21535; GCN2-NEXT: v_mov_b32_e32 v1, s11536; GCN2-NEXT: v_mov_b32_e32 v3, s31537; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]1538; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1539; GCN2-NEXT: buffer_wbinvl1_vol1540; GCN2-NEXT: s_cbranch_execnz .LBB10_21541; GCN2-NEXT: .LBB10_4: ; %atomicrmw.private1542; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 01543; GCN2-NEXT: s_cselect_b32 s0, s0, -11544; GCN2-NEXT: v_mov_b32_e32 v0, s01545; GCN2-NEXT: s_add_i32 s0, s0, 41546; GCN2-NEXT: v_mov_b32_e32 v1, s01547; GCN2-NEXT: buffer_load_dword v2, v0, s[88:91], 0 offen1548; GCN2-NEXT: buffer_load_dword v3, v1, s[88:91], 0 offen1549; GCN2-NEXT: s_waitcnt vmcnt(1)1550; GCN2-NEXT: v_and_b32_e32 v2, s2, v21551; GCN2-NEXT: s_waitcnt vmcnt(0)1552; GCN2-NEXT: v_and_b32_e32 v3, s3, v31553; GCN2-NEXT: buffer_store_dword v2, v0, s[88:91], 0 offen1554; GCN2-NEXT: buffer_store_dword v3, v1, s[88:91], 0 offen1555; GCN2-NEXT: s_endpgm1556;1557; GFX12-LABEL: atomic_and_i64_addr64_offset:1558; GFX12: ; %bb.0: ; %entry1559; GFX12-NEXT: s_clause 0x11560; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x341561; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x241562; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base1563; GFX12-NEXT: s_wait_kmcnt 0x01564; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 31565; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)1566; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]1567; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 321568; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)1569; GFX12-NEXT: s_cmp_eq_u32 s1, s71570; GFX12-NEXT: s_cselect_b32 s4, -1, 01571; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s41572; GFX12-NEXT: s_mov_b32 s4, -11573; GFX12-NEXT: s_cbranch_vccnz .LBB10_31574; GFX12-NEXT: ; %bb.1: ; %Flow1575; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s41576; GFX12-NEXT: s_cbranch_vccz .LBB10_41577; GFX12-NEXT: .LBB10_2: ; %atomicrmw.phi1578; GFX12-NEXT: s_endpgm1579; GFX12-NEXT: .LBB10_3: ; %atomicrmw.global1580; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11581; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s31582; GFX12-NEXT: flat_atomic_and_b64 v[0:1], v[2:3] scope:SCOPE_DEV1583; GFX12-NEXT: s_wait_storecnt_dscnt 0x01584; GFX12-NEXT: global_inv scope:SCOPE_DEV1585; GFX12-NEXT: s_cbranch_execnz .LBB10_21586; GFX12-NEXT: .LBB10_4: ; %atomicrmw.private1587; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 01588; GFX12-NEXT: s_cselect_b32 s0, s0, -11589; GFX12-NEXT: scratch_load_b64 v[0:1], off, s01590; GFX12-NEXT: s_wait_loadcnt 0x01591; GFX12-NEXT: v_and_b32_e32 v1, s3, v11592; GFX12-NEXT: v_and_b32_e32 v0, s2, v01593; GFX12-NEXT: scratch_store_b64 off, v[0:1], s01594; GFX12-NEXT: s_endpgm1595entry:1596 %ptr = getelementptr inbounds i64, ptr %out, i64 %index1597 %gep = getelementptr inbounds i64, ptr %ptr, i64 41598 %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01599 ret void1600}1601 1602define amdgpu_kernel void @atomic_and_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {1603; GCN1-LABEL: atomic_and_i64_ret_addr64_offset:1604; GCN1: ; %bb.0: ; %entry1605; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD01606; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD11607; GCN1-NEXT: s_mov_b32 s18, -11608; GCN1-NEXT: s_mov_b32 s19, 0xe8f0001609; GCN1-NEXT: s_add_u32 s16, s16, s111610; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x91611; GCN1-NEXT: s_load_dword s2, s[4:5], 0x411612; GCN1-NEXT: s_addc_u32 s17, s17, 01613; GCN1-NEXT: s_waitcnt lgkmcnt(0)1614; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 31615; GCN1-NEXT: s_add_u32 s0, s8, s01616; GCN1-NEXT: s_addc_u32 s1, s9, s11617; GCN1-NEXT: s_add_u32 s0, s0, 321618; GCN1-NEXT: s_addc_u32 s1, s1, 01619; GCN1-NEXT: s_cmp_eq_u32 s1, s21620; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 01621; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]1622; GCN1-NEXT: s_cbranch_vccz .LBB11_21623; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global1624; GCN1-NEXT: v_mov_b32_e32 v0, s01625; GCN1-NEXT: v_mov_b32_e32 v2, s121626; GCN1-NEXT: v_mov_b32_e32 v1, s11627; GCN1-NEXT: v_mov_b32_e32 v3, s131628; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc1629; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1630; GCN1-NEXT: buffer_wbinvl1_vol1631; GCN1-NEXT: s_cbranch_execz .LBB11_31632; GCN1-NEXT: s_branch .LBB11_41633; GCN1-NEXT: .LBB11_2:1634; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr11635; GCN1-NEXT: .LBB11_3: ; %atomicrmw.private1636; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 01637; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec1638; GCN1-NEXT: s_cselect_b32 s0, s0, -11639; GCN1-NEXT: v_mov_b32_e32 v2, s01640; GCN1-NEXT: s_add_i32 s0, s0, 41641; GCN1-NEXT: v_mov_b32_e32 v3, s01642; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen1643; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen1644; GCN1-NEXT: s_waitcnt vmcnt(1)1645; GCN1-NEXT: v_and_b32_e32 v4, s12, v01646; GCN1-NEXT: s_waitcnt vmcnt(0)1647; GCN1-NEXT: v_and_b32_e32 v5, s13, v11648; GCN1-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen1649; GCN1-NEXT: buffer_store_dword v5, v3, s[16:19], 0 offen1650; GCN1-NEXT: .LBB11_4: ; %atomicrmw.end1651; GCN1-NEXT: v_mov_b32_e32 v2, s101652; GCN1-NEXT: v_mov_b32_e32 v3, s111653; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1654; GCN1-NEXT: s_endpgm1655;1656; GCN2-LABEL: atomic_and_i64_ret_addr64_offset:1657; GCN2: ; %bb.0: ; %entry1658; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD01659; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD11660; GCN2-NEXT: s_mov_b32 s90, -11661; GCN2-NEXT: s_mov_b32 s91, 0xe800001662; GCN2-NEXT: s_add_u32 s88, s88, s111663; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x241664; GCN2-NEXT: s_load_dword s2, s[4:5], 0x1041665; GCN2-NEXT: s_addc_u32 s89, s89, 01666; GCN2-NEXT: s_waitcnt lgkmcnt(0)1667; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 31668; GCN2-NEXT: s_add_u32 s0, s8, s01669; GCN2-NEXT: s_addc_u32 s1, s9, s11670; GCN2-NEXT: s_add_u32 s0, s0, 321671; GCN2-NEXT: s_addc_u32 s1, s1, 01672; GCN2-NEXT: s_cmp_eq_u32 s1, s21673; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 01674; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]1675; GCN2-NEXT: s_cbranch_vccz .LBB11_21676; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global1677; GCN2-NEXT: v_mov_b32_e32 v0, s01678; GCN2-NEXT: v_mov_b32_e32 v2, s121679; GCN2-NEXT: v_mov_b32_e32 v1, s11680; GCN2-NEXT: v_mov_b32_e32 v3, s131681; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc1682; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1683; GCN2-NEXT: buffer_wbinvl1_vol1684; GCN2-NEXT: s_cbranch_execz .LBB11_31685; GCN2-NEXT: s_branch .LBB11_41686; GCN2-NEXT: .LBB11_2:1687; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr11688; GCN2-NEXT: .LBB11_3: ; %atomicrmw.private1689; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 01690; GCN2-NEXT: s_cselect_b32 s0, s0, -11691; GCN2-NEXT: v_mov_b32_e32 v2, s01692; GCN2-NEXT: s_add_i32 s0, s0, 41693; GCN2-NEXT: v_mov_b32_e32 v3, s01694; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen1695; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen1696; GCN2-NEXT: s_waitcnt vmcnt(1)1697; GCN2-NEXT: v_and_b32_e32 v4, s12, v01698; GCN2-NEXT: s_waitcnt vmcnt(0)1699; GCN2-NEXT: v_and_b32_e32 v5, s13, v11700; GCN2-NEXT: buffer_store_dword v4, v2, s[88:91], 0 offen1701; GCN2-NEXT: buffer_store_dword v5, v3, s[88:91], 0 offen1702; GCN2-NEXT: .LBB11_4: ; %atomicrmw.end1703; GCN2-NEXT: v_mov_b32_e32 v2, s101704; GCN2-NEXT: v_mov_b32_e32 v3, s111705; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1706; GCN2-NEXT: s_endpgm1707;1708; GFX12-LABEL: atomic_and_i64_ret_addr64_offset:1709; GFX12: ; %bb.0: ; %entry1710; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x241711; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base1712; GFX12-NEXT: s_wait_kmcnt 0x01713; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 31714; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)1715; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]1716; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 321717; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)1718; GFX12-NEXT: s_cmp_eq_u32 s1, s91719; GFX12-NEXT: s_cselect_b32 s6, -1, 01720; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s61721; GFX12-NEXT: s_cbranch_vccz .LBB11_21722; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global1723; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11724; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s51725; GFX12-NEXT: flat_atomic_and_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV1726; GFX12-NEXT: s_wait_loadcnt_dscnt 0x01727; GFX12-NEXT: global_inv scope:SCOPE_DEV1728; GFX12-NEXT: s_cbranch_execz .LBB11_31729; GFX12-NEXT: s_branch .LBB11_41730; GFX12-NEXT: .LBB11_2:1731; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr11732; GFX12-NEXT: .LBB11_3: ; %atomicrmw.private1733; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 01734; GFX12-NEXT: s_cselect_b32 s0, s0, -11735; GFX12-NEXT: scratch_load_b64 v[0:1], off, s01736; GFX12-NEXT: s_wait_loadcnt 0x01737; GFX12-NEXT: v_and_b32_e32 v3, s5, v11738; GFX12-NEXT: v_and_b32_e32 v2, s4, v01739; GFX12-NEXT: scratch_store_b64 off, v[2:3], s01740; GFX12-NEXT: .LBB11_4: ; %atomicrmw.end1741; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s31742; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]1743; GFX12-NEXT: s_endpgm1744entry:1745 %ptr = getelementptr inbounds i64, ptr %out, i64 %index1746 %gep = getelementptr inbounds i64, ptr %ptr, i64 41747 %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01748 store i64 %tmp0, ptr %out21749 ret void1750}1751 1752define amdgpu_kernel void @atomic_and_i64(ptr %out, i64 %in) {1753; GCN1-LABEL: atomic_and_i64:1754; GCN1: ; %bb.0: ; %entry1755; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD01756; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD11757; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91758; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d1759; GCN1-NEXT: s_mov_b32 s14, -11760; GCN1-NEXT: s_mov_b32 s15, 0xe8f0001761; GCN1-NEXT: s_add_u32 s12, s12, s111762; GCN1-NEXT: s_addc_u32 s13, s13, 01763; GCN1-NEXT: s_waitcnt lgkmcnt(0)1764; GCN1-NEXT: s_cmp_eq_u32 s1, s41765; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 01766; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]1767; GCN1-NEXT: s_mov_b64 s[4:5], -11768; GCN1-NEXT: s_cbranch_vccnz .LBB12_31769; GCN1-NEXT: ; %bb.1: ; %Flow1770; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]1771; GCN1-NEXT: s_cbranch_vccz .LBB12_41772; GCN1-NEXT: .LBB12_2: ; %atomicrmw.phi1773; GCN1-NEXT: s_endpgm1774; GCN1-NEXT: .LBB12_3: ; %atomicrmw.global1775; GCN1-NEXT: v_mov_b32_e32 v0, s01776; GCN1-NEXT: v_mov_b32_e32 v2, s21777; GCN1-NEXT: v_mov_b32_e32 v1, s11778; GCN1-NEXT: v_mov_b32_e32 v3, s31779; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]1780; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1781; GCN1-NEXT: buffer_wbinvl1_vol1782; GCN1-NEXT: s_cbranch_execnz .LBB12_21783; GCN1-NEXT: .LBB12_4: ; %atomicrmw.private1784; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 01785; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec1786; GCN1-NEXT: s_cselect_b32 s0, s0, -11787; GCN1-NEXT: v_mov_b32_e32 v0, s01788; GCN1-NEXT: s_add_i32 s0, s0, 41789; GCN1-NEXT: v_mov_b32_e32 v1, s01790; GCN1-NEXT: buffer_load_dword v2, v0, s[12:15], 0 offen1791; GCN1-NEXT: buffer_load_dword v3, v1, s[12:15], 0 offen1792; GCN1-NEXT: s_waitcnt vmcnt(1)1793; GCN1-NEXT: v_and_b32_e32 v2, s2, v21794; GCN1-NEXT: s_waitcnt vmcnt(0)1795; GCN1-NEXT: v_and_b32_e32 v3, s3, v31796; GCN1-NEXT: buffer_store_dword v2, v0, s[12:15], 0 offen1797; GCN1-NEXT: buffer_store_dword v3, v1, s[12:15], 0 offen1798; GCN1-NEXT: s_endpgm1799;1800; GCN2-LABEL: atomic_and_i64:1801; GCN2: ; %bb.0: ; %entry1802; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD01803; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD11804; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241805; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf41806; GCN2-NEXT: s_mov_b32 s90, -11807; GCN2-NEXT: s_mov_b32 s91, 0xe800001808; GCN2-NEXT: s_add_u32 s88, s88, s111809; GCN2-NEXT: s_addc_u32 s89, s89, 01810; GCN2-NEXT: s_waitcnt lgkmcnt(0)1811; GCN2-NEXT: s_cmp_eq_u32 s1, s41812; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 01813; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]1814; GCN2-NEXT: s_mov_b64 s[4:5], -11815; GCN2-NEXT: s_cbranch_vccnz .LBB12_31816; GCN2-NEXT: ; %bb.1: ; %Flow1817; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]1818; GCN2-NEXT: s_cbranch_vccz .LBB12_41819; GCN2-NEXT: .LBB12_2: ; %atomicrmw.phi1820; GCN2-NEXT: s_endpgm1821; GCN2-NEXT: .LBB12_3: ; %atomicrmw.global1822; GCN2-NEXT: v_mov_b32_e32 v0, s01823; GCN2-NEXT: v_mov_b32_e32 v2, s21824; GCN2-NEXT: v_mov_b32_e32 v1, s11825; GCN2-NEXT: v_mov_b32_e32 v3, s31826; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]1827; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1828; GCN2-NEXT: buffer_wbinvl1_vol1829; GCN2-NEXT: s_cbranch_execnz .LBB12_21830; GCN2-NEXT: .LBB12_4: ; %atomicrmw.private1831; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 01832; GCN2-NEXT: s_cselect_b32 s0, s0, -11833; GCN2-NEXT: v_mov_b32_e32 v0, s01834; GCN2-NEXT: s_add_i32 s0, s0, 41835; GCN2-NEXT: v_mov_b32_e32 v1, s01836; GCN2-NEXT: buffer_load_dword v2, v0, s[88:91], 0 offen1837; GCN2-NEXT: buffer_load_dword v3, v1, s[88:91], 0 offen1838; GCN2-NEXT: s_waitcnt vmcnt(1)1839; GCN2-NEXT: v_and_b32_e32 v2, s2, v21840; GCN2-NEXT: s_waitcnt vmcnt(0)1841; GCN2-NEXT: v_and_b32_e32 v3, s3, v31842; GCN2-NEXT: buffer_store_dword v2, v0, s[88:91], 0 offen1843; GCN2-NEXT: buffer_store_dword v3, v1, s[88:91], 0 offen1844; GCN2-NEXT: s_endpgm1845;1846; GFX12-LABEL: atomic_and_i64:1847; GFX12: ; %bb.0: ; %entry1848; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x241849; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base1850; GFX12-NEXT: s_wait_kmcnt 0x01851; GFX12-NEXT: s_cmp_eq_u32 s1, s71852; GFX12-NEXT: s_cselect_b32 s4, -1, 01853; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1854; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s41855; GFX12-NEXT: s_mov_b32 s4, -11856; GFX12-NEXT: s_cbranch_vccnz .LBB12_31857; GFX12-NEXT: ; %bb.1: ; %Flow1858; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s41859; GFX12-NEXT: s_cbranch_vccz .LBB12_41860; GFX12-NEXT: .LBB12_2: ; %atomicrmw.phi1861; GFX12-NEXT: s_endpgm1862; GFX12-NEXT: .LBB12_3: ; %atomicrmw.global1863; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11864; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s31865; GFX12-NEXT: flat_atomic_and_b64 v[0:1], v[2:3] scope:SCOPE_DEV1866; GFX12-NEXT: s_wait_storecnt_dscnt 0x01867; GFX12-NEXT: global_inv scope:SCOPE_DEV1868; GFX12-NEXT: s_cbranch_execnz .LBB12_21869; GFX12-NEXT: .LBB12_4: ; %atomicrmw.private1870; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 01871; GFX12-NEXT: s_cselect_b32 s0, s0, -11872; GFX12-NEXT: scratch_load_b64 v[0:1], off, s01873; GFX12-NEXT: s_wait_loadcnt 0x01874; GFX12-NEXT: v_and_b32_e32 v1, s3, v11875; GFX12-NEXT: v_and_b32_e32 v0, s2, v01876; GFX12-NEXT: scratch_store_b64 off, v[0:1], s01877; GFX12-NEXT: s_endpgm1878entry:1879 %tmp0 = atomicrmw volatile and ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01880 ret void1881}1882 1883define amdgpu_kernel void @atomic_and_i64_ret(ptr %out, ptr %out2, i64 %in) {1884; GCN1-LABEL: atomic_and_i64_ret:1885; GCN1: ; %bb.0: ; %entry1886; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD01887; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD11888; GCN1-NEXT: s_mov_b32 s14, -11889; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f1890; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91891; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd1892; GCN1-NEXT: s_mov_b32 s15, 0xe8f0001893; GCN1-NEXT: s_add_u32 s12, s12, s111894; GCN1-NEXT: s_addc_u32 s13, s13, 01895; GCN1-NEXT: s_waitcnt lgkmcnt(0)1896; GCN1-NEXT: s_cmp_eq_u32 s1, s61897; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 01898; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]1899; GCN1-NEXT: s_cbranch_vccz .LBB13_21900; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global1901; GCN1-NEXT: v_mov_b32_e32 v0, s01902; GCN1-NEXT: v_mov_b32_e32 v2, s41903; GCN1-NEXT: v_mov_b32_e32 v1, s11904; GCN1-NEXT: v_mov_b32_e32 v3, s51905; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc1906; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1907; GCN1-NEXT: buffer_wbinvl1_vol1908; GCN1-NEXT: s_cbranch_execz .LBB13_31909; GCN1-NEXT: s_branch .LBB13_41910; GCN1-NEXT: .LBB13_2:1911; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr11912; GCN1-NEXT: .LBB13_3: ; %atomicrmw.private1913; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 01914; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec1915; GCN1-NEXT: s_cselect_b32 s0, s0, -11916; GCN1-NEXT: v_mov_b32_e32 v2, s01917; GCN1-NEXT: s_add_i32 s0, s0, 41918; GCN1-NEXT: v_mov_b32_e32 v3, s01919; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen1920; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen1921; GCN1-NEXT: s_waitcnt vmcnt(1)1922; GCN1-NEXT: v_and_b32_e32 v4, s4, v01923; GCN1-NEXT: s_waitcnt vmcnt(0)1924; GCN1-NEXT: v_and_b32_e32 v5, s5, v11925; GCN1-NEXT: buffer_store_dword v4, v2, s[12:15], 0 offen1926; GCN1-NEXT: buffer_store_dword v5, v3, s[12:15], 0 offen1927; GCN1-NEXT: .LBB13_4: ; %atomicrmw.end1928; GCN1-NEXT: v_mov_b32_e32 v2, s21929; GCN1-NEXT: v_mov_b32_e32 v3, s31930; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1931; GCN1-NEXT: s_endpgm1932;1933; GCN2-LABEL: atomic_and_i64_ret:1934; GCN2: ; %bb.0: ; %entry1935; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD01936; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD11937; GCN2-NEXT: s_mov_b32 s90, -11938; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc1939; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241940; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341941; GCN2-NEXT: s_mov_b32 s91, 0xe800001942; GCN2-NEXT: s_add_u32 s88, s88, s111943; GCN2-NEXT: s_addc_u32 s89, s89, 01944; GCN2-NEXT: s_waitcnt lgkmcnt(0)1945; GCN2-NEXT: s_cmp_eq_u32 s1, s61946; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 01947; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]1948; GCN2-NEXT: s_cbranch_vccz .LBB13_21949; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global1950; GCN2-NEXT: v_mov_b32_e32 v0, s01951; GCN2-NEXT: v_mov_b32_e32 v2, s41952; GCN2-NEXT: v_mov_b32_e32 v1, s11953; GCN2-NEXT: v_mov_b32_e32 v3, s51954; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc1955; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1956; GCN2-NEXT: buffer_wbinvl1_vol1957; GCN2-NEXT: s_cbranch_execz .LBB13_31958; GCN2-NEXT: s_branch .LBB13_41959; GCN2-NEXT: .LBB13_2:1960; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr11961; GCN2-NEXT: .LBB13_3: ; %atomicrmw.private1962; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 01963; GCN2-NEXT: s_cselect_b32 s0, s0, -11964; GCN2-NEXT: v_mov_b32_e32 v2, s01965; GCN2-NEXT: s_add_i32 s0, s0, 41966; GCN2-NEXT: v_mov_b32_e32 v3, s01967; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen1968; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen1969; GCN2-NEXT: s_waitcnt vmcnt(1)1970; GCN2-NEXT: v_and_b32_e32 v4, s4, v01971; GCN2-NEXT: s_waitcnt vmcnt(0)1972; GCN2-NEXT: v_and_b32_e32 v5, s5, v11973; GCN2-NEXT: buffer_store_dword v4, v2, s[88:91], 0 offen1974; GCN2-NEXT: buffer_store_dword v5, v3, s[88:91], 0 offen1975; GCN2-NEXT: .LBB13_4: ; %atomicrmw.end1976; GCN2-NEXT: v_mov_b32_e32 v2, s21977; GCN2-NEXT: v_mov_b32_e32 v3, s31978; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1979; GCN2-NEXT: s_endpgm1980;1981; GFX12-LABEL: atomic_and_i64_ret:1982; GFX12: ; %bb.0: ; %entry1983; GFX12-NEXT: s_clause 0x11984; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x241985; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x341986; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base1987; GFX12-NEXT: s_wait_kmcnt 0x01988; GFX12-NEXT: s_cmp_eq_u32 s1, s71989; GFX12-NEXT: s_cselect_b32 s6, -1, 01990; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1991; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s61992; GFX12-NEXT: s_cbranch_vccz .LBB13_21993; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global1994; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11995; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s51996; GFX12-NEXT: flat_atomic_and_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV1997; GFX12-NEXT: s_wait_loadcnt_dscnt 0x01998; GFX12-NEXT: global_inv scope:SCOPE_DEV1999; GFX12-NEXT: s_cbranch_execz .LBB13_32000; GFX12-NEXT: s_branch .LBB13_42001; GFX12-NEXT: .LBB13_2:2002; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr12003; GFX12-NEXT: .LBB13_3: ; %atomicrmw.private2004; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 02005; GFX12-NEXT: s_cselect_b32 s0, s0, -12006; GFX12-NEXT: scratch_load_b64 v[0:1], off, s02007; GFX12-NEXT: s_wait_loadcnt 0x02008; GFX12-NEXT: v_and_b32_e32 v3, s5, v12009; GFX12-NEXT: v_and_b32_e32 v2, s4, v02010; GFX12-NEXT: scratch_store_b64 off, v[2:3], s02011; GFX12-NEXT: .LBB13_4: ; %atomicrmw.end2012; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s32013; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]2014; GFX12-NEXT: s_endpgm2015entry:2016 %tmp0 = atomicrmw volatile and ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !02017 store i64 %tmp0, ptr %out22018 ret void2019}2020 2021define amdgpu_kernel void @atomic_and_i64_addr64(ptr %out, i64 %in, i64 %index) {2022; GCN1-LABEL: atomic_and_i64_addr64:2023; GCN1: ; %bb.0: ; %entry2024; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD02025; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD12026; GCN1-NEXT: s_mov_b32 s14, -12027; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd2028; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92029; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f2030; GCN1-NEXT: s_mov_b32 s15, 0xe8f0002031; GCN1-NEXT: s_add_u32 s12, s12, s112032; GCN1-NEXT: s_addc_u32 s13, s13, 02033; GCN1-NEXT: s_waitcnt lgkmcnt(0)2034; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 32035; GCN1-NEXT: s_add_u32 s0, s0, s42036; GCN1-NEXT: s_addc_u32 s1, s1, s52037; GCN1-NEXT: s_cmp_eq_u32 s1, s82038; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 02039; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]2040; GCN1-NEXT: s_mov_b64 s[4:5], -12041; GCN1-NEXT: s_cbranch_vccnz .LBB14_32042; GCN1-NEXT: ; %bb.1: ; %Flow2043; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]2044; GCN1-NEXT: s_cbranch_vccz .LBB14_42045; GCN1-NEXT: .LBB14_2: ; %atomicrmw.phi2046; GCN1-NEXT: s_endpgm2047; GCN1-NEXT: .LBB14_3: ; %atomicrmw.global2048; GCN1-NEXT: v_mov_b32_e32 v0, s02049; GCN1-NEXT: v_mov_b32_e32 v2, s22050; GCN1-NEXT: v_mov_b32_e32 v1, s12051; GCN1-NEXT: v_mov_b32_e32 v3, s32052; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]2053; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2054; GCN1-NEXT: buffer_wbinvl1_vol2055; GCN1-NEXT: s_cbranch_execnz .LBB14_22056; GCN1-NEXT: .LBB14_4: ; %atomicrmw.private2057; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 02058; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec2059; GCN1-NEXT: s_cselect_b32 s0, s0, -12060; GCN1-NEXT: v_mov_b32_e32 v0, s02061; GCN1-NEXT: s_add_i32 s0, s0, 42062; GCN1-NEXT: v_mov_b32_e32 v1, s02063; GCN1-NEXT: buffer_load_dword v2, v0, s[12:15], 0 offen2064; GCN1-NEXT: buffer_load_dword v3, v1, s[12:15], 0 offen2065; GCN1-NEXT: s_waitcnt vmcnt(1)2066; GCN1-NEXT: v_and_b32_e32 v2, s2, v22067; GCN1-NEXT: s_waitcnt vmcnt(0)2068; GCN1-NEXT: v_and_b32_e32 v3, s3, v32069; GCN1-NEXT: buffer_store_dword v2, v0, s[12:15], 0 offen2070; GCN1-NEXT: buffer_store_dword v3, v1, s[12:15], 0 offen2071; GCN1-NEXT: s_endpgm2072;2073; GCN2-LABEL: atomic_and_i64_addr64:2074; GCN2: ; %bb.0: ; %entry2075; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD02076; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD12077; GCN2-NEXT: s_mov_b32 s90, -12078; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342079; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242080; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc2081; GCN2-NEXT: s_mov_b32 s91, 0xe800002082; GCN2-NEXT: s_add_u32 s88, s88, s112083; GCN2-NEXT: s_addc_u32 s89, s89, 02084; GCN2-NEXT: s_waitcnt lgkmcnt(0)2085; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 32086; GCN2-NEXT: s_add_u32 s0, s0, s42087; GCN2-NEXT: s_addc_u32 s1, s1, s52088; GCN2-NEXT: s_cmp_eq_u32 s1, s82089; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 02090; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]2091; GCN2-NEXT: s_mov_b64 s[4:5], -12092; GCN2-NEXT: s_cbranch_vccnz .LBB14_32093; GCN2-NEXT: ; %bb.1: ; %Flow2094; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]2095; GCN2-NEXT: s_cbranch_vccz .LBB14_42096; GCN2-NEXT: .LBB14_2: ; %atomicrmw.phi2097; GCN2-NEXT: s_endpgm2098; GCN2-NEXT: .LBB14_3: ; %atomicrmw.global2099; GCN2-NEXT: v_mov_b32_e32 v0, s02100; GCN2-NEXT: v_mov_b32_e32 v2, s22101; GCN2-NEXT: v_mov_b32_e32 v1, s12102; GCN2-NEXT: v_mov_b32_e32 v3, s32103; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]2104; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2105; GCN2-NEXT: buffer_wbinvl1_vol2106; GCN2-NEXT: s_cbranch_execnz .LBB14_22107; GCN2-NEXT: .LBB14_4: ; %atomicrmw.private2108; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 02109; GCN2-NEXT: s_cselect_b32 s0, s0, -12110; GCN2-NEXT: v_mov_b32_e32 v0, s02111; GCN2-NEXT: s_add_i32 s0, s0, 42112; GCN2-NEXT: v_mov_b32_e32 v1, s02113; GCN2-NEXT: buffer_load_dword v2, v0, s[88:91], 0 offen2114; GCN2-NEXT: buffer_load_dword v3, v1, s[88:91], 0 offen2115; GCN2-NEXT: s_waitcnt vmcnt(1)2116; GCN2-NEXT: v_and_b32_e32 v2, s2, v22117; GCN2-NEXT: s_waitcnt vmcnt(0)2118; GCN2-NEXT: v_and_b32_e32 v3, s3, v32119; GCN2-NEXT: buffer_store_dword v2, v0, s[88:91], 0 offen2120; GCN2-NEXT: buffer_store_dword v3, v1, s[88:91], 0 offen2121; GCN2-NEXT: s_endpgm2122;2123; GFX12-LABEL: atomic_and_i64_addr64:2124; GFX12: ; %bb.0: ; %entry2125; GFX12-NEXT: s_clause 0x12126; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x342127; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x242128; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base2129; GFX12-NEXT: s_wait_kmcnt 0x02130; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 32131; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)2132; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]2133; GFX12-NEXT: s_cmp_eq_u32 s1, s72134; GFX12-NEXT: s_cselect_b32 s4, -1, 02135; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2136; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42137; GFX12-NEXT: s_mov_b32 s4, -12138; GFX12-NEXT: s_cbranch_vccnz .LBB14_32139; GFX12-NEXT: ; %bb.1: ; %Flow2140; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42141; GFX12-NEXT: s_cbranch_vccz .LBB14_42142; GFX12-NEXT: .LBB14_2: ; %atomicrmw.phi2143; GFX12-NEXT: s_endpgm2144; GFX12-NEXT: .LBB14_3: ; %atomicrmw.global2145; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12146; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s32147; GFX12-NEXT: flat_atomic_and_b64 v[0:1], v[2:3] scope:SCOPE_DEV2148; GFX12-NEXT: s_wait_storecnt_dscnt 0x02149; GFX12-NEXT: global_inv scope:SCOPE_DEV2150; GFX12-NEXT: s_cbranch_execnz .LBB14_22151; GFX12-NEXT: .LBB14_4: ; %atomicrmw.private2152; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 02153; GFX12-NEXT: s_cselect_b32 s0, s0, -12154; GFX12-NEXT: scratch_load_b64 v[0:1], off, s02155; GFX12-NEXT: s_wait_loadcnt 0x02156; GFX12-NEXT: v_and_b32_e32 v1, s3, v12157; GFX12-NEXT: v_and_b32_e32 v0, s2, v02158; GFX12-NEXT: scratch_store_b64 off, v[0:1], s02159; GFX12-NEXT: s_endpgm2160entry:2161 %ptr = getelementptr inbounds i64, ptr %out, i64 %index2162 %tmp0 = atomicrmw volatile and ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !02163 ret void2164}2165 2166define amdgpu_kernel void @atomic_and_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {2167; GCN1-LABEL: atomic_and_i64_ret_addr64:2168; GCN1: ; %bb.0: ; %entry2169; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD02170; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD12171; GCN1-NEXT: s_mov_b32 s18, -12172; GCN1-NEXT: s_mov_b32 s19, 0xe8f0002173; GCN1-NEXT: s_add_u32 s16, s16, s112174; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x92175; GCN1-NEXT: s_load_dword s2, s[4:5], 0x412176; GCN1-NEXT: s_addc_u32 s17, s17, 02177; GCN1-NEXT: s_waitcnt lgkmcnt(0)2178; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 32179; GCN1-NEXT: s_add_u32 s0, s8, s02180; GCN1-NEXT: s_addc_u32 s1, s9, s12181; GCN1-NEXT: s_cmp_eq_u32 s1, s22182; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 02183; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]2184; GCN1-NEXT: s_cbranch_vccz .LBB15_22185; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global2186; GCN1-NEXT: v_mov_b32_e32 v0, s02187; GCN1-NEXT: v_mov_b32_e32 v2, s122188; GCN1-NEXT: v_mov_b32_e32 v1, s12189; GCN1-NEXT: v_mov_b32_e32 v3, s132190; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc2191; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2192; GCN1-NEXT: buffer_wbinvl1_vol2193; GCN1-NEXT: s_cbranch_execz .LBB15_32194; GCN1-NEXT: s_branch .LBB15_42195; GCN1-NEXT: .LBB15_2:2196; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr12197; GCN1-NEXT: .LBB15_3: ; %atomicrmw.private2198; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 02199; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec2200; GCN1-NEXT: s_cselect_b32 s0, s0, -12201; GCN1-NEXT: v_mov_b32_e32 v2, s02202; GCN1-NEXT: s_add_i32 s0, s0, 42203; GCN1-NEXT: v_mov_b32_e32 v3, s02204; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen2205; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen2206; GCN1-NEXT: s_waitcnt vmcnt(1)2207; GCN1-NEXT: v_and_b32_e32 v4, s12, v02208; GCN1-NEXT: s_waitcnt vmcnt(0)2209; GCN1-NEXT: v_and_b32_e32 v5, s13, v12210; GCN1-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen2211; GCN1-NEXT: buffer_store_dword v5, v3, s[16:19], 0 offen2212; GCN1-NEXT: .LBB15_4: ; %atomicrmw.end2213; GCN1-NEXT: v_mov_b32_e32 v2, s102214; GCN1-NEXT: v_mov_b32_e32 v3, s112215; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]2216; GCN1-NEXT: s_endpgm2217;2218; GCN2-LABEL: atomic_and_i64_ret_addr64:2219; GCN2: ; %bb.0: ; %entry2220; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD02221; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD12222; GCN2-NEXT: s_mov_b32 s90, -12223; GCN2-NEXT: s_mov_b32 s91, 0xe800002224; GCN2-NEXT: s_add_u32 s88, s88, s112225; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x242226; GCN2-NEXT: s_load_dword s2, s[4:5], 0x1042227; GCN2-NEXT: s_addc_u32 s89, s89, 02228; GCN2-NEXT: s_waitcnt lgkmcnt(0)2229; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 32230; GCN2-NEXT: s_add_u32 s0, s8, s02231; GCN2-NEXT: s_addc_u32 s1, s9, s12232; GCN2-NEXT: s_cmp_eq_u32 s1, s22233; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 02234; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]2235; GCN2-NEXT: s_cbranch_vccz .LBB15_22236; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global2237; GCN2-NEXT: v_mov_b32_e32 v0, s02238; GCN2-NEXT: v_mov_b32_e32 v2, s122239; GCN2-NEXT: v_mov_b32_e32 v1, s12240; GCN2-NEXT: v_mov_b32_e32 v3, s132241; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc2242; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2243; GCN2-NEXT: buffer_wbinvl1_vol2244; GCN2-NEXT: s_cbranch_execz .LBB15_32245; GCN2-NEXT: s_branch .LBB15_42246; GCN2-NEXT: .LBB15_2:2247; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr12248; GCN2-NEXT: .LBB15_3: ; %atomicrmw.private2249; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 02250; GCN2-NEXT: s_cselect_b32 s0, s0, -12251; GCN2-NEXT: v_mov_b32_e32 v2, s02252; GCN2-NEXT: s_add_i32 s0, s0, 42253; GCN2-NEXT: v_mov_b32_e32 v3, s02254; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen2255; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen2256; GCN2-NEXT: s_waitcnt vmcnt(1)2257; GCN2-NEXT: v_and_b32_e32 v4, s12, v02258; GCN2-NEXT: s_waitcnt vmcnt(0)2259; GCN2-NEXT: v_and_b32_e32 v5, s13, v12260; GCN2-NEXT: buffer_store_dword v4, v2, s[88:91], 0 offen2261; GCN2-NEXT: buffer_store_dword v5, v3, s[88:91], 0 offen2262; GCN2-NEXT: .LBB15_4: ; %atomicrmw.end2263; GCN2-NEXT: v_mov_b32_e32 v2, s102264; GCN2-NEXT: v_mov_b32_e32 v3, s112265; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]2266; GCN2-NEXT: s_endpgm2267;2268; GFX12-LABEL: atomic_and_i64_ret_addr64:2269; GFX12: ; %bb.0: ; %entry2270; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x242271; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base2272; GFX12-NEXT: s_wait_kmcnt 0x02273; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 32274; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)2275; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]2276; GFX12-NEXT: s_cmp_eq_u32 s1, s92277; GFX12-NEXT: s_cselect_b32 s6, -1, 02278; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2279; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s62280; GFX12-NEXT: s_cbranch_vccz .LBB15_22281; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global2282; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12283; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s52284; GFX12-NEXT: flat_atomic_and_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV2285; GFX12-NEXT: s_wait_loadcnt_dscnt 0x02286; GFX12-NEXT: global_inv scope:SCOPE_DEV2287; GFX12-NEXT: s_cbranch_execz .LBB15_32288; GFX12-NEXT: s_branch .LBB15_42289; GFX12-NEXT: .LBB15_2:2290; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr12291; GFX12-NEXT: .LBB15_3: ; %atomicrmw.private2292; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 02293; GFX12-NEXT: s_cselect_b32 s0, s0, -12294; GFX12-NEXT: scratch_load_b64 v[0:1], off, s02295; GFX12-NEXT: s_wait_loadcnt 0x02296; GFX12-NEXT: v_and_b32_e32 v3, s5, v12297; GFX12-NEXT: v_and_b32_e32 v2, s4, v02298; GFX12-NEXT: scratch_store_b64 off, v[2:3], s02299; GFX12-NEXT: .LBB15_4: ; %atomicrmw.end2300; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s32301; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]2302; GFX12-NEXT: s_endpgm2303entry:2304 %ptr = getelementptr inbounds i64, ptr %out, i64 %index2305 %tmp0 = atomicrmw volatile and ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !02306 store i64 %tmp0, ptr %out22307 ret void2308}2309 2310define amdgpu_kernel void @atomic_sub_i64_offset(ptr %out, i64 %in) {2311; GCN1-LABEL: atomic_sub_i64_offset:2312; GCN1: ; %bb.0: ; %entry2313; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD02314; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD12315; GCN1-NEXT: s_mov_b32 s14, -12316; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92317; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d2318; GCN1-NEXT: s_mov_b32 s15, 0xe8f0002319; GCN1-NEXT: s_add_u32 s12, s12, s112320; GCN1-NEXT: s_addc_u32 s13, s13, 02321; GCN1-NEXT: s_waitcnt lgkmcnt(0)2322; GCN1-NEXT: s_add_u32 s0, s0, 322323; GCN1-NEXT: s_addc_u32 s1, s1, 02324; GCN1-NEXT: s_cmp_eq_u32 s1, s42325; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 02326; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]2327; GCN1-NEXT: s_mov_b64 s[4:5], -12328; GCN1-NEXT: s_cbranch_vccnz .LBB16_32329; GCN1-NEXT: ; %bb.1: ; %Flow2330; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]2331; GCN1-NEXT: s_cbranch_vccz .LBB16_42332; GCN1-NEXT: .LBB16_2: ; %atomicrmw.phi2333; GCN1-NEXT: s_endpgm2334; GCN1-NEXT: .LBB16_3: ; %atomicrmw.global2335; GCN1-NEXT: v_mov_b32_e32 v0, s02336; GCN1-NEXT: v_mov_b32_e32 v2, s22337; GCN1-NEXT: v_mov_b32_e32 v1, s12338; GCN1-NEXT: v_mov_b32_e32 v3, s32339; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]2340; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2341; GCN1-NEXT: buffer_wbinvl1_vol2342; GCN1-NEXT: s_cbranch_execnz .LBB16_22343; GCN1-NEXT: .LBB16_4: ; %atomicrmw.private2344; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 02345; GCN1-NEXT: v_mov_b32_e32 v4, s32346; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec2347; GCN1-NEXT: s_cselect_b32 s0, s0, -12348; GCN1-NEXT: v_mov_b32_e32 v0, s02349; GCN1-NEXT: s_add_i32 s0, s0, 42350; GCN1-NEXT: buffer_load_dword v1, v0, s[12:15], 0 offen2351; GCN1-NEXT: v_mov_b32_e32 v2, s02352; GCN1-NEXT: buffer_load_dword v3, v2, s[12:15], 0 offen2353; GCN1-NEXT: s_waitcnt vmcnt(1)2354; GCN1-NEXT: v_subrev_i32_e32 v1, vcc, s2, v12355; GCN1-NEXT: s_waitcnt vmcnt(0)2356; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc2357; GCN1-NEXT: buffer_store_dword v1, v0, s[12:15], 0 offen2358; GCN1-NEXT: buffer_store_dword v3, v2, s[12:15], 0 offen2359; GCN1-NEXT: s_endpgm2360;2361; GCN2-LABEL: atomic_sub_i64_offset:2362; GCN2: ; %bb.0: ; %entry2363; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD02364; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD12365; GCN2-NEXT: s_mov_b32 s90, -12366; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242367; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf42368; GCN2-NEXT: s_mov_b32 s91, 0xe800002369; GCN2-NEXT: s_add_u32 s88, s88, s112370; GCN2-NEXT: s_addc_u32 s89, s89, 02371; GCN2-NEXT: s_waitcnt lgkmcnt(0)2372; GCN2-NEXT: s_add_u32 s0, s0, 322373; GCN2-NEXT: s_addc_u32 s1, s1, 02374; GCN2-NEXT: s_cmp_eq_u32 s1, s42375; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 02376; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]2377; GCN2-NEXT: s_mov_b64 s[4:5], -12378; GCN2-NEXT: s_cbranch_vccnz .LBB16_32379; GCN2-NEXT: ; %bb.1: ; %Flow2380; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]2381; GCN2-NEXT: s_cbranch_vccz .LBB16_42382; GCN2-NEXT: .LBB16_2: ; %atomicrmw.phi2383; GCN2-NEXT: s_endpgm2384; GCN2-NEXT: .LBB16_3: ; %atomicrmw.global2385; GCN2-NEXT: v_mov_b32_e32 v0, s02386; GCN2-NEXT: v_mov_b32_e32 v2, s22387; GCN2-NEXT: v_mov_b32_e32 v1, s12388; GCN2-NEXT: v_mov_b32_e32 v3, s32389; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]2390; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2391; GCN2-NEXT: buffer_wbinvl1_vol2392; GCN2-NEXT: s_cbranch_execnz .LBB16_22393; GCN2-NEXT: .LBB16_4: ; %atomicrmw.private2394; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 02395; GCN2-NEXT: s_cselect_b32 s0, s0, -12396; GCN2-NEXT: v_mov_b32_e32 v0, s02397; GCN2-NEXT: s_add_i32 s0, s0, 42398; GCN2-NEXT: buffer_load_dword v1, v0, s[88:91], 0 offen2399; GCN2-NEXT: v_mov_b32_e32 v2, s02400; GCN2-NEXT: buffer_load_dword v3, v2, s[88:91], 0 offen2401; GCN2-NEXT: v_mov_b32_e32 v4, s32402; GCN2-NEXT: s_waitcnt vmcnt(1)2403; GCN2-NEXT: v_subrev_u32_e32 v1, vcc, s2, v12404; GCN2-NEXT: s_waitcnt vmcnt(0)2405; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc2406; GCN2-NEXT: buffer_store_dword v1, v0, s[88:91], 0 offen2407; GCN2-NEXT: buffer_store_dword v3, v2, s[88:91], 0 offen2408; GCN2-NEXT: s_endpgm2409;2410; GFX12-LABEL: atomic_sub_i64_offset:2411; GFX12: ; %bb.0: ; %entry2412; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x242413; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base2414; GFX12-NEXT: s_wait_kmcnt 0x02415; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 322416; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)2417; GFX12-NEXT: s_cmp_eq_u32 s1, s72418; GFX12-NEXT: s_cselect_b32 s4, -1, 02419; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42420; GFX12-NEXT: s_mov_b32 s4, -12421; GFX12-NEXT: s_cbranch_vccnz .LBB16_32422; GFX12-NEXT: ; %bb.1: ; %Flow2423; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42424; GFX12-NEXT: s_cbranch_vccz .LBB16_42425; GFX12-NEXT: .LBB16_2: ; %atomicrmw.phi2426; GFX12-NEXT: s_endpgm2427; GFX12-NEXT: .LBB16_3: ; %atomicrmw.global2428; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12429; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s32430; GFX12-NEXT: flat_atomic_sub_u64 v[0:1], v[2:3] scope:SCOPE_DEV2431; GFX12-NEXT: s_wait_storecnt_dscnt 0x02432; GFX12-NEXT: global_inv scope:SCOPE_DEV2433; GFX12-NEXT: s_cbranch_execnz .LBB16_22434; GFX12-NEXT: .LBB16_4: ; %atomicrmw.private2435; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 02436; GFX12-NEXT: s_cselect_b32 s0, s0, -12437; GFX12-NEXT: scratch_load_b64 v[0:1], off, s02438; GFX12-NEXT: s_wait_loadcnt 0x02439; GFX12-NEXT: v_sub_co_u32 v0, vcc_lo, v0, s22440; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)2441; GFX12-NEXT: v_subrev_co_ci_u32_e64 v1, null, s3, v1, vcc_lo2442; GFX12-NEXT: scratch_store_b64 off, v[0:1], s02443; GFX12-NEXT: s_endpgm2444entry:2445 %gep = getelementptr inbounds i64, ptr %out, i64 42446 %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !02447 ret void2448}2449 2450define amdgpu_kernel void @atomic_sub_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {2451; GCN1-LABEL: atomic_sub_i64_ret_offset:2452; GCN1: ; %bb.0: ; %entry2453; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD02454; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD12455; GCN1-NEXT: s_mov_b32 s14, -12456; GCN1-NEXT: s_mov_b32 s15, 0xe8f0002457; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92458; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f2459; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd2460; GCN1-NEXT: s_add_u32 s12, s12, s112461; GCN1-NEXT: s_addc_u32 s13, s13, 02462; GCN1-NEXT: s_waitcnt lgkmcnt(0)2463; GCN1-NEXT: s_add_u32 s0, s0, 322464; GCN1-NEXT: s_addc_u32 s1, s1, 02465; GCN1-NEXT: s_cmp_eq_u32 s1, s62466; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 02467; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]2468; GCN1-NEXT: s_cbranch_vccz .LBB17_22469; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global2470; GCN1-NEXT: v_mov_b32_e32 v0, s02471; GCN1-NEXT: v_mov_b32_e32 v2, s42472; GCN1-NEXT: v_mov_b32_e32 v1, s12473; GCN1-NEXT: v_mov_b32_e32 v3, s52474; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc2475; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2476; GCN1-NEXT: buffer_wbinvl1_vol2477; GCN1-NEXT: s_cbranch_execz .LBB17_32478; GCN1-NEXT: s_branch .LBB17_42479; GCN1-NEXT: .LBB17_2:2480; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr12481; GCN1-NEXT: .LBB17_3: ; %atomicrmw.private2482; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 02483; GCN1-NEXT: v_mov_b32_e32 v4, s52484; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec2485; GCN1-NEXT: s_cselect_b32 s0, s0, -12486; GCN1-NEXT: v_mov_b32_e32 v2, s02487; GCN1-NEXT: s_add_i32 s0, s0, 42488; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen2489; GCN1-NEXT: v_mov_b32_e32 v3, s02490; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen2491; GCN1-NEXT: s_waitcnt vmcnt(1)2492; GCN1-NEXT: v_subrev_i32_e32 v5, vcc, s4, v02493; GCN1-NEXT: s_waitcnt vmcnt(0)2494; GCN1-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc2495; GCN1-NEXT: buffer_store_dword v5, v2, s[12:15], 0 offen2496; GCN1-NEXT: buffer_store_dword v4, v3, s[12:15], 0 offen2497; GCN1-NEXT: .LBB17_4: ; %atomicrmw.end2498; GCN1-NEXT: v_mov_b32_e32 v2, s22499; GCN1-NEXT: v_mov_b32_e32 v3, s32500; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]2501; GCN1-NEXT: s_endpgm2502;2503; GCN2-LABEL: atomic_sub_i64_ret_offset:2504; GCN2: ; %bb.0: ; %entry2505; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD02506; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD12507; GCN2-NEXT: s_mov_b32 s90, -12508; GCN2-NEXT: s_mov_b32 s91, 0xe800002509; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242510; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc2511; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x342512; GCN2-NEXT: s_add_u32 s88, s88, s112513; GCN2-NEXT: s_addc_u32 s89, s89, 02514; GCN2-NEXT: s_waitcnt lgkmcnt(0)2515; GCN2-NEXT: s_add_u32 s0, s0, 322516; GCN2-NEXT: s_addc_u32 s1, s1, 02517; GCN2-NEXT: s_cmp_eq_u32 s1, s62518; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 02519; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]2520; GCN2-NEXT: s_cbranch_vccz .LBB17_22521; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global2522; GCN2-NEXT: v_mov_b32_e32 v0, s02523; GCN2-NEXT: v_mov_b32_e32 v2, s42524; GCN2-NEXT: v_mov_b32_e32 v1, s12525; GCN2-NEXT: v_mov_b32_e32 v3, s52526; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc2527; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2528; GCN2-NEXT: buffer_wbinvl1_vol2529; GCN2-NEXT: s_cbranch_execz .LBB17_32530; GCN2-NEXT: s_branch .LBB17_42531; GCN2-NEXT: .LBB17_2:2532; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr12533; GCN2-NEXT: .LBB17_3: ; %atomicrmw.private2534; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 02535; GCN2-NEXT: s_cselect_b32 s0, s0, -12536; GCN2-NEXT: v_mov_b32_e32 v2, s02537; GCN2-NEXT: s_add_i32 s0, s0, 42538; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen2539; GCN2-NEXT: v_mov_b32_e32 v3, s02540; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen2541; GCN2-NEXT: v_mov_b32_e32 v4, s52542; GCN2-NEXT: s_waitcnt vmcnt(1)2543; GCN2-NEXT: v_subrev_u32_e32 v5, vcc, s4, v02544; GCN2-NEXT: s_waitcnt vmcnt(0)2545; GCN2-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc2546; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen2547; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen2548; GCN2-NEXT: .LBB17_4: ; %atomicrmw.end2549; GCN2-NEXT: v_mov_b32_e32 v2, s22550; GCN2-NEXT: v_mov_b32_e32 v3, s32551; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]2552; GCN2-NEXT: s_endpgm2553;2554; GFX12-LABEL: atomic_sub_i64_ret_offset:2555; GFX12: ; %bb.0: ; %entry2556; GFX12-NEXT: s_clause 0x12557; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x242558; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x342559; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base2560; GFX12-NEXT: s_wait_kmcnt 0x02561; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 322562; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)2563; GFX12-NEXT: s_cmp_eq_u32 s1, s72564; GFX12-NEXT: s_cselect_b32 s6, -1, 02565; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s62566; GFX12-NEXT: s_cbranch_vccz .LBB17_22567; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global2568; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12569; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s52570; GFX12-NEXT: flat_atomic_sub_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV2571; GFX12-NEXT: s_wait_loadcnt_dscnt 0x02572; GFX12-NEXT: global_inv scope:SCOPE_DEV2573; GFX12-NEXT: s_cbranch_execz .LBB17_32574; GFX12-NEXT: s_branch .LBB17_42575; GFX12-NEXT: .LBB17_2:2576; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr12577; GFX12-NEXT: .LBB17_3: ; %atomicrmw.private2578; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 02579; GFX12-NEXT: s_cselect_b32 s0, s0, -12580; GFX12-NEXT: scratch_load_b64 v[0:1], off, s02581; GFX12-NEXT: s_wait_loadcnt 0x02582; GFX12-NEXT: v_sub_co_u32 v2, vcc_lo, v0, s42583; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)2584; GFX12-NEXT: v_subrev_co_ci_u32_e64 v3, null, s5, v1, vcc_lo2585; GFX12-NEXT: scratch_store_b64 off, v[2:3], s02586; GFX12-NEXT: .LBB17_4: ; %atomicrmw.end2587; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s32588; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]2589; GFX12-NEXT: s_endpgm2590entry:2591 %gep = getelementptr inbounds i64, ptr %out, i64 42592 %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !02593 store i64 %tmp0, ptr %out22594 ret void2595}2596 2597define amdgpu_kernel void @atomic_sub_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {2598; GCN1-LABEL: atomic_sub_i64_addr64_offset:2599; GCN1: ; %bb.0: ; %entry2600; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD02601; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD12602; GCN1-NEXT: s_mov_b32 s14, -12603; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd2604; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92605; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f2606; GCN1-NEXT: s_mov_b32 s15, 0xe8f0002607; GCN1-NEXT: s_add_u32 s12, s12, s112608; GCN1-NEXT: s_addc_u32 s13, s13, 02609; GCN1-NEXT: s_waitcnt lgkmcnt(0)2610; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 32611; GCN1-NEXT: s_add_u32 s0, s0, s42612; GCN1-NEXT: s_addc_u32 s1, s1, s52613; GCN1-NEXT: s_add_u32 s0, s0, 322614; GCN1-NEXT: s_addc_u32 s1, s1, 02615; GCN1-NEXT: s_cmp_eq_u32 s1, s82616; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 02617; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]2618; GCN1-NEXT: s_mov_b64 s[4:5], -12619; GCN1-NEXT: s_cbranch_vccnz .LBB18_32620; GCN1-NEXT: ; %bb.1: ; %Flow2621; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]2622; GCN1-NEXT: s_cbranch_vccz .LBB18_42623; GCN1-NEXT: .LBB18_2: ; %atomicrmw.phi2624; GCN1-NEXT: s_endpgm2625; GCN1-NEXT: .LBB18_3: ; %atomicrmw.global2626; GCN1-NEXT: v_mov_b32_e32 v0, s02627; GCN1-NEXT: v_mov_b32_e32 v2, s22628; GCN1-NEXT: v_mov_b32_e32 v1, s12629; GCN1-NEXT: v_mov_b32_e32 v3, s32630; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]2631; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2632; GCN1-NEXT: buffer_wbinvl1_vol2633; GCN1-NEXT: s_cbranch_execnz .LBB18_22634; GCN1-NEXT: .LBB18_4: ; %atomicrmw.private2635; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 02636; GCN1-NEXT: v_mov_b32_e32 v4, s32637; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec2638; GCN1-NEXT: s_cselect_b32 s0, s0, -12639; GCN1-NEXT: v_mov_b32_e32 v0, s02640; GCN1-NEXT: s_add_i32 s0, s0, 42641; GCN1-NEXT: buffer_load_dword v1, v0, s[12:15], 0 offen2642; GCN1-NEXT: v_mov_b32_e32 v2, s02643; GCN1-NEXT: buffer_load_dword v3, v2, s[12:15], 0 offen2644; GCN1-NEXT: s_waitcnt vmcnt(1)2645; GCN1-NEXT: v_subrev_i32_e32 v1, vcc, s2, v12646; GCN1-NEXT: s_waitcnt vmcnt(0)2647; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc2648; GCN1-NEXT: buffer_store_dword v1, v0, s[12:15], 0 offen2649; GCN1-NEXT: buffer_store_dword v3, v2, s[12:15], 0 offen2650; GCN1-NEXT: s_endpgm2651;2652; GCN2-LABEL: atomic_sub_i64_addr64_offset:2653; GCN2: ; %bb.0: ; %entry2654; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD02655; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD12656; GCN2-NEXT: s_mov_b32 s90, -12657; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342658; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242659; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc2660; GCN2-NEXT: s_mov_b32 s91, 0xe800002661; GCN2-NEXT: s_add_u32 s88, s88, s112662; GCN2-NEXT: s_addc_u32 s89, s89, 02663; GCN2-NEXT: s_waitcnt lgkmcnt(0)2664; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 32665; GCN2-NEXT: s_add_u32 s0, s0, s42666; GCN2-NEXT: s_addc_u32 s1, s1, s52667; GCN2-NEXT: s_add_u32 s0, s0, 322668; GCN2-NEXT: s_addc_u32 s1, s1, 02669; GCN2-NEXT: s_cmp_eq_u32 s1, s82670; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 02671; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]2672; GCN2-NEXT: s_mov_b64 s[4:5], -12673; GCN2-NEXT: s_cbranch_vccnz .LBB18_32674; GCN2-NEXT: ; %bb.1: ; %Flow2675; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]2676; GCN2-NEXT: s_cbranch_vccz .LBB18_42677; GCN2-NEXT: .LBB18_2: ; %atomicrmw.phi2678; GCN2-NEXT: s_endpgm2679; GCN2-NEXT: .LBB18_3: ; %atomicrmw.global2680; GCN2-NEXT: v_mov_b32_e32 v0, s02681; GCN2-NEXT: v_mov_b32_e32 v2, s22682; GCN2-NEXT: v_mov_b32_e32 v1, s12683; GCN2-NEXT: v_mov_b32_e32 v3, s32684; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]2685; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2686; GCN2-NEXT: buffer_wbinvl1_vol2687; GCN2-NEXT: s_cbranch_execnz .LBB18_22688; GCN2-NEXT: .LBB18_4: ; %atomicrmw.private2689; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 02690; GCN2-NEXT: s_cselect_b32 s0, s0, -12691; GCN2-NEXT: v_mov_b32_e32 v0, s02692; GCN2-NEXT: s_add_i32 s0, s0, 42693; GCN2-NEXT: buffer_load_dword v1, v0, s[88:91], 0 offen2694; GCN2-NEXT: v_mov_b32_e32 v2, s02695; GCN2-NEXT: buffer_load_dword v3, v2, s[88:91], 0 offen2696; GCN2-NEXT: v_mov_b32_e32 v4, s32697; GCN2-NEXT: s_waitcnt vmcnt(1)2698; GCN2-NEXT: v_subrev_u32_e32 v1, vcc, s2, v12699; GCN2-NEXT: s_waitcnt vmcnt(0)2700; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc2701; GCN2-NEXT: buffer_store_dword v1, v0, s[88:91], 0 offen2702; GCN2-NEXT: buffer_store_dword v3, v2, s[88:91], 0 offen2703; GCN2-NEXT: s_endpgm2704;2705; GFX12-LABEL: atomic_sub_i64_addr64_offset:2706; GFX12: ; %bb.0: ; %entry2707; GFX12-NEXT: s_clause 0x12708; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x342709; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x242710; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base2711; GFX12-NEXT: s_wait_kmcnt 0x02712; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 32713; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)2714; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]2715; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 322716; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)2717; GFX12-NEXT: s_cmp_eq_u32 s1, s72718; GFX12-NEXT: s_cselect_b32 s4, -1, 02719; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42720; GFX12-NEXT: s_mov_b32 s4, -12721; GFX12-NEXT: s_cbranch_vccnz .LBB18_32722; GFX12-NEXT: ; %bb.1: ; %Flow2723; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s42724; GFX12-NEXT: s_cbranch_vccz .LBB18_42725; GFX12-NEXT: .LBB18_2: ; %atomicrmw.phi2726; GFX12-NEXT: s_endpgm2727; GFX12-NEXT: .LBB18_3: ; %atomicrmw.global2728; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12729; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s32730; GFX12-NEXT: flat_atomic_sub_u64 v[0:1], v[2:3] scope:SCOPE_DEV2731; GFX12-NEXT: s_wait_storecnt_dscnt 0x02732; GFX12-NEXT: global_inv scope:SCOPE_DEV2733; GFX12-NEXT: s_cbranch_execnz .LBB18_22734; GFX12-NEXT: .LBB18_4: ; %atomicrmw.private2735; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 02736; GFX12-NEXT: s_cselect_b32 s0, s0, -12737; GFX12-NEXT: scratch_load_b64 v[0:1], off, s02738; GFX12-NEXT: s_wait_loadcnt 0x02739; GFX12-NEXT: v_sub_co_u32 v0, vcc_lo, v0, s22740; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)2741; GFX12-NEXT: v_subrev_co_ci_u32_e64 v1, null, s3, v1, vcc_lo2742; GFX12-NEXT: scratch_store_b64 off, v[0:1], s02743; GFX12-NEXT: s_endpgm2744entry:2745 %ptr = getelementptr inbounds i64, ptr %out, i64 %index2746 %gep = getelementptr inbounds i64, ptr %ptr, i64 42747 %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !02748 ret void2749}2750 2751define amdgpu_kernel void @atomic_sub_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {2752; GCN1-LABEL: atomic_sub_i64_ret_addr64_offset:2753; GCN1: ; %bb.0: ; %entry2754; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD02755; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD12756; GCN1-NEXT: s_mov_b32 s18, -12757; GCN1-NEXT: s_mov_b32 s19, 0xe8f0002758; GCN1-NEXT: s_add_u32 s16, s16, s112759; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x92760; GCN1-NEXT: s_load_dword s2, s[4:5], 0x412761; GCN1-NEXT: s_addc_u32 s17, s17, 02762; GCN1-NEXT: s_waitcnt lgkmcnt(0)2763; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 32764; GCN1-NEXT: s_add_u32 s0, s8, s02765; GCN1-NEXT: s_addc_u32 s1, s9, s12766; GCN1-NEXT: s_add_u32 s0, s0, 322767; GCN1-NEXT: s_addc_u32 s1, s1, 02768; GCN1-NEXT: s_cmp_eq_u32 s1, s22769; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 02770; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]2771; GCN1-NEXT: s_cbranch_vccz .LBB19_22772; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global2773; GCN1-NEXT: v_mov_b32_e32 v0, s02774; GCN1-NEXT: v_mov_b32_e32 v2, s122775; GCN1-NEXT: v_mov_b32_e32 v1, s12776; GCN1-NEXT: v_mov_b32_e32 v3, s132777; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc2778; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2779; GCN1-NEXT: buffer_wbinvl1_vol2780; GCN1-NEXT: s_cbranch_execz .LBB19_32781; GCN1-NEXT: s_branch .LBB19_42782; GCN1-NEXT: .LBB19_2:2783; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr12784; GCN1-NEXT: .LBB19_3: ; %atomicrmw.private2785; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 02786; GCN1-NEXT: v_mov_b32_e32 v4, s132787; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec2788; GCN1-NEXT: s_cselect_b32 s0, s0, -12789; GCN1-NEXT: v_mov_b32_e32 v2, s02790; GCN1-NEXT: s_add_i32 s0, s0, 42791; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen2792; GCN1-NEXT: v_mov_b32_e32 v3, s02793; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen2794; GCN1-NEXT: s_waitcnt vmcnt(1)2795; GCN1-NEXT: v_subrev_i32_e32 v5, vcc, s12, v02796; GCN1-NEXT: s_waitcnt vmcnt(0)2797; GCN1-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc2798; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen2799; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen2800; GCN1-NEXT: .LBB19_4: ; %atomicrmw.end2801; GCN1-NEXT: v_mov_b32_e32 v2, s102802; GCN1-NEXT: v_mov_b32_e32 v3, s112803; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]2804; GCN1-NEXT: s_endpgm2805;2806; GCN2-LABEL: atomic_sub_i64_ret_addr64_offset:2807; GCN2: ; %bb.0: ; %entry2808; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD02809; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD12810; GCN2-NEXT: s_mov_b32 s90, -12811; GCN2-NEXT: s_mov_b32 s91, 0xe800002812; GCN2-NEXT: s_add_u32 s88, s88, s112813; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x242814; GCN2-NEXT: s_load_dword s2, s[4:5], 0x1042815; GCN2-NEXT: s_addc_u32 s89, s89, 02816; GCN2-NEXT: s_waitcnt lgkmcnt(0)2817; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 32818; GCN2-NEXT: s_add_u32 s0, s8, s02819; GCN2-NEXT: s_addc_u32 s1, s9, s12820; GCN2-NEXT: s_add_u32 s0, s0, 322821; GCN2-NEXT: s_addc_u32 s1, s1, 02822; GCN2-NEXT: s_cmp_eq_u32 s1, s22823; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 02824; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]2825; GCN2-NEXT: s_cbranch_vccz .LBB19_22826; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global2827; GCN2-NEXT: v_mov_b32_e32 v0, s02828; GCN2-NEXT: v_mov_b32_e32 v2, s122829; GCN2-NEXT: v_mov_b32_e32 v1, s12830; GCN2-NEXT: v_mov_b32_e32 v3, s132831; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc2832; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2833; GCN2-NEXT: buffer_wbinvl1_vol2834; GCN2-NEXT: s_cbranch_execz .LBB19_32835; GCN2-NEXT: s_branch .LBB19_42836; GCN2-NEXT: .LBB19_2:2837; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr12838; GCN2-NEXT: .LBB19_3: ; %atomicrmw.private2839; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 02840; GCN2-NEXT: s_cselect_b32 s0, s0, -12841; GCN2-NEXT: v_mov_b32_e32 v2, s02842; GCN2-NEXT: s_add_i32 s0, s0, 42843; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen2844; GCN2-NEXT: v_mov_b32_e32 v3, s02845; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen2846; GCN2-NEXT: v_mov_b32_e32 v4, s132847; GCN2-NEXT: s_waitcnt vmcnt(1)2848; GCN2-NEXT: v_subrev_u32_e32 v5, vcc, s12, v02849; GCN2-NEXT: s_waitcnt vmcnt(0)2850; GCN2-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc2851; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen2852; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen2853; GCN2-NEXT: .LBB19_4: ; %atomicrmw.end2854; GCN2-NEXT: v_mov_b32_e32 v2, s102855; GCN2-NEXT: v_mov_b32_e32 v3, s112856; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]2857; GCN2-NEXT: s_endpgm2858;2859; GFX12-LABEL: atomic_sub_i64_ret_addr64_offset:2860; GFX12: ; %bb.0: ; %entry2861; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x242862; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base2863; GFX12-NEXT: s_wait_kmcnt 0x02864; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 32865; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)2866; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]2867; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 322868; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)2869; GFX12-NEXT: s_cmp_eq_u32 s1, s92870; GFX12-NEXT: s_cselect_b32 s6, -1, 02871; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s62872; GFX12-NEXT: s_cbranch_vccz .LBB19_22873; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global2874; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12875; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s52876; GFX12-NEXT: flat_atomic_sub_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV2877; GFX12-NEXT: s_wait_loadcnt_dscnt 0x02878; GFX12-NEXT: global_inv scope:SCOPE_DEV2879; GFX12-NEXT: s_cbranch_execz .LBB19_32880; GFX12-NEXT: s_branch .LBB19_42881; GFX12-NEXT: .LBB19_2:2882; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr12883; GFX12-NEXT: .LBB19_3: ; %atomicrmw.private2884; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 02885; GFX12-NEXT: s_cselect_b32 s0, s0, -12886; GFX12-NEXT: scratch_load_b64 v[0:1], off, s02887; GFX12-NEXT: s_wait_loadcnt 0x02888; GFX12-NEXT: v_sub_co_u32 v2, vcc_lo, v0, s42889; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)2890; GFX12-NEXT: v_subrev_co_ci_u32_e64 v3, null, s5, v1, vcc_lo2891; GFX12-NEXT: scratch_store_b64 off, v[2:3], s02892; GFX12-NEXT: .LBB19_4: ; %atomicrmw.end2893; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s32894; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]2895; GFX12-NEXT: s_endpgm2896entry:2897 %ptr = getelementptr inbounds i64, ptr %out, i64 %index2898 %gep = getelementptr inbounds i64, ptr %ptr, i64 42899 %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !02900 store i64 %tmp0, ptr %out22901 ret void2902}2903 2904define amdgpu_kernel void @atomic_sub_i64(ptr %out, i64 %in) {2905; GCN1-LABEL: atomic_sub_i64:2906; GCN1: ; %bb.0: ; %entry2907; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD02908; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD12909; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92910; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d2911; GCN1-NEXT: s_mov_b32 s14, -12912; GCN1-NEXT: s_mov_b32 s15, 0xe8f0002913; GCN1-NEXT: s_add_u32 s12, s12, s112914; GCN1-NEXT: s_addc_u32 s13, s13, 02915; GCN1-NEXT: s_waitcnt lgkmcnt(0)2916; GCN1-NEXT: s_cmp_eq_u32 s1, s42917; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 02918; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]2919; GCN1-NEXT: s_mov_b64 s[4:5], -12920; GCN1-NEXT: s_cbranch_vccnz .LBB20_32921; GCN1-NEXT: ; %bb.1: ; %Flow2922; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]2923; GCN1-NEXT: s_cbranch_vccz .LBB20_42924; GCN1-NEXT: .LBB20_2: ; %atomicrmw.phi2925; GCN1-NEXT: s_endpgm2926; GCN1-NEXT: .LBB20_3: ; %atomicrmw.global2927; GCN1-NEXT: v_mov_b32_e32 v0, s02928; GCN1-NEXT: v_mov_b32_e32 v2, s22929; GCN1-NEXT: v_mov_b32_e32 v1, s12930; GCN1-NEXT: v_mov_b32_e32 v3, s32931; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]2932; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2933; GCN1-NEXT: buffer_wbinvl1_vol2934; GCN1-NEXT: s_cbranch_execnz .LBB20_22935; GCN1-NEXT: .LBB20_4: ; %atomicrmw.private2936; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 02937; GCN1-NEXT: v_mov_b32_e32 v4, s32938; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec2939; GCN1-NEXT: s_cselect_b32 s0, s0, -12940; GCN1-NEXT: v_mov_b32_e32 v0, s02941; GCN1-NEXT: s_add_i32 s0, s0, 42942; GCN1-NEXT: buffer_load_dword v1, v0, s[12:15], 0 offen2943; GCN1-NEXT: v_mov_b32_e32 v2, s02944; GCN1-NEXT: buffer_load_dword v3, v2, s[12:15], 0 offen2945; GCN1-NEXT: s_waitcnt vmcnt(1)2946; GCN1-NEXT: v_subrev_i32_e32 v1, vcc, s2, v12947; GCN1-NEXT: s_waitcnt vmcnt(0)2948; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc2949; GCN1-NEXT: buffer_store_dword v1, v0, s[12:15], 0 offen2950; GCN1-NEXT: buffer_store_dword v3, v2, s[12:15], 0 offen2951; GCN1-NEXT: s_endpgm2952;2953; GCN2-LABEL: atomic_sub_i64:2954; GCN2: ; %bb.0: ; %entry2955; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD02956; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD12957; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242958; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf42959; GCN2-NEXT: s_mov_b32 s90, -12960; GCN2-NEXT: s_mov_b32 s91, 0xe800002961; GCN2-NEXT: s_add_u32 s88, s88, s112962; GCN2-NEXT: s_addc_u32 s89, s89, 02963; GCN2-NEXT: s_waitcnt lgkmcnt(0)2964; GCN2-NEXT: s_cmp_eq_u32 s1, s42965; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 02966; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]2967; GCN2-NEXT: s_mov_b64 s[4:5], -12968; GCN2-NEXT: s_cbranch_vccnz .LBB20_32969; GCN2-NEXT: ; %bb.1: ; %Flow2970; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]2971; GCN2-NEXT: s_cbranch_vccz .LBB20_42972; GCN2-NEXT: .LBB20_2: ; %atomicrmw.phi2973; GCN2-NEXT: s_endpgm2974; GCN2-NEXT: .LBB20_3: ; %atomicrmw.global2975; GCN2-NEXT: v_mov_b32_e32 v0, s02976; GCN2-NEXT: v_mov_b32_e32 v2, s22977; GCN2-NEXT: v_mov_b32_e32 v1, s12978; GCN2-NEXT: v_mov_b32_e32 v3, s32979; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]2980; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2981; GCN2-NEXT: buffer_wbinvl1_vol2982; GCN2-NEXT: s_cbranch_execnz .LBB20_22983; GCN2-NEXT: .LBB20_4: ; %atomicrmw.private2984; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 02985; GCN2-NEXT: s_cselect_b32 s0, s0, -12986; GCN2-NEXT: v_mov_b32_e32 v0, s02987; GCN2-NEXT: s_add_i32 s0, s0, 42988; GCN2-NEXT: buffer_load_dword v1, v0, s[88:91], 0 offen2989; GCN2-NEXT: v_mov_b32_e32 v2, s02990; GCN2-NEXT: buffer_load_dword v3, v2, s[88:91], 0 offen2991; GCN2-NEXT: v_mov_b32_e32 v4, s32992; GCN2-NEXT: s_waitcnt vmcnt(1)2993; GCN2-NEXT: v_subrev_u32_e32 v1, vcc, s2, v12994; GCN2-NEXT: s_waitcnt vmcnt(0)2995; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc2996; GCN2-NEXT: buffer_store_dword v1, v0, s[88:91], 0 offen2997; GCN2-NEXT: buffer_store_dword v3, v2, s[88:91], 0 offen2998; GCN2-NEXT: s_endpgm2999;3000; GFX12-LABEL: atomic_sub_i64:3001; GFX12: ; %bb.0: ; %entry3002; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x243003; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base3004; GFX12-NEXT: s_wait_kmcnt 0x03005; GFX12-NEXT: s_cmp_eq_u32 s1, s73006; GFX12-NEXT: s_cselect_b32 s4, -1, 03007; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)3008; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s43009; GFX12-NEXT: s_mov_b32 s4, -13010; GFX12-NEXT: s_cbranch_vccnz .LBB20_33011; GFX12-NEXT: ; %bb.1: ; %Flow3012; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s43013; GFX12-NEXT: s_cbranch_vccz .LBB20_43014; GFX12-NEXT: .LBB20_2: ; %atomicrmw.phi3015; GFX12-NEXT: s_endpgm3016; GFX12-NEXT: .LBB20_3: ; %atomicrmw.global3017; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13018; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s33019; GFX12-NEXT: flat_atomic_sub_u64 v[0:1], v[2:3] scope:SCOPE_DEV3020; GFX12-NEXT: s_wait_storecnt_dscnt 0x03021; GFX12-NEXT: global_inv scope:SCOPE_DEV3022; GFX12-NEXT: s_cbranch_execnz .LBB20_23023; GFX12-NEXT: .LBB20_4: ; %atomicrmw.private3024; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 03025; GFX12-NEXT: s_cselect_b32 s0, s0, -13026; GFX12-NEXT: scratch_load_b64 v[0:1], off, s03027; GFX12-NEXT: s_wait_loadcnt 0x03028; GFX12-NEXT: v_sub_co_u32 v0, vcc_lo, v0, s23029; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)3030; GFX12-NEXT: v_subrev_co_ci_u32_e64 v1, null, s3, v1, vcc_lo3031; GFX12-NEXT: scratch_store_b64 off, v[0:1], s03032; GFX12-NEXT: s_endpgm3033entry:3034 %tmp0 = atomicrmw volatile sub ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !03035 ret void3036}3037 3038define amdgpu_kernel void @atomic_sub_i64_ret(ptr %out, ptr %out2, i64 %in) {3039; GCN1-LABEL: atomic_sub_i64_ret:3040; GCN1: ; %bb.0: ; %entry3041; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD03042; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD13043; GCN1-NEXT: s_mov_b32 s14, -13044; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f3045; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93046; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd3047; GCN1-NEXT: s_mov_b32 s15, 0xe8f0003048; GCN1-NEXT: s_add_u32 s12, s12, s113049; GCN1-NEXT: s_addc_u32 s13, s13, 03050; GCN1-NEXT: s_waitcnt lgkmcnt(0)3051; GCN1-NEXT: s_cmp_eq_u32 s1, s63052; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 03053; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]3054; GCN1-NEXT: s_cbranch_vccz .LBB21_23055; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global3056; GCN1-NEXT: v_mov_b32_e32 v0, s03057; GCN1-NEXT: v_mov_b32_e32 v2, s43058; GCN1-NEXT: v_mov_b32_e32 v1, s13059; GCN1-NEXT: v_mov_b32_e32 v3, s53060; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc3061; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3062; GCN1-NEXT: buffer_wbinvl1_vol3063; GCN1-NEXT: s_cbranch_execz .LBB21_33064; GCN1-NEXT: s_branch .LBB21_43065; GCN1-NEXT: .LBB21_2:3066; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr13067; GCN1-NEXT: .LBB21_3: ; %atomicrmw.private3068; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 03069; GCN1-NEXT: v_mov_b32_e32 v4, s53070; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec3071; GCN1-NEXT: s_cselect_b32 s0, s0, -13072; GCN1-NEXT: v_mov_b32_e32 v2, s03073; GCN1-NEXT: s_add_i32 s0, s0, 43074; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen3075; GCN1-NEXT: v_mov_b32_e32 v3, s03076; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen3077; GCN1-NEXT: s_waitcnt vmcnt(1)3078; GCN1-NEXT: v_subrev_i32_e32 v5, vcc, s4, v03079; GCN1-NEXT: s_waitcnt vmcnt(0)3080; GCN1-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc3081; GCN1-NEXT: buffer_store_dword v5, v2, s[12:15], 0 offen3082; GCN1-NEXT: buffer_store_dword v4, v3, s[12:15], 0 offen3083; GCN1-NEXT: .LBB21_4: ; %atomicrmw.end3084; GCN1-NEXT: v_mov_b32_e32 v2, s23085; GCN1-NEXT: v_mov_b32_e32 v3, s33086; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]3087; GCN1-NEXT: s_endpgm3088;3089; GCN2-LABEL: atomic_sub_i64_ret:3090; GCN2: ; %bb.0: ; %entry3091; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD03092; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD13093; GCN2-NEXT: s_mov_b32 s90, -13094; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc3095; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243096; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x343097; GCN2-NEXT: s_mov_b32 s91, 0xe800003098; GCN2-NEXT: s_add_u32 s88, s88, s113099; GCN2-NEXT: s_addc_u32 s89, s89, 03100; GCN2-NEXT: s_waitcnt lgkmcnt(0)3101; GCN2-NEXT: s_cmp_eq_u32 s1, s63102; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 03103; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]3104; GCN2-NEXT: s_cbranch_vccz .LBB21_23105; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global3106; GCN2-NEXT: v_mov_b32_e32 v0, s03107; GCN2-NEXT: v_mov_b32_e32 v2, s43108; GCN2-NEXT: v_mov_b32_e32 v1, s13109; GCN2-NEXT: v_mov_b32_e32 v3, s53110; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc3111; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3112; GCN2-NEXT: buffer_wbinvl1_vol3113; GCN2-NEXT: s_cbranch_execz .LBB21_33114; GCN2-NEXT: s_branch .LBB21_43115; GCN2-NEXT: .LBB21_2:3116; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr13117; GCN2-NEXT: .LBB21_3: ; %atomicrmw.private3118; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 03119; GCN2-NEXT: s_cselect_b32 s0, s0, -13120; GCN2-NEXT: v_mov_b32_e32 v2, s03121; GCN2-NEXT: s_add_i32 s0, s0, 43122; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen3123; GCN2-NEXT: v_mov_b32_e32 v3, s03124; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen3125; GCN2-NEXT: v_mov_b32_e32 v4, s53126; GCN2-NEXT: s_waitcnt vmcnt(1)3127; GCN2-NEXT: v_subrev_u32_e32 v5, vcc, s4, v03128; GCN2-NEXT: s_waitcnt vmcnt(0)3129; GCN2-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc3130; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen3131; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen3132; GCN2-NEXT: .LBB21_4: ; %atomicrmw.end3133; GCN2-NEXT: v_mov_b32_e32 v2, s23134; GCN2-NEXT: v_mov_b32_e32 v3, s33135; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]3136; GCN2-NEXT: s_endpgm3137;3138; GFX12-LABEL: atomic_sub_i64_ret:3139; GFX12: ; %bb.0: ; %entry3140; GFX12-NEXT: s_clause 0x13141; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x243142; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x343143; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base3144; GFX12-NEXT: s_wait_kmcnt 0x03145; GFX12-NEXT: s_cmp_eq_u32 s1, s73146; GFX12-NEXT: s_cselect_b32 s6, -1, 03147; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)3148; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s63149; GFX12-NEXT: s_cbranch_vccz .LBB21_23150; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global3151; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13152; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s53153; GFX12-NEXT: flat_atomic_sub_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV3154; GFX12-NEXT: s_wait_loadcnt_dscnt 0x03155; GFX12-NEXT: global_inv scope:SCOPE_DEV3156; GFX12-NEXT: s_cbranch_execz .LBB21_33157; GFX12-NEXT: s_branch .LBB21_43158; GFX12-NEXT: .LBB21_2:3159; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr13160; GFX12-NEXT: .LBB21_3: ; %atomicrmw.private3161; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 03162; GFX12-NEXT: s_cselect_b32 s0, s0, -13163; GFX12-NEXT: scratch_load_b64 v[0:1], off, s03164; GFX12-NEXT: s_wait_loadcnt 0x03165; GFX12-NEXT: v_sub_co_u32 v2, vcc_lo, v0, s43166; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)3167; GFX12-NEXT: v_subrev_co_ci_u32_e64 v3, null, s5, v1, vcc_lo3168; GFX12-NEXT: scratch_store_b64 off, v[2:3], s03169; GFX12-NEXT: .LBB21_4: ; %atomicrmw.end3170; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s33171; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]3172; GFX12-NEXT: s_endpgm3173entry:3174 %tmp0 = atomicrmw volatile sub ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !03175 store i64 %tmp0, ptr %out23176 ret void3177}3178 3179define amdgpu_kernel void @atomic_sub_i64_addr64(ptr %out, i64 %in, i64 %index) {3180; GCN1-LABEL: atomic_sub_i64_addr64:3181; GCN1: ; %bb.0: ; %entry3182; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD03183; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD13184; GCN1-NEXT: s_mov_b32 s14, -13185; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd3186; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93187; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f3188; GCN1-NEXT: s_mov_b32 s15, 0xe8f0003189; GCN1-NEXT: s_add_u32 s12, s12, s113190; GCN1-NEXT: s_addc_u32 s13, s13, 03191; GCN1-NEXT: s_waitcnt lgkmcnt(0)3192; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 33193; GCN1-NEXT: s_add_u32 s0, s0, s43194; GCN1-NEXT: s_addc_u32 s1, s1, s53195; GCN1-NEXT: s_cmp_eq_u32 s1, s83196; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 03197; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]3198; GCN1-NEXT: s_mov_b64 s[4:5], -13199; GCN1-NEXT: s_cbranch_vccnz .LBB22_33200; GCN1-NEXT: ; %bb.1: ; %Flow3201; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]3202; GCN1-NEXT: s_cbranch_vccz .LBB22_43203; GCN1-NEXT: .LBB22_2: ; %atomicrmw.phi3204; GCN1-NEXT: s_endpgm3205; GCN1-NEXT: .LBB22_3: ; %atomicrmw.global3206; GCN1-NEXT: v_mov_b32_e32 v0, s03207; GCN1-NEXT: v_mov_b32_e32 v2, s23208; GCN1-NEXT: v_mov_b32_e32 v1, s13209; GCN1-NEXT: v_mov_b32_e32 v3, s33210; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]3211; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3212; GCN1-NEXT: buffer_wbinvl1_vol3213; GCN1-NEXT: s_cbranch_execnz .LBB22_23214; GCN1-NEXT: .LBB22_4: ; %atomicrmw.private3215; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 03216; GCN1-NEXT: v_mov_b32_e32 v4, s33217; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec3218; GCN1-NEXT: s_cselect_b32 s0, s0, -13219; GCN1-NEXT: v_mov_b32_e32 v0, s03220; GCN1-NEXT: s_add_i32 s0, s0, 43221; GCN1-NEXT: buffer_load_dword v1, v0, s[12:15], 0 offen3222; GCN1-NEXT: v_mov_b32_e32 v2, s03223; GCN1-NEXT: buffer_load_dword v3, v2, s[12:15], 0 offen3224; GCN1-NEXT: s_waitcnt vmcnt(1)3225; GCN1-NEXT: v_subrev_i32_e32 v1, vcc, s2, v13226; GCN1-NEXT: s_waitcnt vmcnt(0)3227; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc3228; GCN1-NEXT: buffer_store_dword v1, v0, s[12:15], 0 offen3229; GCN1-NEXT: buffer_store_dword v3, v2, s[12:15], 0 offen3230; GCN1-NEXT: s_endpgm3231;3232; GCN2-LABEL: atomic_sub_i64_addr64:3233; GCN2: ; %bb.0: ; %entry3234; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD03235; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD13236; GCN2-NEXT: s_mov_b32 s90, -13237; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343238; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243239; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc3240; GCN2-NEXT: s_mov_b32 s91, 0xe800003241; GCN2-NEXT: s_add_u32 s88, s88, s113242; GCN2-NEXT: s_addc_u32 s89, s89, 03243; GCN2-NEXT: s_waitcnt lgkmcnt(0)3244; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 33245; GCN2-NEXT: s_add_u32 s0, s0, s43246; GCN2-NEXT: s_addc_u32 s1, s1, s53247; GCN2-NEXT: s_cmp_eq_u32 s1, s83248; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 03249; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]3250; GCN2-NEXT: s_mov_b64 s[4:5], -13251; GCN2-NEXT: s_cbranch_vccnz .LBB22_33252; GCN2-NEXT: ; %bb.1: ; %Flow3253; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]3254; GCN2-NEXT: s_cbranch_vccz .LBB22_43255; GCN2-NEXT: .LBB22_2: ; %atomicrmw.phi3256; GCN2-NEXT: s_endpgm3257; GCN2-NEXT: .LBB22_3: ; %atomicrmw.global3258; GCN2-NEXT: v_mov_b32_e32 v0, s03259; GCN2-NEXT: v_mov_b32_e32 v2, s23260; GCN2-NEXT: v_mov_b32_e32 v1, s13261; GCN2-NEXT: v_mov_b32_e32 v3, s33262; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]3263; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3264; GCN2-NEXT: buffer_wbinvl1_vol3265; GCN2-NEXT: s_cbranch_execnz .LBB22_23266; GCN2-NEXT: .LBB22_4: ; %atomicrmw.private3267; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 03268; GCN2-NEXT: s_cselect_b32 s0, s0, -13269; GCN2-NEXT: v_mov_b32_e32 v0, s03270; GCN2-NEXT: s_add_i32 s0, s0, 43271; GCN2-NEXT: buffer_load_dword v1, v0, s[88:91], 0 offen3272; GCN2-NEXT: v_mov_b32_e32 v2, s03273; GCN2-NEXT: buffer_load_dword v3, v2, s[88:91], 0 offen3274; GCN2-NEXT: v_mov_b32_e32 v4, s33275; GCN2-NEXT: s_waitcnt vmcnt(1)3276; GCN2-NEXT: v_subrev_u32_e32 v1, vcc, s2, v13277; GCN2-NEXT: s_waitcnt vmcnt(0)3278; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc3279; GCN2-NEXT: buffer_store_dword v1, v0, s[88:91], 0 offen3280; GCN2-NEXT: buffer_store_dword v3, v2, s[88:91], 0 offen3281; GCN2-NEXT: s_endpgm3282;3283; GFX12-LABEL: atomic_sub_i64_addr64:3284; GFX12: ; %bb.0: ; %entry3285; GFX12-NEXT: s_clause 0x13286; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x343287; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x243288; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base3289; GFX12-NEXT: s_wait_kmcnt 0x03290; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 33291; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)3292; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]3293; GFX12-NEXT: s_cmp_eq_u32 s1, s73294; GFX12-NEXT: s_cselect_b32 s4, -1, 03295; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)3296; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s43297; GFX12-NEXT: s_mov_b32 s4, -13298; GFX12-NEXT: s_cbranch_vccnz .LBB22_33299; GFX12-NEXT: ; %bb.1: ; %Flow3300; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s43301; GFX12-NEXT: s_cbranch_vccz .LBB22_43302; GFX12-NEXT: .LBB22_2: ; %atomicrmw.phi3303; GFX12-NEXT: s_endpgm3304; GFX12-NEXT: .LBB22_3: ; %atomicrmw.global3305; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13306; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s33307; GFX12-NEXT: flat_atomic_sub_u64 v[0:1], v[2:3] scope:SCOPE_DEV3308; GFX12-NEXT: s_wait_storecnt_dscnt 0x03309; GFX12-NEXT: global_inv scope:SCOPE_DEV3310; GFX12-NEXT: s_cbranch_execnz .LBB22_23311; GFX12-NEXT: .LBB22_4: ; %atomicrmw.private3312; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 03313; GFX12-NEXT: s_cselect_b32 s0, s0, -13314; GFX12-NEXT: scratch_load_b64 v[0:1], off, s03315; GFX12-NEXT: s_wait_loadcnt 0x03316; GFX12-NEXT: v_sub_co_u32 v0, vcc_lo, v0, s23317; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)3318; GFX12-NEXT: v_subrev_co_ci_u32_e64 v1, null, s3, v1, vcc_lo3319; GFX12-NEXT: scratch_store_b64 off, v[0:1], s03320; GFX12-NEXT: s_endpgm3321entry:3322 %ptr = getelementptr inbounds i64, ptr %out, i64 %index3323 %tmp0 = atomicrmw volatile sub ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !03324 ret void3325}3326 3327define amdgpu_kernel void @atomic_sub_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {3328; GCN1-LABEL: atomic_sub_i64_ret_addr64:3329; GCN1: ; %bb.0: ; %entry3330; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD03331; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD13332; GCN1-NEXT: s_mov_b32 s18, -13333; GCN1-NEXT: s_mov_b32 s19, 0xe8f0003334; GCN1-NEXT: s_add_u32 s16, s16, s113335; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x93336; GCN1-NEXT: s_load_dword s2, s[4:5], 0x413337; GCN1-NEXT: s_addc_u32 s17, s17, 03338; GCN1-NEXT: s_waitcnt lgkmcnt(0)3339; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 33340; GCN1-NEXT: s_add_u32 s0, s8, s03341; GCN1-NEXT: s_addc_u32 s1, s9, s13342; GCN1-NEXT: s_cmp_eq_u32 s1, s23343; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 03344; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]3345; GCN1-NEXT: s_cbranch_vccz .LBB23_23346; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global3347; GCN1-NEXT: v_mov_b32_e32 v0, s03348; GCN1-NEXT: v_mov_b32_e32 v2, s123349; GCN1-NEXT: v_mov_b32_e32 v1, s13350; GCN1-NEXT: v_mov_b32_e32 v3, s133351; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc3352; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3353; GCN1-NEXT: buffer_wbinvl1_vol3354; GCN1-NEXT: s_cbranch_execz .LBB23_33355; GCN1-NEXT: s_branch .LBB23_43356; GCN1-NEXT: .LBB23_2:3357; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr13358; GCN1-NEXT: .LBB23_3: ; %atomicrmw.private3359; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 03360; GCN1-NEXT: v_mov_b32_e32 v4, s133361; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec3362; GCN1-NEXT: s_cselect_b32 s0, s0, -13363; GCN1-NEXT: v_mov_b32_e32 v2, s03364; GCN1-NEXT: s_add_i32 s0, s0, 43365; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen3366; GCN1-NEXT: v_mov_b32_e32 v3, s03367; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen3368; GCN1-NEXT: s_waitcnt vmcnt(1)3369; GCN1-NEXT: v_subrev_i32_e32 v5, vcc, s12, v03370; GCN1-NEXT: s_waitcnt vmcnt(0)3371; GCN1-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc3372; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen3373; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen3374; GCN1-NEXT: .LBB23_4: ; %atomicrmw.end3375; GCN1-NEXT: v_mov_b32_e32 v2, s103376; GCN1-NEXT: v_mov_b32_e32 v3, s113377; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]3378; GCN1-NEXT: s_endpgm3379;3380; GCN2-LABEL: atomic_sub_i64_ret_addr64:3381; GCN2: ; %bb.0: ; %entry3382; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD03383; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD13384; GCN2-NEXT: s_mov_b32 s90, -13385; GCN2-NEXT: s_mov_b32 s91, 0xe800003386; GCN2-NEXT: s_add_u32 s88, s88, s113387; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x243388; GCN2-NEXT: s_load_dword s2, s[4:5], 0x1043389; GCN2-NEXT: s_addc_u32 s89, s89, 03390; GCN2-NEXT: s_waitcnt lgkmcnt(0)3391; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 33392; GCN2-NEXT: s_add_u32 s0, s8, s03393; GCN2-NEXT: s_addc_u32 s1, s9, s13394; GCN2-NEXT: s_cmp_eq_u32 s1, s23395; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 03396; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]3397; GCN2-NEXT: s_cbranch_vccz .LBB23_23398; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global3399; GCN2-NEXT: v_mov_b32_e32 v0, s03400; GCN2-NEXT: v_mov_b32_e32 v2, s123401; GCN2-NEXT: v_mov_b32_e32 v1, s13402; GCN2-NEXT: v_mov_b32_e32 v3, s133403; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc3404; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3405; GCN2-NEXT: buffer_wbinvl1_vol3406; GCN2-NEXT: s_cbranch_execz .LBB23_33407; GCN2-NEXT: s_branch .LBB23_43408; GCN2-NEXT: .LBB23_2:3409; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr13410; GCN2-NEXT: .LBB23_3: ; %atomicrmw.private3411; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 03412; GCN2-NEXT: s_cselect_b32 s0, s0, -13413; GCN2-NEXT: v_mov_b32_e32 v2, s03414; GCN2-NEXT: s_add_i32 s0, s0, 43415; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen3416; GCN2-NEXT: v_mov_b32_e32 v3, s03417; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen3418; GCN2-NEXT: v_mov_b32_e32 v4, s133419; GCN2-NEXT: s_waitcnt vmcnt(1)3420; GCN2-NEXT: v_subrev_u32_e32 v5, vcc, s12, v03421; GCN2-NEXT: s_waitcnt vmcnt(0)3422; GCN2-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc3423; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen3424; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen3425; GCN2-NEXT: .LBB23_4: ; %atomicrmw.end3426; GCN2-NEXT: v_mov_b32_e32 v2, s103427; GCN2-NEXT: v_mov_b32_e32 v3, s113428; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]3429; GCN2-NEXT: s_endpgm3430;3431; GFX12-LABEL: atomic_sub_i64_ret_addr64:3432; GFX12: ; %bb.0: ; %entry3433; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x243434; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base3435; GFX12-NEXT: s_wait_kmcnt 0x03436; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 33437; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)3438; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]3439; GFX12-NEXT: s_cmp_eq_u32 s1, s93440; GFX12-NEXT: s_cselect_b32 s6, -1, 03441; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)3442; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s63443; GFX12-NEXT: s_cbranch_vccz .LBB23_23444; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global3445; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13446; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s53447; GFX12-NEXT: flat_atomic_sub_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV3448; GFX12-NEXT: s_wait_loadcnt_dscnt 0x03449; GFX12-NEXT: global_inv scope:SCOPE_DEV3450; GFX12-NEXT: s_cbranch_execz .LBB23_33451; GFX12-NEXT: s_branch .LBB23_43452; GFX12-NEXT: .LBB23_2:3453; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr13454; GFX12-NEXT: .LBB23_3: ; %atomicrmw.private3455; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 03456; GFX12-NEXT: s_cselect_b32 s0, s0, -13457; GFX12-NEXT: scratch_load_b64 v[0:1], off, s03458; GFX12-NEXT: s_wait_loadcnt 0x03459; GFX12-NEXT: v_sub_co_u32 v2, vcc_lo, v0, s43460; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)3461; GFX12-NEXT: v_subrev_co_ci_u32_e64 v3, null, s5, v1, vcc_lo3462; GFX12-NEXT: scratch_store_b64 off, v[2:3], s03463; GFX12-NEXT: .LBB23_4: ; %atomicrmw.end3464; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s33465; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]3466; GFX12-NEXT: s_endpgm3467entry:3468 %ptr = getelementptr inbounds i64, ptr %out, i64 %index3469 %tmp0 = atomicrmw volatile sub ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !03470 store i64 %tmp0, ptr %out23471 ret void3472}3473 3474define amdgpu_kernel void @atomic_max_i64_offset(ptr %out, i64 %in) {3475; GCN1-LABEL: atomic_max_i64_offset:3476; GCN1: ; %bb.0: ; %entry3477; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD03478; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD13479; GCN1-NEXT: s_mov_b32 s14, -13480; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93481; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d3482; GCN1-NEXT: s_mov_b32 s15, 0xe8f0003483; GCN1-NEXT: s_add_u32 s12, s12, s113484; GCN1-NEXT: s_addc_u32 s13, s13, 03485; GCN1-NEXT: s_waitcnt lgkmcnt(0)3486; GCN1-NEXT: s_add_u32 s0, s0, 323487; GCN1-NEXT: s_addc_u32 s1, s1, 03488; GCN1-NEXT: s_cmp_eq_u32 s1, s43489; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 03490; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]3491; GCN1-NEXT: s_mov_b64 s[4:5], -13492; GCN1-NEXT: s_cbranch_vccnz .LBB24_33493; GCN1-NEXT: ; %bb.1: ; %Flow3494; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]3495; GCN1-NEXT: s_cbranch_vccz .LBB24_43496; GCN1-NEXT: .LBB24_2: ; %atomicrmw.phi3497; GCN1-NEXT: s_endpgm3498; GCN1-NEXT: .LBB24_3: ; %atomicrmw.global3499; GCN1-NEXT: v_mov_b32_e32 v0, s03500; GCN1-NEXT: v_mov_b32_e32 v2, s23501; GCN1-NEXT: v_mov_b32_e32 v1, s13502; GCN1-NEXT: v_mov_b32_e32 v3, s33503; GCN1-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3]3504; GCN1-NEXT: s_waitcnt lgkmcnt(0)3505; GCN1-NEXT: s_cbranch_execnz .LBB24_23506; GCN1-NEXT: .LBB24_4: ; %atomicrmw.private3507; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 03508; GCN1-NEXT: v_mov_b32_e32 v5, s23509; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec3510; GCN1-NEXT: s_cselect_b32 s0, s0, -13511; GCN1-NEXT: v_mov_b32_e32 v2, s03512; GCN1-NEXT: s_add_i32 s0, s0, 43513; GCN1-NEXT: v_mov_b32_e32 v3, s03514; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen3515; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen3516; GCN1-NEXT: v_mov_b32_e32 v4, s33517; GCN1-NEXT: s_waitcnt vmcnt(0)3518; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]3519; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc3520; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc3521; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen3522; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen3523; GCN1-NEXT: s_endpgm3524;3525; GCN2-LABEL: atomic_max_i64_offset:3526; GCN2: ; %bb.0: ; %entry3527; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD03528; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD13529; GCN2-NEXT: s_mov_b32 s90, -13530; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243531; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf43532; GCN2-NEXT: s_mov_b32 s91, 0xe800003533; GCN2-NEXT: s_add_u32 s88, s88, s113534; GCN2-NEXT: s_addc_u32 s89, s89, 03535; GCN2-NEXT: s_waitcnt lgkmcnt(0)3536; GCN2-NEXT: s_add_u32 s0, s0, 323537; GCN2-NEXT: s_addc_u32 s1, s1, 03538; GCN2-NEXT: s_cmp_eq_u32 s1, s43539; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 03540; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]3541; GCN2-NEXT: s_mov_b64 s[4:5], -13542; GCN2-NEXT: s_cbranch_vccnz .LBB24_33543; GCN2-NEXT: ; %bb.1: ; %Flow3544; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]3545; GCN2-NEXT: s_cbranch_vccz .LBB24_43546; GCN2-NEXT: .LBB24_2: ; %atomicrmw.phi3547; GCN2-NEXT: s_endpgm3548; GCN2-NEXT: .LBB24_3: ; %atomicrmw.global3549; GCN2-NEXT: v_mov_b32_e32 v0, s03550; GCN2-NEXT: v_mov_b32_e32 v2, s23551; GCN2-NEXT: v_mov_b32_e32 v1, s13552; GCN2-NEXT: v_mov_b32_e32 v3, s33553; GCN2-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3]3554; GCN2-NEXT: s_waitcnt lgkmcnt(0)3555; GCN2-NEXT: s_cbranch_execnz .LBB24_23556; GCN2-NEXT: .LBB24_4: ; %atomicrmw.private3557; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 03558; GCN2-NEXT: s_cselect_b32 s0, s0, -13559; GCN2-NEXT: v_mov_b32_e32 v2, s03560; GCN2-NEXT: s_add_i32 s0, s0, 43561; GCN2-NEXT: v_mov_b32_e32 v3, s03562; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen3563; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen3564; GCN2-NEXT: v_mov_b32_e32 v5, s23565; GCN2-NEXT: v_mov_b32_e32 v4, s33566; GCN2-NEXT: s_waitcnt vmcnt(0)3567; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]3568; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc3569; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc3570; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen3571; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen3572; GCN2-NEXT: s_endpgm3573;3574; GFX12-LABEL: atomic_max_i64_offset:3575; GFX12: ; %bb.0: ; %entry3576; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x243577; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base3578; GFX12-NEXT: s_wait_kmcnt 0x03579; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 323580; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)3581; GFX12-NEXT: s_cmp_eq_u32 s1, s73582; GFX12-NEXT: s_cselect_b32 s4, -1, 03583; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s43584; GFX12-NEXT: s_mov_b32 s4, -13585; GFX12-NEXT: s_cbranch_vccnz .LBB24_33586; GFX12-NEXT: ; %bb.1: ; %Flow3587; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s43588; GFX12-NEXT: s_cbranch_vccz .LBB24_43589; GFX12-NEXT: .LBB24_2: ; %atomicrmw.phi3590; GFX12-NEXT: s_endpgm3591; GFX12-NEXT: .LBB24_3: ; %atomicrmw.global3592; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13593; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s33594; GFX12-NEXT: flat_atomic_max_i64 v[0:1], v[2:3] scope:SCOPE_SE3595; GFX12-NEXT: s_wait_storecnt_dscnt 0x03596; GFX12-NEXT: global_inv scope:SCOPE_SE3597; GFX12-NEXT: s_cbranch_execnz .LBB24_23598; GFX12-NEXT: .LBB24_4: ; %atomicrmw.private3599; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 03600; GFX12-NEXT: s_cselect_b32 s0, s0, -13601; GFX12-NEXT: scratch_load_b64 v[0:1], off, s03602; GFX12-NEXT: s_wait_loadcnt 0x03603; GFX12-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[2:3], v[0:1]3604; GFX12-NEXT: v_cndmask_b32_e32 v1, s3, v1, vcc_lo3605; GFX12-NEXT: v_cndmask_b32_e32 v0, s2, v0, vcc_lo3606; GFX12-NEXT: scratch_store_b64 off, v[0:1], s03607; GFX12-NEXT: s_endpgm3608entry:3609 %gep = getelementptr inbounds i64, ptr %out, i64 43610 %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !03611 ret void3612}3613 3614define amdgpu_kernel void @atomic_max_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {3615; GCN1-LABEL: atomic_max_i64_ret_offset:3616; GCN1: ; %bb.0: ; %entry3617; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD03618; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD13619; GCN1-NEXT: s_mov_b32 s14, -13620; GCN1-NEXT: s_mov_b32 s15, 0xe8f0003621; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93622; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f3623; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd3624; GCN1-NEXT: s_add_u32 s12, s12, s113625; GCN1-NEXT: s_addc_u32 s13, s13, 03626; GCN1-NEXT: s_waitcnt lgkmcnt(0)3627; GCN1-NEXT: s_add_u32 s0, s0, 323628; GCN1-NEXT: s_addc_u32 s1, s1, 03629; GCN1-NEXT: s_cmp_eq_u32 s1, s63630; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 03631; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]3632; GCN1-NEXT: s_cbranch_vccz .LBB25_23633; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global3634; GCN1-NEXT: v_mov_b32_e32 v0, s03635; GCN1-NEXT: v_mov_b32_e32 v2, s43636; GCN1-NEXT: v_mov_b32_e32 v1, s13637; GCN1-NEXT: v_mov_b32_e32 v3, s53638; GCN1-NEXT: flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc3639; GCN1-NEXT: s_waitcnt lgkmcnt(0)3640; GCN1-NEXT: s_cbranch_execz .LBB25_33641; GCN1-NEXT: s_branch .LBB25_43642; GCN1-NEXT: .LBB25_2:3643; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr13644; GCN1-NEXT: .LBB25_3: ; %atomicrmw.private3645; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 03646; GCN1-NEXT: v_mov_b32_e32 v5, s43647; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec3648; GCN1-NEXT: s_cselect_b32 s0, s0, -13649; GCN1-NEXT: v_mov_b32_e32 v2, s03650; GCN1-NEXT: s_add_i32 s0, s0, 43651; GCN1-NEXT: v_mov_b32_e32 v3, s03652; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen3653; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen3654; GCN1-NEXT: v_mov_b32_e32 v4, s53655; GCN1-NEXT: s_waitcnt vmcnt(0)3656; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]3657; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc3658; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc3659; GCN1-NEXT: buffer_store_dword v5, v2, s[12:15], 0 offen3660; GCN1-NEXT: buffer_store_dword v4, v3, s[12:15], 0 offen3661; GCN1-NEXT: .LBB25_4: ; %atomicrmw.end3662; GCN1-NEXT: v_mov_b32_e32 v2, s23663; GCN1-NEXT: v_mov_b32_e32 v3, s33664; GCN1-NEXT: s_waitcnt vmcnt(0)3665; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]3666; GCN1-NEXT: s_endpgm3667;3668; GCN2-LABEL: atomic_max_i64_ret_offset:3669; GCN2: ; %bb.0: ; %entry3670; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD03671; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD13672; GCN2-NEXT: s_mov_b32 s90, -13673; GCN2-NEXT: s_mov_b32 s91, 0xe800003674; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243675; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc3676; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x343677; GCN2-NEXT: s_add_u32 s88, s88, s113678; GCN2-NEXT: s_addc_u32 s89, s89, 03679; GCN2-NEXT: s_waitcnt lgkmcnt(0)3680; GCN2-NEXT: s_add_u32 s0, s0, 323681; GCN2-NEXT: s_addc_u32 s1, s1, 03682; GCN2-NEXT: s_cmp_eq_u32 s1, s63683; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 03684; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]3685; GCN2-NEXT: s_cbranch_vccz .LBB25_23686; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global3687; GCN2-NEXT: v_mov_b32_e32 v0, s03688; GCN2-NEXT: v_mov_b32_e32 v2, s43689; GCN2-NEXT: v_mov_b32_e32 v1, s13690; GCN2-NEXT: v_mov_b32_e32 v3, s53691; GCN2-NEXT: flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc3692; GCN2-NEXT: s_waitcnt lgkmcnt(0)3693; GCN2-NEXT: s_cbranch_execz .LBB25_33694; GCN2-NEXT: s_branch .LBB25_43695; GCN2-NEXT: .LBB25_2:3696; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr13697; GCN2-NEXT: .LBB25_3: ; %atomicrmw.private3698; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 03699; GCN2-NEXT: s_cselect_b32 s0, s0, -13700; GCN2-NEXT: v_mov_b32_e32 v2, s03701; GCN2-NEXT: s_add_i32 s0, s0, 43702; GCN2-NEXT: v_mov_b32_e32 v3, s03703; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen3704; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen3705; GCN2-NEXT: v_mov_b32_e32 v5, s43706; GCN2-NEXT: v_mov_b32_e32 v4, s53707; GCN2-NEXT: s_waitcnt vmcnt(0)3708; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]3709; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc3710; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc3711; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen3712; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen3713; GCN2-NEXT: .LBB25_4: ; %atomicrmw.end3714; GCN2-NEXT: v_mov_b32_e32 v2, s23715; GCN2-NEXT: v_mov_b32_e32 v3, s33716; GCN2-NEXT: s_waitcnt vmcnt(0)3717; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]3718; GCN2-NEXT: s_endpgm3719;3720; GFX12-LABEL: atomic_max_i64_ret_offset:3721; GFX12: ; %bb.0: ; %entry3722; GFX12-NEXT: s_clause 0x13723; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x243724; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x343725; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base3726; GFX12-NEXT: s_wait_kmcnt 0x03727; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 323728; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)3729; GFX12-NEXT: s_cmp_eq_u32 s1, s73730; GFX12-NEXT: s_cselect_b32 s6, -1, 03731; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s63732; GFX12-NEXT: s_cbranch_vccz .LBB25_23733; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global3734; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13735; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s53736; GFX12-NEXT: flat_atomic_max_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE3737; GFX12-NEXT: s_wait_loadcnt_dscnt 0x03738; GFX12-NEXT: global_inv scope:SCOPE_SE3739; GFX12-NEXT: s_cbranch_execz .LBB25_33740; GFX12-NEXT: s_branch .LBB25_43741; GFX12-NEXT: .LBB25_2:3742; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr13743; GFX12-NEXT: .LBB25_3: ; %atomicrmw.private3744; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 03745; GFX12-NEXT: s_cselect_b32 s0, s0, -13746; GFX12-NEXT: scratch_load_b64 v[0:1], off, s03747; GFX12-NEXT: s_wait_loadcnt 0x03748; GFX12-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[4:5], v[0:1]3749; GFX12-NEXT: v_cndmask_b32_e32 v3, s5, v1, vcc_lo3750; GFX12-NEXT: v_cndmask_b32_e32 v2, s4, v0, vcc_lo3751; GFX12-NEXT: scratch_store_b64 off, v[2:3], s03752; GFX12-NEXT: .LBB25_4: ; %atomicrmw.end3753; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s33754; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]3755; GFX12-NEXT: s_endpgm3756entry:3757 %gep = getelementptr inbounds i64, ptr %out, i64 43758 %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !03759 store i64 %tmp0, ptr %out23760 ret void3761}3762 3763define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {3764; GCN1-LABEL: atomic_max_i64_addr64_offset:3765; GCN1: ; %bb.0: ; %entry3766; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD03767; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD13768; GCN1-NEXT: s_mov_b32 s14, -13769; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd3770; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93771; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f3772; GCN1-NEXT: s_mov_b32 s15, 0xe8f0003773; GCN1-NEXT: s_add_u32 s12, s12, s113774; GCN1-NEXT: s_addc_u32 s13, s13, 03775; GCN1-NEXT: s_waitcnt lgkmcnt(0)3776; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 33777; GCN1-NEXT: s_add_u32 s0, s0, s43778; GCN1-NEXT: s_addc_u32 s1, s1, s53779; GCN1-NEXT: s_add_u32 s0, s0, 323780; GCN1-NEXT: s_addc_u32 s1, s1, 03781; GCN1-NEXT: s_cmp_eq_u32 s1, s83782; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 03783; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]3784; GCN1-NEXT: s_mov_b64 s[4:5], -13785; GCN1-NEXT: s_cbranch_vccnz .LBB26_33786; GCN1-NEXT: ; %bb.1: ; %Flow3787; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]3788; GCN1-NEXT: s_cbranch_vccz .LBB26_43789; GCN1-NEXT: .LBB26_2: ; %atomicrmw.phi3790; GCN1-NEXT: s_endpgm3791; GCN1-NEXT: .LBB26_3: ; %atomicrmw.global3792; GCN1-NEXT: v_mov_b32_e32 v0, s03793; GCN1-NEXT: v_mov_b32_e32 v2, s23794; GCN1-NEXT: v_mov_b32_e32 v1, s13795; GCN1-NEXT: v_mov_b32_e32 v3, s33796; GCN1-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3]3797; GCN1-NEXT: s_waitcnt lgkmcnt(0)3798; GCN1-NEXT: s_cbranch_execnz .LBB26_23799; GCN1-NEXT: .LBB26_4: ; %atomicrmw.private3800; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 03801; GCN1-NEXT: v_mov_b32_e32 v5, s23802; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec3803; GCN1-NEXT: s_cselect_b32 s0, s0, -13804; GCN1-NEXT: v_mov_b32_e32 v2, s03805; GCN1-NEXT: s_add_i32 s0, s0, 43806; GCN1-NEXT: v_mov_b32_e32 v3, s03807; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen3808; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen3809; GCN1-NEXT: v_mov_b32_e32 v4, s33810; GCN1-NEXT: s_waitcnt vmcnt(0)3811; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]3812; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc3813; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc3814; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen3815; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen3816; GCN1-NEXT: s_endpgm3817;3818; GCN2-LABEL: atomic_max_i64_addr64_offset:3819; GCN2: ; %bb.0: ; %entry3820; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD03821; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD13822; GCN2-NEXT: s_mov_b32 s90, -13823; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343824; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243825; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc3826; GCN2-NEXT: s_mov_b32 s91, 0xe800003827; GCN2-NEXT: s_add_u32 s88, s88, s113828; GCN2-NEXT: s_addc_u32 s89, s89, 03829; GCN2-NEXT: s_waitcnt lgkmcnt(0)3830; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 33831; GCN2-NEXT: s_add_u32 s0, s0, s43832; GCN2-NEXT: s_addc_u32 s1, s1, s53833; GCN2-NEXT: s_add_u32 s0, s0, 323834; GCN2-NEXT: s_addc_u32 s1, s1, 03835; GCN2-NEXT: s_cmp_eq_u32 s1, s83836; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 03837; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]3838; GCN2-NEXT: s_mov_b64 s[4:5], -13839; GCN2-NEXT: s_cbranch_vccnz .LBB26_33840; GCN2-NEXT: ; %bb.1: ; %Flow3841; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]3842; GCN2-NEXT: s_cbranch_vccz .LBB26_43843; GCN2-NEXT: .LBB26_2: ; %atomicrmw.phi3844; GCN2-NEXT: s_endpgm3845; GCN2-NEXT: .LBB26_3: ; %atomicrmw.global3846; GCN2-NEXT: v_mov_b32_e32 v0, s03847; GCN2-NEXT: v_mov_b32_e32 v2, s23848; GCN2-NEXT: v_mov_b32_e32 v1, s13849; GCN2-NEXT: v_mov_b32_e32 v3, s33850; GCN2-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3]3851; GCN2-NEXT: s_waitcnt lgkmcnt(0)3852; GCN2-NEXT: s_cbranch_execnz .LBB26_23853; GCN2-NEXT: .LBB26_4: ; %atomicrmw.private3854; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 03855; GCN2-NEXT: s_cselect_b32 s0, s0, -13856; GCN2-NEXT: v_mov_b32_e32 v2, s03857; GCN2-NEXT: s_add_i32 s0, s0, 43858; GCN2-NEXT: v_mov_b32_e32 v3, s03859; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen3860; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen3861; GCN2-NEXT: v_mov_b32_e32 v5, s23862; GCN2-NEXT: v_mov_b32_e32 v4, s33863; GCN2-NEXT: s_waitcnt vmcnt(0)3864; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]3865; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc3866; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc3867; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen3868; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen3869; GCN2-NEXT: s_endpgm3870;3871; GFX12-LABEL: atomic_max_i64_addr64_offset:3872; GFX12: ; %bb.0: ; %entry3873; GFX12-NEXT: s_clause 0x13874; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x343875; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x243876; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base3877; GFX12-NEXT: s_wait_kmcnt 0x03878; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 33879; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)3880; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]3881; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 323882; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)3883; GFX12-NEXT: s_cmp_eq_u32 s1, s73884; GFX12-NEXT: s_cselect_b32 s4, -1, 03885; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s43886; GFX12-NEXT: s_mov_b32 s4, -13887; GFX12-NEXT: s_cbranch_vccnz .LBB26_33888; GFX12-NEXT: ; %bb.1: ; %Flow3889; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s43890; GFX12-NEXT: s_cbranch_vccz .LBB26_43891; GFX12-NEXT: .LBB26_2: ; %atomicrmw.phi3892; GFX12-NEXT: s_endpgm3893; GFX12-NEXT: .LBB26_3: ; %atomicrmw.global3894; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13895; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s33896; GFX12-NEXT: flat_atomic_max_i64 v[0:1], v[2:3] scope:SCOPE_SE3897; GFX12-NEXT: s_wait_storecnt_dscnt 0x03898; GFX12-NEXT: global_inv scope:SCOPE_SE3899; GFX12-NEXT: s_cbranch_execnz .LBB26_23900; GFX12-NEXT: .LBB26_4: ; %atomicrmw.private3901; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 03902; GFX12-NEXT: s_cselect_b32 s0, s0, -13903; GFX12-NEXT: scratch_load_b64 v[0:1], off, s03904; GFX12-NEXT: s_wait_loadcnt 0x03905; GFX12-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[2:3], v[0:1]3906; GFX12-NEXT: v_cndmask_b32_e32 v1, s3, v1, vcc_lo3907; GFX12-NEXT: v_cndmask_b32_e32 v0, s2, v0, vcc_lo3908; GFX12-NEXT: scratch_store_b64 off, v[0:1], s03909; GFX12-NEXT: s_endpgm3910entry:3911 %ptr = getelementptr inbounds i64, ptr %out, i64 %index3912 %gep = getelementptr inbounds i64, ptr %ptr, i64 43913 %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !03914 ret void3915}3916 3917define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {3918; GCN1-LABEL: atomic_max_i64_ret_addr64_offset:3919; GCN1: ; %bb.0: ; %entry3920; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD03921; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD13922; GCN1-NEXT: s_mov_b32 s18, -13923; GCN1-NEXT: s_mov_b32 s19, 0xe8f0003924; GCN1-NEXT: s_add_u32 s16, s16, s113925; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x93926; GCN1-NEXT: s_load_dword s2, s[4:5], 0x413927; GCN1-NEXT: s_addc_u32 s17, s17, 03928; GCN1-NEXT: s_waitcnt lgkmcnt(0)3929; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 33930; GCN1-NEXT: s_add_u32 s0, s8, s03931; GCN1-NEXT: s_addc_u32 s1, s9, s13932; GCN1-NEXT: s_add_u32 s0, s0, 323933; GCN1-NEXT: s_addc_u32 s1, s1, 03934; GCN1-NEXT: s_cmp_eq_u32 s1, s23935; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 03936; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]3937; GCN1-NEXT: s_cbranch_vccz .LBB27_23938; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global3939; GCN1-NEXT: v_mov_b32_e32 v0, s03940; GCN1-NEXT: v_mov_b32_e32 v2, s123941; GCN1-NEXT: v_mov_b32_e32 v1, s13942; GCN1-NEXT: v_mov_b32_e32 v3, s133943; GCN1-NEXT: flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc3944; GCN1-NEXT: s_waitcnt lgkmcnt(0)3945; GCN1-NEXT: s_cbranch_execz .LBB27_33946; GCN1-NEXT: s_branch .LBB27_43947; GCN1-NEXT: .LBB27_2:3948; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr13949; GCN1-NEXT: .LBB27_3: ; %atomicrmw.private3950; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 03951; GCN1-NEXT: v_mov_b32_e32 v5, s123952; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec3953; GCN1-NEXT: s_cselect_b32 s0, s0, -13954; GCN1-NEXT: v_mov_b32_e32 v2, s03955; GCN1-NEXT: s_add_i32 s0, s0, 43956; GCN1-NEXT: v_mov_b32_e32 v3, s03957; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen3958; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen3959; GCN1-NEXT: v_mov_b32_e32 v4, s133960; GCN1-NEXT: s_waitcnt vmcnt(0)3961; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]3962; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc3963; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc3964; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen3965; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen3966; GCN1-NEXT: .LBB27_4: ; %atomicrmw.end3967; GCN1-NEXT: v_mov_b32_e32 v2, s103968; GCN1-NEXT: v_mov_b32_e32 v3, s113969; GCN1-NEXT: s_waitcnt vmcnt(0)3970; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]3971; GCN1-NEXT: s_endpgm3972;3973; GCN2-LABEL: atomic_max_i64_ret_addr64_offset:3974; GCN2: ; %bb.0: ; %entry3975; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD03976; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD13977; GCN2-NEXT: s_mov_b32 s90, -13978; GCN2-NEXT: s_mov_b32 s91, 0xe800003979; GCN2-NEXT: s_add_u32 s88, s88, s113980; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x243981; GCN2-NEXT: s_load_dword s2, s[4:5], 0x1043982; GCN2-NEXT: s_addc_u32 s89, s89, 03983; GCN2-NEXT: s_waitcnt lgkmcnt(0)3984; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 33985; GCN2-NEXT: s_add_u32 s0, s8, s03986; GCN2-NEXT: s_addc_u32 s1, s9, s13987; GCN2-NEXT: s_add_u32 s0, s0, 323988; GCN2-NEXT: s_addc_u32 s1, s1, 03989; GCN2-NEXT: s_cmp_eq_u32 s1, s23990; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 03991; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]3992; GCN2-NEXT: s_cbranch_vccz .LBB27_23993; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global3994; GCN2-NEXT: v_mov_b32_e32 v0, s03995; GCN2-NEXT: v_mov_b32_e32 v2, s123996; GCN2-NEXT: v_mov_b32_e32 v1, s13997; GCN2-NEXT: v_mov_b32_e32 v3, s133998; GCN2-NEXT: flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc3999; GCN2-NEXT: s_waitcnt lgkmcnt(0)4000; GCN2-NEXT: s_cbranch_execz .LBB27_34001; GCN2-NEXT: s_branch .LBB27_44002; GCN2-NEXT: .LBB27_2:4003; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr14004; GCN2-NEXT: .LBB27_3: ; %atomicrmw.private4005; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 04006; GCN2-NEXT: s_cselect_b32 s0, s0, -14007; GCN2-NEXT: v_mov_b32_e32 v2, s04008; GCN2-NEXT: s_add_i32 s0, s0, 44009; GCN2-NEXT: v_mov_b32_e32 v3, s04010; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen4011; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen4012; GCN2-NEXT: v_mov_b32_e32 v5, s124013; GCN2-NEXT: v_mov_b32_e32 v4, s134014; GCN2-NEXT: s_waitcnt vmcnt(0)4015; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]4016; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc4017; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc4018; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen4019; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen4020; GCN2-NEXT: .LBB27_4: ; %atomicrmw.end4021; GCN2-NEXT: v_mov_b32_e32 v2, s104022; GCN2-NEXT: v_mov_b32_e32 v3, s114023; GCN2-NEXT: s_waitcnt vmcnt(0)4024; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]4025; GCN2-NEXT: s_endpgm4026;4027; GFX12-LABEL: atomic_max_i64_ret_addr64_offset:4028; GFX12: ; %bb.0: ; %entry4029; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x244030; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base4031; GFX12-NEXT: s_wait_kmcnt 0x04032; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 34033; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)4034; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]4035; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 324036; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)4037; GFX12-NEXT: s_cmp_eq_u32 s1, s94038; GFX12-NEXT: s_cselect_b32 s6, -1, 04039; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s64040; GFX12-NEXT: s_cbranch_vccz .LBB27_24041; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global4042; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14043; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s54044; GFX12-NEXT: flat_atomic_max_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE4045; GFX12-NEXT: s_wait_loadcnt_dscnt 0x04046; GFX12-NEXT: global_inv scope:SCOPE_SE4047; GFX12-NEXT: s_cbranch_execz .LBB27_34048; GFX12-NEXT: s_branch .LBB27_44049; GFX12-NEXT: .LBB27_2:4050; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr14051; GFX12-NEXT: .LBB27_3: ; %atomicrmw.private4052; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 04053; GFX12-NEXT: s_cselect_b32 s0, s0, -14054; GFX12-NEXT: scratch_load_b64 v[0:1], off, s04055; GFX12-NEXT: s_wait_loadcnt 0x04056; GFX12-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[4:5], v[0:1]4057; GFX12-NEXT: v_cndmask_b32_e32 v3, s5, v1, vcc_lo4058; GFX12-NEXT: v_cndmask_b32_e32 v2, s4, v0, vcc_lo4059; GFX12-NEXT: scratch_store_b64 off, v[2:3], s04060; GFX12-NEXT: .LBB27_4: ; %atomicrmw.end4061; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s34062; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]4063; GFX12-NEXT: s_endpgm4064entry:4065 %ptr = getelementptr inbounds i64, ptr %out, i64 %index4066 %gep = getelementptr inbounds i64, ptr %ptr, i64 44067 %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !04068 store i64 %tmp0, ptr %out24069 ret void4070}4071 4072define amdgpu_kernel void @atomic_max_i64(ptr %out, i64 %in) {4073; GCN1-LABEL: atomic_max_i64:4074; GCN1: ; %bb.0: ; %entry4075; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD04076; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD14077; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94078; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d4079; GCN1-NEXT: s_mov_b32 s14, -14080; GCN1-NEXT: s_mov_b32 s15, 0xe8f0004081; GCN1-NEXT: s_add_u32 s12, s12, s114082; GCN1-NEXT: s_addc_u32 s13, s13, 04083; GCN1-NEXT: s_waitcnt lgkmcnt(0)4084; GCN1-NEXT: s_cmp_eq_u32 s1, s44085; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 04086; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]4087; GCN1-NEXT: s_mov_b64 s[4:5], -14088; GCN1-NEXT: s_cbranch_vccnz .LBB28_34089; GCN1-NEXT: ; %bb.1: ; %Flow4090; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]4091; GCN1-NEXT: s_cbranch_vccz .LBB28_44092; GCN1-NEXT: .LBB28_2: ; %atomicrmw.phi4093; GCN1-NEXT: s_endpgm4094; GCN1-NEXT: .LBB28_3: ; %atomicrmw.global4095; GCN1-NEXT: v_mov_b32_e32 v0, s04096; GCN1-NEXT: v_mov_b32_e32 v2, s24097; GCN1-NEXT: v_mov_b32_e32 v1, s14098; GCN1-NEXT: v_mov_b32_e32 v3, s34099; GCN1-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3]4100; GCN1-NEXT: s_waitcnt lgkmcnt(0)4101; GCN1-NEXT: s_cbranch_execnz .LBB28_24102; GCN1-NEXT: .LBB28_4: ; %atomicrmw.private4103; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 04104; GCN1-NEXT: v_mov_b32_e32 v5, s24105; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec4106; GCN1-NEXT: s_cselect_b32 s0, s0, -14107; GCN1-NEXT: v_mov_b32_e32 v2, s04108; GCN1-NEXT: s_add_i32 s0, s0, 44109; GCN1-NEXT: v_mov_b32_e32 v3, s04110; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen4111; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen4112; GCN1-NEXT: v_mov_b32_e32 v4, s34113; GCN1-NEXT: s_waitcnt vmcnt(0)4114; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]4115; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc4116; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc4117; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen4118; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen4119; GCN1-NEXT: s_endpgm4120;4121; GCN2-LABEL: atomic_max_i64:4122; GCN2: ; %bb.0: ; %entry4123; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD04124; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD14125; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244126; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf44127; GCN2-NEXT: s_mov_b32 s90, -14128; GCN2-NEXT: s_mov_b32 s91, 0xe800004129; GCN2-NEXT: s_add_u32 s88, s88, s114130; GCN2-NEXT: s_addc_u32 s89, s89, 04131; GCN2-NEXT: s_waitcnt lgkmcnt(0)4132; GCN2-NEXT: s_cmp_eq_u32 s1, s44133; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 04134; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]4135; GCN2-NEXT: s_mov_b64 s[4:5], -14136; GCN2-NEXT: s_cbranch_vccnz .LBB28_34137; GCN2-NEXT: ; %bb.1: ; %Flow4138; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]4139; GCN2-NEXT: s_cbranch_vccz .LBB28_44140; GCN2-NEXT: .LBB28_2: ; %atomicrmw.phi4141; GCN2-NEXT: s_endpgm4142; GCN2-NEXT: .LBB28_3: ; %atomicrmw.global4143; GCN2-NEXT: v_mov_b32_e32 v0, s04144; GCN2-NEXT: v_mov_b32_e32 v2, s24145; GCN2-NEXT: v_mov_b32_e32 v1, s14146; GCN2-NEXT: v_mov_b32_e32 v3, s34147; GCN2-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3]4148; GCN2-NEXT: s_waitcnt lgkmcnt(0)4149; GCN2-NEXT: s_cbranch_execnz .LBB28_24150; GCN2-NEXT: .LBB28_4: ; %atomicrmw.private4151; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 04152; GCN2-NEXT: s_cselect_b32 s0, s0, -14153; GCN2-NEXT: v_mov_b32_e32 v2, s04154; GCN2-NEXT: s_add_i32 s0, s0, 44155; GCN2-NEXT: v_mov_b32_e32 v3, s04156; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen4157; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen4158; GCN2-NEXT: v_mov_b32_e32 v5, s24159; GCN2-NEXT: v_mov_b32_e32 v4, s34160; GCN2-NEXT: s_waitcnt vmcnt(0)4161; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]4162; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc4163; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc4164; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen4165; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen4166; GCN2-NEXT: s_endpgm4167;4168; GFX12-LABEL: atomic_max_i64:4169; GFX12: ; %bb.0: ; %entry4170; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x244171; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base4172; GFX12-NEXT: s_wait_kmcnt 0x04173; GFX12-NEXT: s_cmp_eq_u32 s1, s74174; GFX12-NEXT: s_cselect_b32 s4, -1, 04175; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4176; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s44177; GFX12-NEXT: s_mov_b32 s4, -14178; GFX12-NEXT: s_cbranch_vccnz .LBB28_34179; GFX12-NEXT: ; %bb.1: ; %Flow4180; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s44181; GFX12-NEXT: s_cbranch_vccz .LBB28_44182; GFX12-NEXT: .LBB28_2: ; %atomicrmw.phi4183; GFX12-NEXT: s_endpgm4184; GFX12-NEXT: .LBB28_3: ; %atomicrmw.global4185; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14186; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s34187; GFX12-NEXT: flat_atomic_max_i64 v[0:1], v[2:3] scope:SCOPE_SE4188; GFX12-NEXT: s_wait_storecnt_dscnt 0x04189; GFX12-NEXT: global_inv scope:SCOPE_SE4190; GFX12-NEXT: s_cbranch_execnz .LBB28_24191; GFX12-NEXT: .LBB28_4: ; %atomicrmw.private4192; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 04193; GFX12-NEXT: s_cselect_b32 s0, s0, -14194; GFX12-NEXT: scratch_load_b64 v[0:1], off, s04195; GFX12-NEXT: s_wait_loadcnt 0x04196; GFX12-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[2:3], v[0:1]4197; GFX12-NEXT: v_cndmask_b32_e32 v1, s3, v1, vcc_lo4198; GFX12-NEXT: v_cndmask_b32_e32 v0, s2, v0, vcc_lo4199; GFX12-NEXT: scratch_store_b64 off, v[0:1], s04200; GFX12-NEXT: s_endpgm4201entry:4202 %tmp0 = atomicrmw volatile max ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !04203 ret void4204}4205 4206define amdgpu_kernel void @atomic_max_i64_ret(ptr %out, ptr %out2, i64 %in) {4207; GCN1-LABEL: atomic_max_i64_ret:4208; GCN1: ; %bb.0: ; %entry4209; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD04210; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD14211; GCN1-NEXT: s_mov_b32 s14, -14212; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f4213; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94214; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd4215; GCN1-NEXT: s_mov_b32 s15, 0xe8f0004216; GCN1-NEXT: s_add_u32 s12, s12, s114217; GCN1-NEXT: s_addc_u32 s13, s13, 04218; GCN1-NEXT: s_waitcnt lgkmcnt(0)4219; GCN1-NEXT: s_cmp_eq_u32 s1, s64220; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 04221; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]4222; GCN1-NEXT: s_cbranch_vccz .LBB29_24223; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global4224; GCN1-NEXT: v_mov_b32_e32 v0, s04225; GCN1-NEXT: v_mov_b32_e32 v2, s44226; GCN1-NEXT: v_mov_b32_e32 v1, s14227; GCN1-NEXT: v_mov_b32_e32 v3, s54228; GCN1-NEXT: flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc4229; GCN1-NEXT: s_waitcnt lgkmcnt(0)4230; GCN1-NEXT: s_cbranch_execz .LBB29_34231; GCN1-NEXT: s_branch .LBB29_44232; GCN1-NEXT: .LBB29_2:4233; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr14234; GCN1-NEXT: .LBB29_3: ; %atomicrmw.private4235; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 04236; GCN1-NEXT: v_mov_b32_e32 v5, s44237; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec4238; GCN1-NEXT: s_cselect_b32 s0, s0, -14239; GCN1-NEXT: v_mov_b32_e32 v2, s04240; GCN1-NEXT: s_add_i32 s0, s0, 44241; GCN1-NEXT: v_mov_b32_e32 v3, s04242; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen4243; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen4244; GCN1-NEXT: v_mov_b32_e32 v4, s54245; GCN1-NEXT: s_waitcnt vmcnt(0)4246; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]4247; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc4248; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc4249; GCN1-NEXT: buffer_store_dword v5, v2, s[12:15], 0 offen4250; GCN1-NEXT: buffer_store_dword v4, v3, s[12:15], 0 offen4251; GCN1-NEXT: .LBB29_4: ; %atomicrmw.end4252; GCN1-NEXT: v_mov_b32_e32 v2, s24253; GCN1-NEXT: v_mov_b32_e32 v3, s34254; GCN1-NEXT: s_waitcnt vmcnt(0)4255; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]4256; GCN1-NEXT: s_endpgm4257;4258; GCN2-LABEL: atomic_max_i64_ret:4259; GCN2: ; %bb.0: ; %entry4260; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD04261; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD14262; GCN2-NEXT: s_mov_b32 s90, -14263; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc4264; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244265; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x344266; GCN2-NEXT: s_mov_b32 s91, 0xe800004267; GCN2-NEXT: s_add_u32 s88, s88, s114268; GCN2-NEXT: s_addc_u32 s89, s89, 04269; GCN2-NEXT: s_waitcnt lgkmcnt(0)4270; GCN2-NEXT: s_cmp_eq_u32 s1, s64271; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 04272; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]4273; GCN2-NEXT: s_cbranch_vccz .LBB29_24274; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global4275; GCN2-NEXT: v_mov_b32_e32 v0, s04276; GCN2-NEXT: v_mov_b32_e32 v2, s44277; GCN2-NEXT: v_mov_b32_e32 v1, s14278; GCN2-NEXT: v_mov_b32_e32 v3, s54279; GCN2-NEXT: flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc4280; GCN2-NEXT: s_waitcnt lgkmcnt(0)4281; GCN2-NEXT: s_cbranch_execz .LBB29_34282; GCN2-NEXT: s_branch .LBB29_44283; GCN2-NEXT: .LBB29_2:4284; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr14285; GCN2-NEXT: .LBB29_3: ; %atomicrmw.private4286; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 04287; GCN2-NEXT: s_cselect_b32 s0, s0, -14288; GCN2-NEXT: v_mov_b32_e32 v2, s04289; GCN2-NEXT: s_add_i32 s0, s0, 44290; GCN2-NEXT: v_mov_b32_e32 v3, s04291; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen4292; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen4293; GCN2-NEXT: v_mov_b32_e32 v5, s44294; GCN2-NEXT: v_mov_b32_e32 v4, s54295; GCN2-NEXT: s_waitcnt vmcnt(0)4296; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]4297; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc4298; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc4299; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen4300; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen4301; GCN2-NEXT: .LBB29_4: ; %atomicrmw.end4302; GCN2-NEXT: v_mov_b32_e32 v2, s24303; GCN2-NEXT: v_mov_b32_e32 v3, s34304; GCN2-NEXT: s_waitcnt vmcnt(0)4305; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]4306; GCN2-NEXT: s_endpgm4307;4308; GFX12-LABEL: atomic_max_i64_ret:4309; GFX12: ; %bb.0: ; %entry4310; GFX12-NEXT: s_clause 0x14311; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x244312; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x344313; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base4314; GFX12-NEXT: s_wait_kmcnt 0x04315; GFX12-NEXT: s_cmp_eq_u32 s1, s74316; GFX12-NEXT: s_cselect_b32 s6, -1, 04317; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4318; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s64319; GFX12-NEXT: s_cbranch_vccz .LBB29_24320; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global4321; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14322; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s54323; GFX12-NEXT: flat_atomic_max_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE4324; GFX12-NEXT: s_wait_loadcnt_dscnt 0x04325; GFX12-NEXT: global_inv scope:SCOPE_SE4326; GFX12-NEXT: s_cbranch_execz .LBB29_34327; GFX12-NEXT: s_branch .LBB29_44328; GFX12-NEXT: .LBB29_2:4329; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr14330; GFX12-NEXT: .LBB29_3: ; %atomicrmw.private4331; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 04332; GFX12-NEXT: s_cselect_b32 s0, s0, -14333; GFX12-NEXT: scratch_load_b64 v[0:1], off, s04334; GFX12-NEXT: s_wait_loadcnt 0x04335; GFX12-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[4:5], v[0:1]4336; GFX12-NEXT: v_cndmask_b32_e32 v3, s5, v1, vcc_lo4337; GFX12-NEXT: v_cndmask_b32_e32 v2, s4, v0, vcc_lo4338; GFX12-NEXT: scratch_store_b64 off, v[2:3], s04339; GFX12-NEXT: .LBB29_4: ; %atomicrmw.end4340; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s34341; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]4342; GFX12-NEXT: s_endpgm4343entry:4344 %tmp0 = atomicrmw volatile max ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !04345 store i64 %tmp0, ptr %out24346 ret void4347}4348 4349define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index) {4350; GCN1-LABEL: atomic_max_i64_addr64:4351; GCN1: ; %bb.0: ; %entry4352; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD04353; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD14354; GCN1-NEXT: s_mov_b32 s14, -14355; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd4356; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94357; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f4358; GCN1-NEXT: s_mov_b32 s15, 0xe8f0004359; GCN1-NEXT: s_add_u32 s12, s12, s114360; GCN1-NEXT: s_addc_u32 s13, s13, 04361; GCN1-NEXT: s_waitcnt lgkmcnt(0)4362; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 34363; GCN1-NEXT: s_add_u32 s0, s0, s44364; GCN1-NEXT: s_addc_u32 s1, s1, s54365; GCN1-NEXT: s_cmp_eq_u32 s1, s84366; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 04367; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]4368; GCN1-NEXT: s_mov_b64 s[4:5], -14369; GCN1-NEXT: s_cbranch_vccnz .LBB30_34370; GCN1-NEXT: ; %bb.1: ; %Flow4371; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]4372; GCN1-NEXT: s_cbranch_vccz .LBB30_44373; GCN1-NEXT: .LBB30_2: ; %atomicrmw.phi4374; GCN1-NEXT: s_endpgm4375; GCN1-NEXT: .LBB30_3: ; %atomicrmw.global4376; GCN1-NEXT: v_mov_b32_e32 v0, s04377; GCN1-NEXT: v_mov_b32_e32 v2, s24378; GCN1-NEXT: v_mov_b32_e32 v1, s14379; GCN1-NEXT: v_mov_b32_e32 v3, s34380; GCN1-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3]4381; GCN1-NEXT: s_waitcnt lgkmcnt(0)4382; GCN1-NEXT: s_cbranch_execnz .LBB30_24383; GCN1-NEXT: .LBB30_4: ; %atomicrmw.private4384; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 04385; GCN1-NEXT: v_mov_b32_e32 v5, s24386; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec4387; GCN1-NEXT: s_cselect_b32 s0, s0, -14388; GCN1-NEXT: v_mov_b32_e32 v2, s04389; GCN1-NEXT: s_add_i32 s0, s0, 44390; GCN1-NEXT: v_mov_b32_e32 v3, s04391; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen4392; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen4393; GCN1-NEXT: v_mov_b32_e32 v4, s34394; GCN1-NEXT: s_waitcnt vmcnt(0)4395; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]4396; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc4397; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc4398; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen4399; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen4400; GCN1-NEXT: s_endpgm4401;4402; GCN2-LABEL: atomic_max_i64_addr64:4403; GCN2: ; %bb.0: ; %entry4404; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD04405; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD14406; GCN2-NEXT: s_mov_b32 s90, -14407; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x344408; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244409; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc4410; GCN2-NEXT: s_mov_b32 s91, 0xe800004411; GCN2-NEXT: s_add_u32 s88, s88, s114412; GCN2-NEXT: s_addc_u32 s89, s89, 04413; GCN2-NEXT: s_waitcnt lgkmcnt(0)4414; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 34415; GCN2-NEXT: s_add_u32 s0, s0, s44416; GCN2-NEXT: s_addc_u32 s1, s1, s54417; GCN2-NEXT: s_cmp_eq_u32 s1, s84418; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 04419; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]4420; GCN2-NEXT: s_mov_b64 s[4:5], -14421; GCN2-NEXT: s_cbranch_vccnz .LBB30_34422; GCN2-NEXT: ; %bb.1: ; %Flow4423; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]4424; GCN2-NEXT: s_cbranch_vccz .LBB30_44425; GCN2-NEXT: .LBB30_2: ; %atomicrmw.phi4426; GCN2-NEXT: s_endpgm4427; GCN2-NEXT: .LBB30_3: ; %atomicrmw.global4428; GCN2-NEXT: v_mov_b32_e32 v0, s04429; GCN2-NEXT: v_mov_b32_e32 v2, s24430; GCN2-NEXT: v_mov_b32_e32 v1, s14431; GCN2-NEXT: v_mov_b32_e32 v3, s34432; GCN2-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3]4433; GCN2-NEXT: s_waitcnt lgkmcnt(0)4434; GCN2-NEXT: s_cbranch_execnz .LBB30_24435; GCN2-NEXT: .LBB30_4: ; %atomicrmw.private4436; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 04437; GCN2-NEXT: s_cselect_b32 s0, s0, -14438; GCN2-NEXT: v_mov_b32_e32 v2, s04439; GCN2-NEXT: s_add_i32 s0, s0, 44440; GCN2-NEXT: v_mov_b32_e32 v3, s04441; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen4442; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen4443; GCN2-NEXT: v_mov_b32_e32 v5, s24444; GCN2-NEXT: v_mov_b32_e32 v4, s34445; GCN2-NEXT: s_waitcnt vmcnt(0)4446; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]4447; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc4448; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc4449; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen4450; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen4451; GCN2-NEXT: s_endpgm4452;4453; GFX12-LABEL: atomic_max_i64_addr64:4454; GFX12: ; %bb.0: ; %entry4455; GFX12-NEXT: s_clause 0x14456; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x344457; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x244458; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base4459; GFX12-NEXT: s_wait_kmcnt 0x04460; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 34461; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)4462; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]4463; GFX12-NEXT: s_cmp_eq_u32 s1, s74464; GFX12-NEXT: s_cselect_b32 s4, -1, 04465; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4466; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s44467; GFX12-NEXT: s_mov_b32 s4, -14468; GFX12-NEXT: s_cbranch_vccnz .LBB30_34469; GFX12-NEXT: ; %bb.1: ; %Flow4470; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s44471; GFX12-NEXT: s_cbranch_vccz .LBB30_44472; GFX12-NEXT: .LBB30_2: ; %atomicrmw.phi4473; GFX12-NEXT: s_endpgm4474; GFX12-NEXT: .LBB30_3: ; %atomicrmw.global4475; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14476; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s34477; GFX12-NEXT: flat_atomic_max_i64 v[0:1], v[2:3] scope:SCOPE_SE4478; GFX12-NEXT: s_wait_storecnt_dscnt 0x04479; GFX12-NEXT: global_inv scope:SCOPE_SE4480; GFX12-NEXT: s_cbranch_execnz .LBB30_24481; GFX12-NEXT: .LBB30_4: ; %atomicrmw.private4482; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 04483; GFX12-NEXT: s_cselect_b32 s0, s0, -14484; GFX12-NEXT: scratch_load_b64 v[0:1], off, s04485; GFX12-NEXT: s_wait_loadcnt 0x04486; GFX12-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[2:3], v[0:1]4487; GFX12-NEXT: v_cndmask_b32_e32 v1, s3, v1, vcc_lo4488; GFX12-NEXT: v_cndmask_b32_e32 v0, s2, v0, vcc_lo4489; GFX12-NEXT: scratch_store_b64 off, v[0:1], s04490; GFX12-NEXT: s_endpgm4491entry:4492 %ptr = getelementptr inbounds i64, ptr %out, i64 %index4493 %tmp0 = atomicrmw volatile max ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !04494 ret void4495}4496 4497define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {4498; GCN1-LABEL: atomic_max_i64_ret_addr64:4499; GCN1: ; %bb.0: ; %entry4500; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD04501; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD14502; GCN1-NEXT: s_mov_b32 s18, -14503; GCN1-NEXT: s_mov_b32 s19, 0xe8f0004504; GCN1-NEXT: s_add_u32 s16, s16, s114505; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x94506; GCN1-NEXT: s_load_dword s2, s[4:5], 0x414507; GCN1-NEXT: s_addc_u32 s17, s17, 04508; GCN1-NEXT: s_waitcnt lgkmcnt(0)4509; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 34510; GCN1-NEXT: s_add_u32 s0, s8, s04511; GCN1-NEXT: s_addc_u32 s1, s9, s14512; GCN1-NEXT: s_cmp_eq_u32 s1, s24513; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 04514; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]4515; GCN1-NEXT: s_cbranch_vccz .LBB31_24516; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global4517; GCN1-NEXT: v_mov_b32_e32 v0, s04518; GCN1-NEXT: v_mov_b32_e32 v2, s124519; GCN1-NEXT: v_mov_b32_e32 v1, s14520; GCN1-NEXT: v_mov_b32_e32 v3, s134521; GCN1-NEXT: flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc4522; GCN1-NEXT: s_waitcnt lgkmcnt(0)4523; GCN1-NEXT: s_cbranch_execz .LBB31_34524; GCN1-NEXT: s_branch .LBB31_44525; GCN1-NEXT: .LBB31_2:4526; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr14527; GCN1-NEXT: .LBB31_3: ; %atomicrmw.private4528; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 04529; GCN1-NEXT: v_mov_b32_e32 v5, s124530; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec4531; GCN1-NEXT: s_cselect_b32 s0, s0, -14532; GCN1-NEXT: v_mov_b32_e32 v2, s04533; GCN1-NEXT: s_add_i32 s0, s0, 44534; GCN1-NEXT: v_mov_b32_e32 v3, s04535; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen4536; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen4537; GCN1-NEXT: v_mov_b32_e32 v4, s134538; GCN1-NEXT: s_waitcnt vmcnt(0)4539; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]4540; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc4541; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc4542; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen4543; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen4544; GCN1-NEXT: .LBB31_4: ; %atomicrmw.end4545; GCN1-NEXT: v_mov_b32_e32 v2, s104546; GCN1-NEXT: v_mov_b32_e32 v3, s114547; GCN1-NEXT: s_waitcnt vmcnt(0)4548; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]4549; GCN1-NEXT: s_endpgm4550;4551; GCN2-LABEL: atomic_max_i64_ret_addr64:4552; GCN2: ; %bb.0: ; %entry4553; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD04554; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD14555; GCN2-NEXT: s_mov_b32 s90, -14556; GCN2-NEXT: s_mov_b32 s91, 0xe800004557; GCN2-NEXT: s_add_u32 s88, s88, s114558; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x244559; GCN2-NEXT: s_load_dword s2, s[4:5], 0x1044560; GCN2-NEXT: s_addc_u32 s89, s89, 04561; GCN2-NEXT: s_waitcnt lgkmcnt(0)4562; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 34563; GCN2-NEXT: s_add_u32 s0, s8, s04564; GCN2-NEXT: s_addc_u32 s1, s9, s14565; GCN2-NEXT: s_cmp_eq_u32 s1, s24566; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 04567; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]4568; GCN2-NEXT: s_cbranch_vccz .LBB31_24569; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global4570; GCN2-NEXT: v_mov_b32_e32 v0, s04571; GCN2-NEXT: v_mov_b32_e32 v2, s124572; GCN2-NEXT: v_mov_b32_e32 v1, s14573; GCN2-NEXT: v_mov_b32_e32 v3, s134574; GCN2-NEXT: flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc4575; GCN2-NEXT: s_waitcnt lgkmcnt(0)4576; GCN2-NEXT: s_cbranch_execz .LBB31_34577; GCN2-NEXT: s_branch .LBB31_44578; GCN2-NEXT: .LBB31_2:4579; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr14580; GCN2-NEXT: .LBB31_3: ; %atomicrmw.private4581; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 04582; GCN2-NEXT: s_cselect_b32 s0, s0, -14583; GCN2-NEXT: v_mov_b32_e32 v2, s04584; GCN2-NEXT: s_add_i32 s0, s0, 44585; GCN2-NEXT: v_mov_b32_e32 v3, s04586; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen4587; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen4588; GCN2-NEXT: v_mov_b32_e32 v5, s124589; GCN2-NEXT: v_mov_b32_e32 v4, s134590; GCN2-NEXT: s_waitcnt vmcnt(0)4591; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]4592; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc4593; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc4594; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen4595; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen4596; GCN2-NEXT: .LBB31_4: ; %atomicrmw.end4597; GCN2-NEXT: v_mov_b32_e32 v2, s104598; GCN2-NEXT: v_mov_b32_e32 v3, s114599; GCN2-NEXT: s_waitcnt vmcnt(0)4600; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]4601; GCN2-NEXT: s_endpgm4602;4603; GFX12-LABEL: atomic_max_i64_ret_addr64:4604; GFX12: ; %bb.0: ; %entry4605; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x244606; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base4607; GFX12-NEXT: s_wait_kmcnt 0x04608; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 34609; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)4610; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]4611; GFX12-NEXT: s_cmp_eq_u32 s1, s94612; GFX12-NEXT: s_cselect_b32 s6, -1, 04613; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4614; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s64615; GFX12-NEXT: s_cbranch_vccz .LBB31_24616; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global4617; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14618; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s54619; GFX12-NEXT: flat_atomic_max_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE4620; GFX12-NEXT: s_wait_loadcnt_dscnt 0x04621; GFX12-NEXT: global_inv scope:SCOPE_SE4622; GFX12-NEXT: s_cbranch_execz .LBB31_34623; GFX12-NEXT: s_branch .LBB31_44624; GFX12-NEXT: .LBB31_2:4625; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr14626; GFX12-NEXT: .LBB31_3: ; %atomicrmw.private4627; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 04628; GFX12-NEXT: s_cselect_b32 s0, s0, -14629; GFX12-NEXT: scratch_load_b64 v[0:1], off, s04630; GFX12-NEXT: s_wait_loadcnt 0x04631; GFX12-NEXT: v_cmp_lt_i64_e32 vcc_lo, s[4:5], v[0:1]4632; GFX12-NEXT: v_cndmask_b32_e32 v3, s5, v1, vcc_lo4633; GFX12-NEXT: v_cndmask_b32_e32 v2, s4, v0, vcc_lo4634; GFX12-NEXT: scratch_store_b64 off, v[2:3], s04635; GFX12-NEXT: .LBB31_4: ; %atomicrmw.end4636; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s34637; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]4638; GFX12-NEXT: s_endpgm4639entry:4640 %ptr = getelementptr inbounds i64, ptr %out, i64 %index4641 %tmp0 = atomicrmw volatile max ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !04642 store i64 %tmp0, ptr %out24643 ret void4644}4645 4646define amdgpu_kernel void @atomic_umax_i64_offset(ptr %out, i64 %in) {4647; GCN1-LABEL: atomic_umax_i64_offset:4648; GCN1: ; %bb.0: ; %entry4649; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD04650; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD14651; GCN1-NEXT: s_mov_b32 s14, -14652; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94653; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d4654; GCN1-NEXT: s_mov_b32 s15, 0xe8f0004655; GCN1-NEXT: s_add_u32 s12, s12, s114656; GCN1-NEXT: s_addc_u32 s13, s13, 04657; GCN1-NEXT: s_waitcnt lgkmcnt(0)4658; GCN1-NEXT: s_add_u32 s0, s0, 324659; GCN1-NEXT: s_addc_u32 s1, s1, 04660; GCN1-NEXT: s_cmp_eq_u32 s1, s44661; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 04662; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]4663; GCN1-NEXT: s_mov_b64 s[4:5], -14664; GCN1-NEXT: s_cbranch_vccnz .LBB32_34665; GCN1-NEXT: ; %bb.1: ; %Flow4666; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]4667; GCN1-NEXT: s_cbranch_vccz .LBB32_44668; GCN1-NEXT: .LBB32_2: ; %atomicrmw.phi4669; GCN1-NEXT: s_endpgm4670; GCN1-NEXT: .LBB32_3: ; %atomicrmw.global4671; GCN1-NEXT: v_mov_b32_e32 v0, s04672; GCN1-NEXT: v_mov_b32_e32 v2, s24673; GCN1-NEXT: v_mov_b32_e32 v1, s14674; GCN1-NEXT: v_mov_b32_e32 v3, s34675; GCN1-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3]4676; GCN1-NEXT: s_waitcnt lgkmcnt(0)4677; GCN1-NEXT: s_cbranch_execnz .LBB32_24678; GCN1-NEXT: .LBB32_4: ; %atomicrmw.private4679; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 04680; GCN1-NEXT: v_mov_b32_e32 v5, s24681; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec4682; GCN1-NEXT: s_cselect_b32 s0, s0, -14683; GCN1-NEXT: v_mov_b32_e32 v2, s04684; GCN1-NEXT: s_add_i32 s0, s0, 44685; GCN1-NEXT: v_mov_b32_e32 v3, s04686; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen4687; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen4688; GCN1-NEXT: v_mov_b32_e32 v4, s34689; GCN1-NEXT: s_waitcnt vmcnt(0)4690; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]4691; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc4692; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc4693; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen4694; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen4695; GCN1-NEXT: s_endpgm4696;4697; GCN2-LABEL: atomic_umax_i64_offset:4698; GCN2: ; %bb.0: ; %entry4699; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD04700; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD14701; GCN2-NEXT: s_mov_b32 s90, -14702; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244703; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf44704; GCN2-NEXT: s_mov_b32 s91, 0xe800004705; GCN2-NEXT: s_add_u32 s88, s88, s114706; GCN2-NEXT: s_addc_u32 s89, s89, 04707; GCN2-NEXT: s_waitcnt lgkmcnt(0)4708; GCN2-NEXT: s_add_u32 s0, s0, 324709; GCN2-NEXT: s_addc_u32 s1, s1, 04710; GCN2-NEXT: s_cmp_eq_u32 s1, s44711; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 04712; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]4713; GCN2-NEXT: s_mov_b64 s[4:5], -14714; GCN2-NEXT: s_cbranch_vccnz .LBB32_34715; GCN2-NEXT: ; %bb.1: ; %Flow4716; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]4717; GCN2-NEXT: s_cbranch_vccz .LBB32_44718; GCN2-NEXT: .LBB32_2: ; %atomicrmw.phi4719; GCN2-NEXT: s_endpgm4720; GCN2-NEXT: .LBB32_3: ; %atomicrmw.global4721; GCN2-NEXT: v_mov_b32_e32 v0, s04722; GCN2-NEXT: v_mov_b32_e32 v2, s24723; GCN2-NEXT: v_mov_b32_e32 v1, s14724; GCN2-NEXT: v_mov_b32_e32 v3, s34725; GCN2-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3]4726; GCN2-NEXT: s_waitcnt lgkmcnt(0)4727; GCN2-NEXT: s_cbranch_execnz .LBB32_24728; GCN2-NEXT: .LBB32_4: ; %atomicrmw.private4729; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 04730; GCN2-NEXT: s_cselect_b32 s0, s0, -14731; GCN2-NEXT: v_mov_b32_e32 v2, s04732; GCN2-NEXT: s_add_i32 s0, s0, 44733; GCN2-NEXT: v_mov_b32_e32 v3, s04734; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen4735; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen4736; GCN2-NEXT: v_mov_b32_e32 v5, s24737; GCN2-NEXT: v_mov_b32_e32 v4, s34738; GCN2-NEXT: s_waitcnt vmcnt(0)4739; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]4740; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc4741; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc4742; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen4743; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen4744; GCN2-NEXT: s_endpgm4745;4746; GFX12-LABEL: atomic_umax_i64_offset:4747; GFX12: ; %bb.0: ; %entry4748; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x244749; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base4750; GFX12-NEXT: s_wait_kmcnt 0x04751; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 324752; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)4753; GFX12-NEXT: s_cmp_eq_u32 s1, s74754; GFX12-NEXT: s_cselect_b32 s4, -1, 04755; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s44756; GFX12-NEXT: s_mov_b32 s4, -14757; GFX12-NEXT: s_cbranch_vccnz .LBB32_34758; GFX12-NEXT: ; %bb.1: ; %Flow4759; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s44760; GFX12-NEXT: s_cbranch_vccz .LBB32_44761; GFX12-NEXT: .LBB32_2: ; %atomicrmw.phi4762; GFX12-NEXT: s_endpgm4763; GFX12-NEXT: .LBB32_3: ; %atomicrmw.global4764; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14765; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s34766; GFX12-NEXT: flat_atomic_max_u64 v[0:1], v[2:3] scope:SCOPE_SE4767; GFX12-NEXT: s_wait_storecnt_dscnt 0x04768; GFX12-NEXT: global_inv scope:SCOPE_SE4769; GFX12-NEXT: s_cbranch_execnz .LBB32_24770; GFX12-NEXT: .LBB32_4: ; %atomicrmw.private4771; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 04772; GFX12-NEXT: s_cselect_b32 s0, s0, -14773; GFX12-NEXT: scratch_load_b64 v[0:1], off, s04774; GFX12-NEXT: s_wait_loadcnt 0x04775; GFX12-NEXT: v_cmp_lt_u64_e32 vcc_lo, s[2:3], v[0:1]4776; GFX12-NEXT: v_cndmask_b32_e32 v1, s3, v1, vcc_lo4777; GFX12-NEXT: v_cndmask_b32_e32 v0, s2, v0, vcc_lo4778; GFX12-NEXT: scratch_store_b64 off, v[0:1], s04779; GFX12-NEXT: s_endpgm4780entry:4781 %gep = getelementptr inbounds i64, ptr %out, i64 44782 %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !04783 ret void4784}4785 4786define amdgpu_kernel void @atomic_umax_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {4787; GCN1-LABEL: atomic_umax_i64_ret_offset:4788; GCN1: ; %bb.0: ; %entry4789; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD04790; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD14791; GCN1-NEXT: s_mov_b32 s14, -14792; GCN1-NEXT: s_mov_b32 s15, 0xe8f0004793; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94794; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f4795; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd4796; GCN1-NEXT: s_add_u32 s12, s12, s114797; GCN1-NEXT: s_addc_u32 s13, s13, 04798; GCN1-NEXT: s_waitcnt lgkmcnt(0)4799; GCN1-NEXT: s_add_u32 s0, s0, 324800; GCN1-NEXT: s_addc_u32 s1, s1, 04801; GCN1-NEXT: s_cmp_eq_u32 s1, s64802; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 04803; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]4804; GCN1-NEXT: s_cbranch_vccz .LBB33_24805; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global4806; GCN1-NEXT: v_mov_b32_e32 v0, s04807; GCN1-NEXT: v_mov_b32_e32 v2, s44808; GCN1-NEXT: v_mov_b32_e32 v1, s14809; GCN1-NEXT: v_mov_b32_e32 v3, s54810; GCN1-NEXT: flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc4811; GCN1-NEXT: s_waitcnt lgkmcnt(0)4812; GCN1-NEXT: s_cbranch_execz .LBB33_34813; GCN1-NEXT: s_branch .LBB33_44814; GCN1-NEXT: .LBB33_2:4815; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr14816; GCN1-NEXT: .LBB33_3: ; %atomicrmw.private4817; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 04818; GCN1-NEXT: v_mov_b32_e32 v5, s44819; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec4820; GCN1-NEXT: s_cselect_b32 s0, s0, -14821; GCN1-NEXT: v_mov_b32_e32 v2, s04822; GCN1-NEXT: s_add_i32 s0, s0, 44823; GCN1-NEXT: v_mov_b32_e32 v3, s04824; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen4825; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen4826; GCN1-NEXT: v_mov_b32_e32 v4, s54827; GCN1-NEXT: s_waitcnt vmcnt(0)4828; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]4829; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc4830; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc4831; GCN1-NEXT: buffer_store_dword v5, v2, s[12:15], 0 offen4832; GCN1-NEXT: buffer_store_dword v4, v3, s[12:15], 0 offen4833; GCN1-NEXT: .LBB33_4: ; %atomicrmw.end4834; GCN1-NEXT: v_mov_b32_e32 v2, s24835; GCN1-NEXT: v_mov_b32_e32 v3, s34836; GCN1-NEXT: s_waitcnt vmcnt(0)4837; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]4838; GCN1-NEXT: s_endpgm4839;4840; GCN2-LABEL: atomic_umax_i64_ret_offset:4841; GCN2: ; %bb.0: ; %entry4842; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD04843; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD14844; GCN2-NEXT: s_mov_b32 s90, -14845; GCN2-NEXT: s_mov_b32 s91, 0xe800004846; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244847; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc4848; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x344849; GCN2-NEXT: s_add_u32 s88, s88, s114850; GCN2-NEXT: s_addc_u32 s89, s89, 04851; GCN2-NEXT: s_waitcnt lgkmcnt(0)4852; GCN2-NEXT: s_add_u32 s0, s0, 324853; GCN2-NEXT: s_addc_u32 s1, s1, 04854; GCN2-NEXT: s_cmp_eq_u32 s1, s64855; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 04856; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]4857; GCN2-NEXT: s_cbranch_vccz .LBB33_24858; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global4859; GCN2-NEXT: v_mov_b32_e32 v0, s04860; GCN2-NEXT: v_mov_b32_e32 v2, s44861; GCN2-NEXT: v_mov_b32_e32 v1, s14862; GCN2-NEXT: v_mov_b32_e32 v3, s54863; GCN2-NEXT: flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc4864; GCN2-NEXT: s_waitcnt lgkmcnt(0)4865; GCN2-NEXT: s_cbranch_execz .LBB33_34866; GCN2-NEXT: s_branch .LBB33_44867; GCN2-NEXT: .LBB33_2:4868; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr14869; GCN2-NEXT: .LBB33_3: ; %atomicrmw.private4870; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 04871; GCN2-NEXT: s_cselect_b32 s0, s0, -14872; GCN2-NEXT: v_mov_b32_e32 v2, s04873; GCN2-NEXT: s_add_i32 s0, s0, 44874; GCN2-NEXT: v_mov_b32_e32 v3, s04875; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen4876; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen4877; GCN2-NEXT: v_mov_b32_e32 v5, s44878; GCN2-NEXT: v_mov_b32_e32 v4, s54879; GCN2-NEXT: s_waitcnt vmcnt(0)4880; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]4881; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc4882; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc4883; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen4884; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen4885; GCN2-NEXT: .LBB33_4: ; %atomicrmw.end4886; GCN2-NEXT: v_mov_b32_e32 v2, s24887; GCN2-NEXT: v_mov_b32_e32 v3, s34888; GCN2-NEXT: s_waitcnt vmcnt(0)4889; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]4890; GCN2-NEXT: s_endpgm4891;4892; GFX12-LABEL: atomic_umax_i64_ret_offset:4893; GFX12: ; %bb.0: ; %entry4894; GFX12-NEXT: s_clause 0x14895; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x244896; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x344897; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base4898; GFX12-NEXT: s_wait_kmcnt 0x04899; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 324900; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)4901; GFX12-NEXT: s_cmp_eq_u32 s1, s74902; GFX12-NEXT: s_cselect_b32 s6, -1, 04903; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s64904; GFX12-NEXT: s_cbranch_vccz .LBB33_24905; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global4906; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14907; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s54908; GFX12-NEXT: flat_atomic_max_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE4909; GFX12-NEXT: s_wait_loadcnt_dscnt 0x04910; GFX12-NEXT: global_inv scope:SCOPE_SE4911; GFX12-NEXT: s_cbranch_execz .LBB33_34912; GFX12-NEXT: s_branch .LBB33_44913; GFX12-NEXT: .LBB33_2:4914; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr14915; GFX12-NEXT: .LBB33_3: ; %atomicrmw.private4916; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 04917; GFX12-NEXT: s_cselect_b32 s0, s0, -14918; GFX12-NEXT: scratch_load_b64 v[0:1], off, s04919; GFX12-NEXT: s_wait_loadcnt 0x04920; GFX12-NEXT: v_cmp_lt_u64_e32 vcc_lo, s[4:5], v[0:1]4921; GFX12-NEXT: v_cndmask_b32_e32 v3, s5, v1, vcc_lo4922; GFX12-NEXT: v_cndmask_b32_e32 v2, s4, v0, vcc_lo4923; GFX12-NEXT: scratch_store_b64 off, v[2:3], s04924; GFX12-NEXT: .LBB33_4: ; %atomicrmw.end4925; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s34926; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]4927; GFX12-NEXT: s_endpgm4928entry:4929 %gep = getelementptr inbounds i64, ptr %out, i64 44930 %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !04931 store i64 %tmp0, ptr %out24932 ret void4933}4934 4935define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {4936; GCN1-LABEL: atomic_umax_i64_addr64_offset:4937; GCN1: ; %bb.0: ; %entry4938; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD04939; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD14940; GCN1-NEXT: s_mov_b32 s14, -14941; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd4942; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94943; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f4944; GCN1-NEXT: s_mov_b32 s15, 0xe8f0004945; GCN1-NEXT: s_add_u32 s12, s12, s114946; GCN1-NEXT: s_addc_u32 s13, s13, 04947; GCN1-NEXT: s_waitcnt lgkmcnt(0)4948; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 34949; GCN1-NEXT: s_add_u32 s0, s0, s44950; GCN1-NEXT: s_addc_u32 s1, s1, s54951; GCN1-NEXT: s_add_u32 s0, s0, 324952; GCN1-NEXT: s_addc_u32 s1, s1, 04953; GCN1-NEXT: s_cmp_eq_u32 s1, s84954; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 04955; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]4956; GCN1-NEXT: s_mov_b64 s[4:5], -14957; GCN1-NEXT: s_cbranch_vccnz .LBB34_34958; GCN1-NEXT: ; %bb.1: ; %Flow4959; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]4960; GCN1-NEXT: s_cbranch_vccz .LBB34_44961; GCN1-NEXT: .LBB34_2: ; %atomicrmw.phi4962; GCN1-NEXT: s_endpgm4963; GCN1-NEXT: .LBB34_3: ; %atomicrmw.global4964; GCN1-NEXT: v_mov_b32_e32 v0, s04965; GCN1-NEXT: v_mov_b32_e32 v2, s24966; GCN1-NEXT: v_mov_b32_e32 v1, s14967; GCN1-NEXT: v_mov_b32_e32 v3, s34968; GCN1-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3]4969; GCN1-NEXT: s_waitcnt lgkmcnt(0)4970; GCN1-NEXT: s_cbranch_execnz .LBB34_24971; GCN1-NEXT: .LBB34_4: ; %atomicrmw.private4972; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 04973; GCN1-NEXT: v_mov_b32_e32 v5, s24974; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec4975; GCN1-NEXT: s_cselect_b32 s0, s0, -14976; GCN1-NEXT: v_mov_b32_e32 v2, s04977; GCN1-NEXT: s_add_i32 s0, s0, 44978; GCN1-NEXT: v_mov_b32_e32 v3, s04979; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen4980; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen4981; GCN1-NEXT: v_mov_b32_e32 v4, s34982; GCN1-NEXT: s_waitcnt vmcnt(0)4983; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]4984; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc4985; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc4986; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen4987; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen4988; GCN1-NEXT: s_endpgm4989;4990; GCN2-LABEL: atomic_umax_i64_addr64_offset:4991; GCN2: ; %bb.0: ; %entry4992; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD04993; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD14994; GCN2-NEXT: s_mov_b32 s90, -14995; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x344996; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244997; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc4998; GCN2-NEXT: s_mov_b32 s91, 0xe800004999; GCN2-NEXT: s_add_u32 s88, s88, s115000; GCN2-NEXT: s_addc_u32 s89, s89, 05001; GCN2-NEXT: s_waitcnt lgkmcnt(0)5002; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 35003; GCN2-NEXT: s_add_u32 s0, s0, s45004; GCN2-NEXT: s_addc_u32 s1, s1, s55005; GCN2-NEXT: s_add_u32 s0, s0, 325006; GCN2-NEXT: s_addc_u32 s1, s1, 05007; GCN2-NEXT: s_cmp_eq_u32 s1, s85008; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 05009; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]5010; GCN2-NEXT: s_mov_b64 s[4:5], -15011; GCN2-NEXT: s_cbranch_vccnz .LBB34_35012; GCN2-NEXT: ; %bb.1: ; %Flow5013; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]5014; GCN2-NEXT: s_cbranch_vccz .LBB34_45015; GCN2-NEXT: .LBB34_2: ; %atomicrmw.phi5016; GCN2-NEXT: s_endpgm5017; GCN2-NEXT: .LBB34_3: ; %atomicrmw.global5018; GCN2-NEXT: v_mov_b32_e32 v0, s05019; GCN2-NEXT: v_mov_b32_e32 v2, s25020; GCN2-NEXT: v_mov_b32_e32 v1, s15021; GCN2-NEXT: v_mov_b32_e32 v3, s35022; GCN2-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3]5023; GCN2-NEXT: s_waitcnt lgkmcnt(0)5024; GCN2-NEXT: s_cbranch_execnz .LBB34_25025; GCN2-NEXT: .LBB34_4: ; %atomicrmw.private5026; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 05027; GCN2-NEXT: s_cselect_b32 s0, s0, -15028; GCN2-NEXT: v_mov_b32_e32 v2, s05029; GCN2-NEXT: s_add_i32 s0, s0, 45030; GCN2-NEXT: v_mov_b32_e32 v3, s05031; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen5032; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen5033; GCN2-NEXT: v_mov_b32_e32 v5, s25034; GCN2-NEXT: v_mov_b32_e32 v4, s35035; GCN2-NEXT: s_waitcnt vmcnt(0)5036; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]5037; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc5038; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc5039; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen5040; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen5041; GCN2-NEXT: s_endpgm5042;5043; GFX12-LABEL: atomic_umax_i64_addr64_offset:5044; GFX12: ; %bb.0: ; %entry5045; GFX12-NEXT: s_clause 0x15046; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x345047; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x245048; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base5049; GFX12-NEXT: s_wait_kmcnt 0x05050; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 35051; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)5052; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]5053; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 325054; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)5055; GFX12-NEXT: s_cmp_eq_u32 s1, s75056; GFX12-NEXT: s_cselect_b32 s4, -1, 05057; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s45058; GFX12-NEXT: s_mov_b32 s4, -15059; GFX12-NEXT: s_cbranch_vccnz .LBB34_35060; GFX12-NEXT: ; %bb.1: ; %Flow5061; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s45062; GFX12-NEXT: s_cbranch_vccz .LBB34_45063; GFX12-NEXT: .LBB34_2: ; %atomicrmw.phi5064; GFX12-NEXT: s_endpgm5065; GFX12-NEXT: .LBB34_3: ; %atomicrmw.global5066; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15067; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s35068; GFX12-NEXT: flat_atomic_max_u64 v[0:1], v[2:3] scope:SCOPE_SE5069; GFX12-NEXT: s_wait_storecnt_dscnt 0x05070; GFX12-NEXT: global_inv scope:SCOPE_SE5071; GFX12-NEXT: s_cbranch_execnz .LBB34_25072; GFX12-NEXT: .LBB34_4: ; %atomicrmw.private5073; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 05074; GFX12-NEXT: s_cselect_b32 s0, s0, -15075; GFX12-NEXT: scratch_load_b64 v[0:1], off, s05076; GFX12-NEXT: s_wait_loadcnt 0x05077; GFX12-NEXT: v_cmp_lt_u64_e32 vcc_lo, s[2:3], v[0:1]5078; GFX12-NEXT: v_cndmask_b32_e32 v1, s3, v1, vcc_lo5079; GFX12-NEXT: v_cndmask_b32_e32 v0, s2, v0, vcc_lo5080; GFX12-NEXT: scratch_store_b64 off, v[0:1], s05081; GFX12-NEXT: s_endpgm5082entry:5083 %ptr = getelementptr inbounds i64, ptr %out, i64 %index5084 %gep = getelementptr inbounds i64, ptr %ptr, i64 45085 %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05086 ret void5087}5088 5089define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {5090; GCN1-LABEL: atomic_umax_i64_ret_addr64_offset:5091; GCN1: ; %bb.0: ; %entry5092; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD05093; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD15094; GCN1-NEXT: s_mov_b32 s18, -15095; GCN1-NEXT: s_mov_b32 s19, 0xe8f0005096; GCN1-NEXT: s_add_u32 s16, s16, s115097; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x95098; GCN1-NEXT: s_load_dword s2, s[4:5], 0x415099; GCN1-NEXT: s_addc_u32 s17, s17, 05100; GCN1-NEXT: s_waitcnt lgkmcnt(0)5101; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 35102; GCN1-NEXT: s_add_u32 s0, s8, s05103; GCN1-NEXT: s_addc_u32 s1, s9, s15104; GCN1-NEXT: s_add_u32 s0, s0, 325105; GCN1-NEXT: s_addc_u32 s1, s1, 05106; GCN1-NEXT: s_cmp_eq_u32 s1, s25107; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 05108; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]5109; GCN1-NEXT: s_cbranch_vccz .LBB35_25110; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global5111; GCN1-NEXT: v_mov_b32_e32 v0, s05112; GCN1-NEXT: v_mov_b32_e32 v2, s125113; GCN1-NEXT: v_mov_b32_e32 v1, s15114; GCN1-NEXT: v_mov_b32_e32 v3, s135115; GCN1-NEXT: flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc5116; GCN1-NEXT: s_waitcnt lgkmcnt(0)5117; GCN1-NEXT: s_cbranch_execz .LBB35_35118; GCN1-NEXT: s_branch .LBB35_45119; GCN1-NEXT: .LBB35_2:5120; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr15121; GCN1-NEXT: .LBB35_3: ; %atomicrmw.private5122; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 05123; GCN1-NEXT: v_mov_b32_e32 v5, s125124; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec5125; GCN1-NEXT: s_cselect_b32 s0, s0, -15126; GCN1-NEXT: v_mov_b32_e32 v2, s05127; GCN1-NEXT: s_add_i32 s0, s0, 45128; GCN1-NEXT: v_mov_b32_e32 v3, s05129; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen5130; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen5131; GCN1-NEXT: v_mov_b32_e32 v4, s135132; GCN1-NEXT: s_waitcnt vmcnt(0)5133; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]5134; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc5135; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc5136; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen5137; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen5138; GCN1-NEXT: .LBB35_4: ; %atomicrmw.end5139; GCN1-NEXT: v_mov_b32_e32 v2, s105140; GCN1-NEXT: v_mov_b32_e32 v3, s115141; GCN1-NEXT: s_waitcnt vmcnt(0)5142; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]5143; GCN1-NEXT: s_endpgm5144;5145; GCN2-LABEL: atomic_umax_i64_ret_addr64_offset:5146; GCN2: ; %bb.0: ; %entry5147; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD05148; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD15149; GCN2-NEXT: s_mov_b32 s90, -15150; GCN2-NEXT: s_mov_b32 s91, 0xe800005151; GCN2-NEXT: s_add_u32 s88, s88, s115152; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x245153; GCN2-NEXT: s_load_dword s2, s[4:5], 0x1045154; GCN2-NEXT: s_addc_u32 s89, s89, 05155; GCN2-NEXT: s_waitcnt lgkmcnt(0)5156; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 35157; GCN2-NEXT: s_add_u32 s0, s8, s05158; GCN2-NEXT: s_addc_u32 s1, s9, s15159; GCN2-NEXT: s_add_u32 s0, s0, 325160; GCN2-NEXT: s_addc_u32 s1, s1, 05161; GCN2-NEXT: s_cmp_eq_u32 s1, s25162; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 05163; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]5164; GCN2-NEXT: s_cbranch_vccz .LBB35_25165; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global5166; GCN2-NEXT: v_mov_b32_e32 v0, s05167; GCN2-NEXT: v_mov_b32_e32 v2, s125168; GCN2-NEXT: v_mov_b32_e32 v1, s15169; GCN2-NEXT: v_mov_b32_e32 v3, s135170; GCN2-NEXT: flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc5171; GCN2-NEXT: s_waitcnt lgkmcnt(0)5172; GCN2-NEXT: s_cbranch_execz .LBB35_35173; GCN2-NEXT: s_branch .LBB35_45174; GCN2-NEXT: .LBB35_2:5175; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr15176; GCN2-NEXT: .LBB35_3: ; %atomicrmw.private5177; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 05178; GCN2-NEXT: s_cselect_b32 s0, s0, -15179; GCN2-NEXT: v_mov_b32_e32 v2, s05180; GCN2-NEXT: s_add_i32 s0, s0, 45181; GCN2-NEXT: v_mov_b32_e32 v3, s05182; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen5183; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen5184; GCN2-NEXT: v_mov_b32_e32 v5, s125185; GCN2-NEXT: v_mov_b32_e32 v4, s135186; GCN2-NEXT: s_waitcnt vmcnt(0)5187; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]5188; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc5189; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc5190; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen5191; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen5192; GCN2-NEXT: .LBB35_4: ; %atomicrmw.end5193; GCN2-NEXT: v_mov_b32_e32 v2, s105194; GCN2-NEXT: v_mov_b32_e32 v3, s115195; GCN2-NEXT: s_waitcnt vmcnt(0)5196; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]5197; GCN2-NEXT: s_endpgm5198;5199; GFX12-LABEL: atomic_umax_i64_ret_addr64_offset:5200; GFX12: ; %bb.0: ; %entry5201; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x245202; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base5203; GFX12-NEXT: s_wait_kmcnt 0x05204; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 35205; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)5206; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]5207; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 325208; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)5209; GFX12-NEXT: s_cmp_eq_u32 s1, s95210; GFX12-NEXT: s_cselect_b32 s6, -1, 05211; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s65212; GFX12-NEXT: s_cbranch_vccz .LBB35_25213; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global5214; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15215; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s55216; GFX12-NEXT: flat_atomic_max_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE5217; GFX12-NEXT: s_wait_loadcnt_dscnt 0x05218; GFX12-NEXT: global_inv scope:SCOPE_SE5219; GFX12-NEXT: s_cbranch_execz .LBB35_35220; GFX12-NEXT: s_branch .LBB35_45221; GFX12-NEXT: .LBB35_2:5222; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr15223; GFX12-NEXT: .LBB35_3: ; %atomicrmw.private5224; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 05225; GFX12-NEXT: s_cselect_b32 s0, s0, -15226; GFX12-NEXT: scratch_load_b64 v[0:1], off, s05227; GFX12-NEXT: s_wait_loadcnt 0x05228; GFX12-NEXT: v_cmp_lt_u64_e32 vcc_lo, s[4:5], v[0:1]5229; GFX12-NEXT: v_cndmask_b32_e32 v3, s5, v1, vcc_lo5230; GFX12-NEXT: v_cndmask_b32_e32 v2, s4, v0, vcc_lo5231; GFX12-NEXT: scratch_store_b64 off, v[2:3], s05232; GFX12-NEXT: .LBB35_4: ; %atomicrmw.end5233; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s35234; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]5235; GFX12-NEXT: s_endpgm5236entry:5237 %ptr = getelementptr inbounds i64, ptr %out, i64 %index5238 %gep = getelementptr inbounds i64, ptr %ptr, i64 45239 %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05240 store i64 %tmp0, ptr %out25241 ret void5242}5243 5244define amdgpu_kernel void @atomic_umax_i64(ptr %out, i64 %in) {5245; GCN1-LABEL: atomic_umax_i64:5246; GCN1: ; %bb.0: ; %entry5247; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD05248; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD15249; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95250; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d5251; GCN1-NEXT: s_mov_b32 s14, -15252; GCN1-NEXT: s_mov_b32 s15, 0xe8f0005253; GCN1-NEXT: s_add_u32 s12, s12, s115254; GCN1-NEXT: s_addc_u32 s13, s13, 05255; GCN1-NEXT: s_waitcnt lgkmcnt(0)5256; GCN1-NEXT: s_cmp_eq_u32 s1, s45257; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 05258; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]5259; GCN1-NEXT: s_mov_b64 s[4:5], -15260; GCN1-NEXT: s_cbranch_vccnz .LBB36_35261; GCN1-NEXT: ; %bb.1: ; %Flow5262; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]5263; GCN1-NEXT: s_cbranch_vccz .LBB36_45264; GCN1-NEXT: .LBB36_2: ; %atomicrmw.phi5265; GCN1-NEXT: s_endpgm5266; GCN1-NEXT: .LBB36_3: ; %atomicrmw.global5267; GCN1-NEXT: v_mov_b32_e32 v0, s05268; GCN1-NEXT: v_mov_b32_e32 v2, s25269; GCN1-NEXT: v_mov_b32_e32 v1, s15270; GCN1-NEXT: v_mov_b32_e32 v3, s35271; GCN1-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3]5272; GCN1-NEXT: s_waitcnt lgkmcnt(0)5273; GCN1-NEXT: s_cbranch_execnz .LBB36_25274; GCN1-NEXT: .LBB36_4: ; %atomicrmw.private5275; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 05276; GCN1-NEXT: v_mov_b32_e32 v5, s25277; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec5278; GCN1-NEXT: s_cselect_b32 s0, s0, -15279; GCN1-NEXT: v_mov_b32_e32 v2, s05280; GCN1-NEXT: s_add_i32 s0, s0, 45281; GCN1-NEXT: v_mov_b32_e32 v3, s05282; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen5283; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen5284; GCN1-NEXT: v_mov_b32_e32 v4, s35285; GCN1-NEXT: s_waitcnt vmcnt(0)5286; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]5287; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc5288; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc5289; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen5290; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen5291; GCN1-NEXT: s_endpgm5292;5293; GCN2-LABEL: atomic_umax_i64:5294; GCN2: ; %bb.0: ; %entry5295; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD05296; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD15297; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245298; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf45299; GCN2-NEXT: s_mov_b32 s90, -15300; GCN2-NEXT: s_mov_b32 s91, 0xe800005301; GCN2-NEXT: s_add_u32 s88, s88, s115302; GCN2-NEXT: s_addc_u32 s89, s89, 05303; GCN2-NEXT: s_waitcnt lgkmcnt(0)5304; GCN2-NEXT: s_cmp_eq_u32 s1, s45305; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 05306; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]5307; GCN2-NEXT: s_mov_b64 s[4:5], -15308; GCN2-NEXT: s_cbranch_vccnz .LBB36_35309; GCN2-NEXT: ; %bb.1: ; %Flow5310; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]5311; GCN2-NEXT: s_cbranch_vccz .LBB36_45312; GCN2-NEXT: .LBB36_2: ; %atomicrmw.phi5313; GCN2-NEXT: s_endpgm5314; GCN2-NEXT: .LBB36_3: ; %atomicrmw.global5315; GCN2-NEXT: v_mov_b32_e32 v0, s05316; GCN2-NEXT: v_mov_b32_e32 v2, s25317; GCN2-NEXT: v_mov_b32_e32 v1, s15318; GCN2-NEXT: v_mov_b32_e32 v3, s35319; GCN2-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3]5320; GCN2-NEXT: s_waitcnt lgkmcnt(0)5321; GCN2-NEXT: s_cbranch_execnz .LBB36_25322; GCN2-NEXT: .LBB36_4: ; %atomicrmw.private5323; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 05324; GCN2-NEXT: s_cselect_b32 s0, s0, -15325; GCN2-NEXT: v_mov_b32_e32 v2, s05326; GCN2-NEXT: s_add_i32 s0, s0, 45327; GCN2-NEXT: v_mov_b32_e32 v3, s05328; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen5329; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen5330; GCN2-NEXT: v_mov_b32_e32 v5, s25331; GCN2-NEXT: v_mov_b32_e32 v4, s35332; GCN2-NEXT: s_waitcnt vmcnt(0)5333; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]5334; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc5335; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc5336; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen5337; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen5338; GCN2-NEXT: s_endpgm5339;5340; GFX12-LABEL: atomic_umax_i64:5341; GFX12: ; %bb.0: ; %entry5342; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x245343; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base5344; GFX12-NEXT: s_wait_kmcnt 0x05345; GFX12-NEXT: s_cmp_eq_u32 s1, s75346; GFX12-NEXT: s_cselect_b32 s4, -1, 05347; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5348; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s45349; GFX12-NEXT: s_mov_b32 s4, -15350; GFX12-NEXT: s_cbranch_vccnz .LBB36_35351; GFX12-NEXT: ; %bb.1: ; %Flow5352; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s45353; GFX12-NEXT: s_cbranch_vccz .LBB36_45354; GFX12-NEXT: .LBB36_2: ; %atomicrmw.phi5355; GFX12-NEXT: s_endpgm5356; GFX12-NEXT: .LBB36_3: ; %atomicrmw.global5357; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15358; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s35359; GFX12-NEXT: flat_atomic_max_u64 v[0:1], v[2:3] scope:SCOPE_SE5360; GFX12-NEXT: s_wait_storecnt_dscnt 0x05361; GFX12-NEXT: global_inv scope:SCOPE_SE5362; GFX12-NEXT: s_cbranch_execnz .LBB36_25363; GFX12-NEXT: .LBB36_4: ; %atomicrmw.private5364; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 05365; GFX12-NEXT: s_cselect_b32 s0, s0, -15366; GFX12-NEXT: scratch_load_b64 v[0:1], off, s05367; GFX12-NEXT: s_wait_loadcnt 0x05368; GFX12-NEXT: v_cmp_lt_u64_e32 vcc_lo, s[2:3], v[0:1]5369; GFX12-NEXT: v_cndmask_b32_e32 v1, s3, v1, vcc_lo5370; GFX12-NEXT: v_cndmask_b32_e32 v0, s2, v0, vcc_lo5371; GFX12-NEXT: scratch_store_b64 off, v[0:1], s05372; GFX12-NEXT: s_endpgm5373entry:5374 %tmp0 = atomicrmw volatile umax ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05375 ret void5376}5377 5378define amdgpu_kernel void @atomic_umax_i64_ret(ptr %out, ptr %out2, i64 %in) {5379; GCN1-LABEL: atomic_umax_i64_ret:5380; GCN1: ; %bb.0: ; %entry5381; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD05382; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD15383; GCN1-NEXT: s_mov_b32 s14, -15384; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f5385; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95386; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd5387; GCN1-NEXT: s_mov_b32 s15, 0xe8f0005388; GCN1-NEXT: s_add_u32 s12, s12, s115389; GCN1-NEXT: s_addc_u32 s13, s13, 05390; GCN1-NEXT: s_waitcnt lgkmcnt(0)5391; GCN1-NEXT: s_cmp_eq_u32 s1, s65392; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 05393; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]5394; GCN1-NEXT: s_cbranch_vccz .LBB37_25395; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global5396; GCN1-NEXT: v_mov_b32_e32 v0, s05397; GCN1-NEXT: v_mov_b32_e32 v2, s45398; GCN1-NEXT: v_mov_b32_e32 v1, s15399; GCN1-NEXT: v_mov_b32_e32 v3, s55400; GCN1-NEXT: flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc5401; GCN1-NEXT: s_waitcnt lgkmcnt(0)5402; GCN1-NEXT: s_cbranch_execz .LBB37_35403; GCN1-NEXT: s_branch .LBB37_45404; GCN1-NEXT: .LBB37_2:5405; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr15406; GCN1-NEXT: .LBB37_3: ; %atomicrmw.private5407; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 05408; GCN1-NEXT: v_mov_b32_e32 v5, s45409; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec5410; GCN1-NEXT: s_cselect_b32 s0, s0, -15411; GCN1-NEXT: v_mov_b32_e32 v2, s05412; GCN1-NEXT: s_add_i32 s0, s0, 45413; GCN1-NEXT: v_mov_b32_e32 v3, s05414; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen5415; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen5416; GCN1-NEXT: v_mov_b32_e32 v4, s55417; GCN1-NEXT: s_waitcnt vmcnt(0)5418; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]5419; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc5420; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc5421; GCN1-NEXT: buffer_store_dword v5, v2, s[12:15], 0 offen5422; GCN1-NEXT: buffer_store_dword v4, v3, s[12:15], 0 offen5423; GCN1-NEXT: .LBB37_4: ; %atomicrmw.end5424; GCN1-NEXT: v_mov_b32_e32 v2, s25425; GCN1-NEXT: v_mov_b32_e32 v3, s35426; GCN1-NEXT: s_waitcnt vmcnt(0)5427; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]5428; GCN1-NEXT: s_endpgm5429;5430; GCN2-LABEL: atomic_umax_i64_ret:5431; GCN2: ; %bb.0: ; %entry5432; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD05433; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD15434; GCN2-NEXT: s_mov_b32 s90, -15435; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc5436; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245437; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x345438; GCN2-NEXT: s_mov_b32 s91, 0xe800005439; GCN2-NEXT: s_add_u32 s88, s88, s115440; GCN2-NEXT: s_addc_u32 s89, s89, 05441; GCN2-NEXT: s_waitcnt lgkmcnt(0)5442; GCN2-NEXT: s_cmp_eq_u32 s1, s65443; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 05444; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]5445; GCN2-NEXT: s_cbranch_vccz .LBB37_25446; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global5447; GCN2-NEXT: v_mov_b32_e32 v0, s05448; GCN2-NEXT: v_mov_b32_e32 v2, s45449; GCN2-NEXT: v_mov_b32_e32 v1, s15450; GCN2-NEXT: v_mov_b32_e32 v3, s55451; GCN2-NEXT: flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc5452; GCN2-NEXT: s_waitcnt lgkmcnt(0)5453; GCN2-NEXT: s_cbranch_execz .LBB37_35454; GCN2-NEXT: s_branch .LBB37_45455; GCN2-NEXT: .LBB37_2:5456; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr15457; GCN2-NEXT: .LBB37_3: ; %atomicrmw.private5458; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 05459; GCN2-NEXT: s_cselect_b32 s0, s0, -15460; GCN2-NEXT: v_mov_b32_e32 v2, s05461; GCN2-NEXT: s_add_i32 s0, s0, 45462; GCN2-NEXT: v_mov_b32_e32 v3, s05463; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen5464; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen5465; GCN2-NEXT: v_mov_b32_e32 v5, s45466; GCN2-NEXT: v_mov_b32_e32 v4, s55467; GCN2-NEXT: s_waitcnt vmcnt(0)5468; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]5469; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc5470; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc5471; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen5472; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen5473; GCN2-NEXT: .LBB37_4: ; %atomicrmw.end5474; GCN2-NEXT: v_mov_b32_e32 v2, s25475; GCN2-NEXT: v_mov_b32_e32 v3, s35476; GCN2-NEXT: s_waitcnt vmcnt(0)5477; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]5478; GCN2-NEXT: s_endpgm5479;5480; GFX12-LABEL: atomic_umax_i64_ret:5481; GFX12: ; %bb.0: ; %entry5482; GFX12-NEXT: s_clause 0x15483; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x245484; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x345485; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base5486; GFX12-NEXT: s_wait_kmcnt 0x05487; GFX12-NEXT: s_cmp_eq_u32 s1, s75488; GFX12-NEXT: s_cselect_b32 s6, -1, 05489; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5490; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s65491; GFX12-NEXT: s_cbranch_vccz .LBB37_25492; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global5493; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15494; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s55495; GFX12-NEXT: flat_atomic_max_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE5496; GFX12-NEXT: s_wait_loadcnt_dscnt 0x05497; GFX12-NEXT: global_inv scope:SCOPE_SE5498; GFX12-NEXT: s_cbranch_execz .LBB37_35499; GFX12-NEXT: s_branch .LBB37_45500; GFX12-NEXT: .LBB37_2:5501; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr15502; GFX12-NEXT: .LBB37_3: ; %atomicrmw.private5503; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 05504; GFX12-NEXT: s_cselect_b32 s0, s0, -15505; GFX12-NEXT: scratch_load_b64 v[0:1], off, s05506; GFX12-NEXT: s_wait_loadcnt 0x05507; GFX12-NEXT: v_cmp_lt_u64_e32 vcc_lo, s[4:5], v[0:1]5508; GFX12-NEXT: v_cndmask_b32_e32 v3, s5, v1, vcc_lo5509; GFX12-NEXT: v_cndmask_b32_e32 v2, s4, v0, vcc_lo5510; GFX12-NEXT: scratch_store_b64 off, v[2:3], s05511; GFX12-NEXT: .LBB37_4: ; %atomicrmw.end5512; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s35513; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]5514; GFX12-NEXT: s_endpgm5515entry:5516 %tmp0 = atomicrmw volatile umax ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05517 store i64 %tmp0, ptr %out25518 ret void5519}5520 5521define amdgpu_kernel void @atomic_umax_i64_addr64(ptr %out, i64 %in, i64 %index) {5522; GCN1-LABEL: atomic_umax_i64_addr64:5523; GCN1: ; %bb.0: ; %entry5524; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD05525; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD15526; GCN1-NEXT: s_mov_b32 s14, -15527; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd5528; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95529; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f5530; GCN1-NEXT: s_mov_b32 s15, 0xe8f0005531; GCN1-NEXT: s_add_u32 s12, s12, s115532; GCN1-NEXT: s_addc_u32 s13, s13, 05533; GCN1-NEXT: s_waitcnt lgkmcnt(0)5534; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 35535; GCN1-NEXT: s_add_u32 s0, s0, s45536; GCN1-NEXT: s_addc_u32 s1, s1, s55537; GCN1-NEXT: s_cmp_eq_u32 s1, s85538; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 05539; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]5540; GCN1-NEXT: s_mov_b64 s[4:5], -15541; GCN1-NEXT: s_cbranch_vccnz .LBB38_35542; GCN1-NEXT: ; %bb.1: ; %Flow5543; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]5544; GCN1-NEXT: s_cbranch_vccz .LBB38_45545; GCN1-NEXT: .LBB38_2: ; %atomicrmw.phi5546; GCN1-NEXT: s_endpgm5547; GCN1-NEXT: .LBB38_3: ; %atomicrmw.global5548; GCN1-NEXT: v_mov_b32_e32 v0, s05549; GCN1-NEXT: v_mov_b32_e32 v2, s25550; GCN1-NEXT: v_mov_b32_e32 v1, s15551; GCN1-NEXT: v_mov_b32_e32 v3, s35552; GCN1-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3]5553; GCN1-NEXT: s_waitcnt lgkmcnt(0)5554; GCN1-NEXT: s_cbranch_execnz .LBB38_25555; GCN1-NEXT: .LBB38_4: ; %atomicrmw.private5556; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 05557; GCN1-NEXT: v_mov_b32_e32 v5, s25558; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec5559; GCN1-NEXT: s_cselect_b32 s0, s0, -15560; GCN1-NEXT: v_mov_b32_e32 v2, s05561; GCN1-NEXT: s_add_i32 s0, s0, 45562; GCN1-NEXT: v_mov_b32_e32 v3, s05563; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen5564; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen5565; GCN1-NEXT: v_mov_b32_e32 v4, s35566; GCN1-NEXT: s_waitcnt vmcnt(0)5567; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]5568; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc5569; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc5570; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen5571; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen5572; GCN1-NEXT: s_endpgm5573;5574; GCN2-LABEL: atomic_umax_i64_addr64:5575; GCN2: ; %bb.0: ; %entry5576; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD05577; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD15578; GCN2-NEXT: s_mov_b32 s90, -15579; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x345580; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245581; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc5582; GCN2-NEXT: s_mov_b32 s91, 0xe800005583; GCN2-NEXT: s_add_u32 s88, s88, s115584; GCN2-NEXT: s_addc_u32 s89, s89, 05585; GCN2-NEXT: s_waitcnt lgkmcnt(0)5586; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 35587; GCN2-NEXT: s_add_u32 s0, s0, s45588; GCN2-NEXT: s_addc_u32 s1, s1, s55589; GCN2-NEXT: s_cmp_eq_u32 s1, s85590; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 05591; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]5592; GCN2-NEXT: s_mov_b64 s[4:5], -15593; GCN2-NEXT: s_cbranch_vccnz .LBB38_35594; GCN2-NEXT: ; %bb.1: ; %Flow5595; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]5596; GCN2-NEXT: s_cbranch_vccz .LBB38_45597; GCN2-NEXT: .LBB38_2: ; %atomicrmw.phi5598; GCN2-NEXT: s_endpgm5599; GCN2-NEXT: .LBB38_3: ; %atomicrmw.global5600; GCN2-NEXT: v_mov_b32_e32 v0, s05601; GCN2-NEXT: v_mov_b32_e32 v2, s25602; GCN2-NEXT: v_mov_b32_e32 v1, s15603; GCN2-NEXT: v_mov_b32_e32 v3, s35604; GCN2-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3]5605; GCN2-NEXT: s_waitcnt lgkmcnt(0)5606; GCN2-NEXT: s_cbranch_execnz .LBB38_25607; GCN2-NEXT: .LBB38_4: ; %atomicrmw.private5608; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 05609; GCN2-NEXT: s_cselect_b32 s0, s0, -15610; GCN2-NEXT: v_mov_b32_e32 v2, s05611; GCN2-NEXT: s_add_i32 s0, s0, 45612; GCN2-NEXT: v_mov_b32_e32 v3, s05613; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen5614; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen5615; GCN2-NEXT: v_mov_b32_e32 v5, s25616; GCN2-NEXT: v_mov_b32_e32 v4, s35617; GCN2-NEXT: s_waitcnt vmcnt(0)5618; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]5619; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc5620; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc5621; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen5622; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen5623; GCN2-NEXT: s_endpgm5624;5625; GFX12-LABEL: atomic_umax_i64_addr64:5626; GFX12: ; %bb.0: ; %entry5627; GFX12-NEXT: s_clause 0x15628; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x345629; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x245630; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base5631; GFX12-NEXT: s_wait_kmcnt 0x05632; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 35633; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)5634; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]5635; GFX12-NEXT: s_cmp_eq_u32 s1, s75636; GFX12-NEXT: s_cselect_b32 s4, -1, 05637; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5638; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s45639; GFX12-NEXT: s_mov_b32 s4, -15640; GFX12-NEXT: s_cbranch_vccnz .LBB38_35641; GFX12-NEXT: ; %bb.1: ; %Flow5642; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s45643; GFX12-NEXT: s_cbranch_vccz .LBB38_45644; GFX12-NEXT: .LBB38_2: ; %atomicrmw.phi5645; GFX12-NEXT: s_endpgm5646; GFX12-NEXT: .LBB38_3: ; %atomicrmw.global5647; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15648; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s35649; GFX12-NEXT: flat_atomic_max_u64 v[0:1], v[2:3] scope:SCOPE_SE5650; GFX12-NEXT: s_wait_storecnt_dscnt 0x05651; GFX12-NEXT: global_inv scope:SCOPE_SE5652; GFX12-NEXT: s_cbranch_execnz .LBB38_25653; GFX12-NEXT: .LBB38_4: ; %atomicrmw.private5654; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 05655; GFX12-NEXT: s_cselect_b32 s0, s0, -15656; GFX12-NEXT: scratch_load_b64 v[0:1], off, s05657; GFX12-NEXT: s_wait_loadcnt 0x05658; GFX12-NEXT: v_cmp_lt_u64_e32 vcc_lo, s[2:3], v[0:1]5659; GFX12-NEXT: v_cndmask_b32_e32 v1, s3, v1, vcc_lo5660; GFX12-NEXT: v_cndmask_b32_e32 v0, s2, v0, vcc_lo5661; GFX12-NEXT: scratch_store_b64 off, v[0:1], s05662; GFX12-NEXT: s_endpgm5663entry:5664 %ptr = getelementptr inbounds i64, ptr %out, i64 %index5665 %tmp0 = atomicrmw volatile umax ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05666 ret void5667}5668 5669define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {5670; GCN1-LABEL: atomic_umax_i64_ret_addr64:5671; GCN1: ; %bb.0: ; %entry5672; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD05673; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD15674; GCN1-NEXT: s_mov_b32 s18, -15675; GCN1-NEXT: s_mov_b32 s19, 0xe8f0005676; GCN1-NEXT: s_add_u32 s16, s16, s115677; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x95678; GCN1-NEXT: s_load_dword s2, s[4:5], 0x415679; GCN1-NEXT: s_addc_u32 s17, s17, 05680; GCN1-NEXT: s_waitcnt lgkmcnt(0)5681; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 35682; GCN1-NEXT: s_add_u32 s0, s8, s05683; GCN1-NEXT: s_addc_u32 s1, s9, s15684; GCN1-NEXT: s_cmp_eq_u32 s1, s25685; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 05686; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]5687; GCN1-NEXT: s_cbranch_vccz .LBB39_25688; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global5689; GCN1-NEXT: v_mov_b32_e32 v0, s05690; GCN1-NEXT: v_mov_b32_e32 v2, s125691; GCN1-NEXT: v_mov_b32_e32 v1, s15692; GCN1-NEXT: v_mov_b32_e32 v3, s135693; GCN1-NEXT: flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc5694; GCN1-NEXT: s_waitcnt lgkmcnt(0)5695; GCN1-NEXT: s_cbranch_execz .LBB39_35696; GCN1-NEXT: s_branch .LBB39_45697; GCN1-NEXT: .LBB39_2:5698; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr15699; GCN1-NEXT: .LBB39_3: ; %atomicrmw.private5700; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 05701; GCN1-NEXT: v_mov_b32_e32 v5, s125702; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec5703; GCN1-NEXT: s_cselect_b32 s0, s0, -15704; GCN1-NEXT: v_mov_b32_e32 v2, s05705; GCN1-NEXT: s_add_i32 s0, s0, 45706; GCN1-NEXT: v_mov_b32_e32 v3, s05707; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen5708; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen5709; GCN1-NEXT: v_mov_b32_e32 v4, s135710; GCN1-NEXT: s_waitcnt vmcnt(0)5711; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]5712; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc5713; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc5714; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen5715; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen5716; GCN1-NEXT: .LBB39_4: ; %atomicrmw.end5717; GCN1-NEXT: v_mov_b32_e32 v2, s105718; GCN1-NEXT: v_mov_b32_e32 v3, s115719; GCN1-NEXT: s_waitcnt vmcnt(0)5720; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]5721; GCN1-NEXT: s_endpgm5722;5723; GCN2-LABEL: atomic_umax_i64_ret_addr64:5724; GCN2: ; %bb.0: ; %entry5725; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD05726; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD15727; GCN2-NEXT: s_mov_b32 s90, -15728; GCN2-NEXT: s_mov_b32 s91, 0xe800005729; GCN2-NEXT: s_add_u32 s88, s88, s115730; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x245731; GCN2-NEXT: s_load_dword s2, s[4:5], 0x1045732; GCN2-NEXT: s_addc_u32 s89, s89, 05733; GCN2-NEXT: s_waitcnt lgkmcnt(0)5734; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 35735; GCN2-NEXT: s_add_u32 s0, s8, s05736; GCN2-NEXT: s_addc_u32 s1, s9, s15737; GCN2-NEXT: s_cmp_eq_u32 s1, s25738; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 05739; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]5740; GCN2-NEXT: s_cbranch_vccz .LBB39_25741; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global5742; GCN2-NEXT: v_mov_b32_e32 v0, s05743; GCN2-NEXT: v_mov_b32_e32 v2, s125744; GCN2-NEXT: v_mov_b32_e32 v1, s15745; GCN2-NEXT: v_mov_b32_e32 v3, s135746; GCN2-NEXT: flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc5747; GCN2-NEXT: s_waitcnt lgkmcnt(0)5748; GCN2-NEXT: s_cbranch_execz .LBB39_35749; GCN2-NEXT: s_branch .LBB39_45750; GCN2-NEXT: .LBB39_2:5751; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr15752; GCN2-NEXT: .LBB39_3: ; %atomicrmw.private5753; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 05754; GCN2-NEXT: s_cselect_b32 s0, s0, -15755; GCN2-NEXT: v_mov_b32_e32 v2, s05756; GCN2-NEXT: s_add_i32 s0, s0, 45757; GCN2-NEXT: v_mov_b32_e32 v3, s05758; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen5759; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen5760; GCN2-NEXT: v_mov_b32_e32 v5, s125761; GCN2-NEXT: v_mov_b32_e32 v4, s135762; GCN2-NEXT: s_waitcnt vmcnt(0)5763; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]5764; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc5765; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc5766; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen5767; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen5768; GCN2-NEXT: .LBB39_4: ; %atomicrmw.end5769; GCN2-NEXT: v_mov_b32_e32 v2, s105770; GCN2-NEXT: v_mov_b32_e32 v3, s115771; GCN2-NEXT: s_waitcnt vmcnt(0)5772; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]5773; GCN2-NEXT: s_endpgm5774;5775; GFX12-LABEL: atomic_umax_i64_ret_addr64:5776; GFX12: ; %bb.0: ; %entry5777; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x245778; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base5779; GFX12-NEXT: s_wait_kmcnt 0x05780; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 35781; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)5782; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]5783; GFX12-NEXT: s_cmp_eq_u32 s1, s95784; GFX12-NEXT: s_cselect_b32 s6, -1, 05785; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5786; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s65787; GFX12-NEXT: s_cbranch_vccz .LBB39_25788; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global5789; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15790; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s55791; GFX12-NEXT: flat_atomic_max_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE5792; GFX12-NEXT: s_wait_loadcnt_dscnt 0x05793; GFX12-NEXT: global_inv scope:SCOPE_SE5794; GFX12-NEXT: s_cbranch_execz .LBB39_35795; GFX12-NEXT: s_branch .LBB39_45796; GFX12-NEXT: .LBB39_2:5797; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr15798; GFX12-NEXT: .LBB39_3: ; %atomicrmw.private5799; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 05800; GFX12-NEXT: s_cselect_b32 s0, s0, -15801; GFX12-NEXT: scratch_load_b64 v[0:1], off, s05802; GFX12-NEXT: s_wait_loadcnt 0x05803; GFX12-NEXT: v_cmp_lt_u64_e32 vcc_lo, s[4:5], v[0:1]5804; GFX12-NEXT: v_cndmask_b32_e32 v3, s5, v1, vcc_lo5805; GFX12-NEXT: v_cndmask_b32_e32 v2, s4, v0, vcc_lo5806; GFX12-NEXT: scratch_store_b64 off, v[2:3], s05807; GFX12-NEXT: .LBB39_4: ; %atomicrmw.end5808; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s35809; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]5810; GFX12-NEXT: s_endpgm5811entry:5812 %ptr = getelementptr inbounds i64, ptr %out, i64 %index5813 %tmp0 = atomicrmw volatile umax ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05814 store i64 %tmp0, ptr %out25815 ret void5816}5817 5818define amdgpu_kernel void @atomic_min_i64_offset(ptr %out, i64 %in) {5819; GCN1-LABEL: atomic_min_i64_offset:5820; GCN1: ; %bb.0: ; %entry5821; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD05822; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD15823; GCN1-NEXT: s_mov_b32 s14, -15824; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95825; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d5826; GCN1-NEXT: s_mov_b32 s15, 0xe8f0005827; GCN1-NEXT: s_add_u32 s12, s12, s115828; GCN1-NEXT: s_addc_u32 s13, s13, 05829; GCN1-NEXT: s_waitcnt lgkmcnt(0)5830; GCN1-NEXT: s_add_u32 s0, s0, 325831; GCN1-NEXT: s_addc_u32 s1, s1, 05832; GCN1-NEXT: s_cmp_eq_u32 s1, s45833; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 05834; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]5835; GCN1-NEXT: s_mov_b64 s[4:5], -15836; GCN1-NEXT: s_cbranch_vccnz .LBB40_35837; GCN1-NEXT: ; %bb.1: ; %Flow5838; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]5839; GCN1-NEXT: s_cbranch_vccz .LBB40_45840; GCN1-NEXT: .LBB40_2: ; %atomicrmw.phi5841; GCN1-NEXT: s_endpgm5842; GCN1-NEXT: .LBB40_3: ; %atomicrmw.global5843; GCN1-NEXT: v_mov_b32_e32 v0, s05844; GCN1-NEXT: v_mov_b32_e32 v2, s25845; GCN1-NEXT: v_mov_b32_e32 v1, s15846; GCN1-NEXT: v_mov_b32_e32 v3, s35847; GCN1-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3]5848; GCN1-NEXT: s_waitcnt lgkmcnt(0)5849; GCN1-NEXT: s_cbranch_execnz .LBB40_25850; GCN1-NEXT: .LBB40_4: ; %atomicrmw.private5851; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 05852; GCN1-NEXT: v_mov_b32_e32 v5, s25853; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec5854; GCN1-NEXT: s_cselect_b32 s0, s0, -15855; GCN1-NEXT: v_mov_b32_e32 v2, s05856; GCN1-NEXT: s_add_i32 s0, s0, 45857; GCN1-NEXT: v_mov_b32_e32 v3, s05858; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen5859; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen5860; GCN1-NEXT: v_mov_b32_e32 v4, s35861; GCN1-NEXT: s_waitcnt vmcnt(0)5862; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]5863; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc5864; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc5865; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen5866; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen5867; GCN1-NEXT: s_endpgm5868;5869; GCN2-LABEL: atomic_min_i64_offset:5870; GCN2: ; %bb.0: ; %entry5871; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD05872; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD15873; GCN2-NEXT: s_mov_b32 s90, -15874; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245875; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf45876; GCN2-NEXT: s_mov_b32 s91, 0xe800005877; GCN2-NEXT: s_add_u32 s88, s88, s115878; GCN2-NEXT: s_addc_u32 s89, s89, 05879; GCN2-NEXT: s_waitcnt lgkmcnt(0)5880; GCN2-NEXT: s_add_u32 s0, s0, 325881; GCN2-NEXT: s_addc_u32 s1, s1, 05882; GCN2-NEXT: s_cmp_eq_u32 s1, s45883; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 05884; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]5885; GCN2-NEXT: s_mov_b64 s[4:5], -15886; GCN2-NEXT: s_cbranch_vccnz .LBB40_35887; GCN2-NEXT: ; %bb.1: ; %Flow5888; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]5889; GCN2-NEXT: s_cbranch_vccz .LBB40_45890; GCN2-NEXT: .LBB40_2: ; %atomicrmw.phi5891; GCN2-NEXT: s_endpgm5892; GCN2-NEXT: .LBB40_3: ; %atomicrmw.global5893; GCN2-NEXT: v_mov_b32_e32 v0, s05894; GCN2-NEXT: v_mov_b32_e32 v2, s25895; GCN2-NEXT: v_mov_b32_e32 v1, s15896; GCN2-NEXT: v_mov_b32_e32 v3, s35897; GCN2-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3]5898; GCN2-NEXT: s_waitcnt lgkmcnt(0)5899; GCN2-NEXT: s_cbranch_execnz .LBB40_25900; GCN2-NEXT: .LBB40_4: ; %atomicrmw.private5901; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 05902; GCN2-NEXT: s_cselect_b32 s0, s0, -15903; GCN2-NEXT: v_mov_b32_e32 v2, s05904; GCN2-NEXT: s_add_i32 s0, s0, 45905; GCN2-NEXT: v_mov_b32_e32 v3, s05906; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen5907; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen5908; GCN2-NEXT: v_mov_b32_e32 v5, s25909; GCN2-NEXT: v_mov_b32_e32 v4, s35910; GCN2-NEXT: s_waitcnt vmcnt(0)5911; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]5912; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc5913; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc5914; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen5915; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen5916; GCN2-NEXT: s_endpgm5917;5918; GFX12-LABEL: atomic_min_i64_offset:5919; GFX12: ; %bb.0: ; %entry5920; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x245921; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base5922; GFX12-NEXT: s_wait_kmcnt 0x05923; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 325924; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)5925; GFX12-NEXT: s_cmp_eq_u32 s1, s75926; GFX12-NEXT: s_cselect_b32 s4, -1, 05927; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s45928; GFX12-NEXT: s_mov_b32 s4, -15929; GFX12-NEXT: s_cbranch_vccnz .LBB40_35930; GFX12-NEXT: ; %bb.1: ; %Flow5931; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s45932; GFX12-NEXT: s_cbranch_vccz .LBB40_45933; GFX12-NEXT: .LBB40_2: ; %atomicrmw.phi5934; GFX12-NEXT: s_endpgm5935; GFX12-NEXT: .LBB40_3: ; %atomicrmw.global5936; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15937; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s35938; GFX12-NEXT: flat_atomic_min_i64 v[0:1], v[2:3] scope:SCOPE_SE5939; GFX12-NEXT: s_wait_storecnt_dscnt 0x05940; GFX12-NEXT: global_inv scope:SCOPE_SE5941; GFX12-NEXT: s_cbranch_execnz .LBB40_25942; GFX12-NEXT: .LBB40_4: ; %atomicrmw.private5943; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 05944; GFX12-NEXT: s_cselect_b32 s0, s0, -15945; GFX12-NEXT: scratch_load_b64 v[0:1], off, s05946; GFX12-NEXT: s_wait_loadcnt 0x05947; GFX12-NEXT: v_cmp_ge_i64_e32 vcc_lo, s[2:3], v[0:1]5948; GFX12-NEXT: v_cndmask_b32_e32 v1, s3, v1, vcc_lo5949; GFX12-NEXT: v_cndmask_b32_e32 v0, s2, v0, vcc_lo5950; GFX12-NEXT: scratch_store_b64 off, v[0:1], s05951; GFX12-NEXT: s_endpgm5952entry:5953 %gep = getelementptr inbounds i64, ptr %out, i64 45954 %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05955 ret void5956}5957 5958define amdgpu_kernel void @atomic_min_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {5959; GCN1-LABEL: atomic_min_i64_ret_offset:5960; GCN1: ; %bb.0: ; %entry5961; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD05962; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD15963; GCN1-NEXT: s_mov_b32 s14, -15964; GCN1-NEXT: s_mov_b32 s15, 0xe8f0005965; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95966; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f5967; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd5968; GCN1-NEXT: s_add_u32 s12, s12, s115969; GCN1-NEXT: s_addc_u32 s13, s13, 05970; GCN1-NEXT: s_waitcnt lgkmcnt(0)5971; GCN1-NEXT: s_add_u32 s0, s0, 325972; GCN1-NEXT: s_addc_u32 s1, s1, 05973; GCN1-NEXT: s_cmp_eq_u32 s1, s65974; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 05975; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]5976; GCN1-NEXT: s_cbranch_vccz .LBB41_25977; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global5978; GCN1-NEXT: v_mov_b32_e32 v0, s05979; GCN1-NEXT: v_mov_b32_e32 v2, s45980; GCN1-NEXT: v_mov_b32_e32 v1, s15981; GCN1-NEXT: v_mov_b32_e32 v3, s55982; GCN1-NEXT: flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc5983; GCN1-NEXT: s_waitcnt lgkmcnt(0)5984; GCN1-NEXT: s_cbranch_execz .LBB41_35985; GCN1-NEXT: s_branch .LBB41_45986; GCN1-NEXT: .LBB41_2:5987; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr15988; GCN1-NEXT: .LBB41_3: ; %atomicrmw.private5989; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 05990; GCN1-NEXT: v_mov_b32_e32 v5, s45991; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec5992; GCN1-NEXT: s_cselect_b32 s0, s0, -15993; GCN1-NEXT: v_mov_b32_e32 v2, s05994; GCN1-NEXT: s_add_i32 s0, s0, 45995; GCN1-NEXT: v_mov_b32_e32 v3, s05996; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen5997; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen5998; GCN1-NEXT: v_mov_b32_e32 v4, s55999; GCN1-NEXT: s_waitcnt vmcnt(0)6000; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[0:1]6001; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc6002; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc6003; GCN1-NEXT: buffer_store_dword v5, v2, s[12:15], 0 offen6004; GCN1-NEXT: buffer_store_dword v4, v3, s[12:15], 0 offen6005; GCN1-NEXT: .LBB41_4: ; %atomicrmw.end6006; GCN1-NEXT: v_mov_b32_e32 v2, s26007; GCN1-NEXT: v_mov_b32_e32 v3, s36008; GCN1-NEXT: s_waitcnt vmcnt(0)6009; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]6010; GCN1-NEXT: s_endpgm6011;6012; GCN2-LABEL: atomic_min_i64_ret_offset:6013; GCN2: ; %bb.0: ; %entry6014; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD06015; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD16016; GCN2-NEXT: s_mov_b32 s90, -16017; GCN2-NEXT: s_mov_b32 s91, 0xe800006018; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246019; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc6020; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x346021; GCN2-NEXT: s_add_u32 s88, s88, s116022; GCN2-NEXT: s_addc_u32 s89, s89, 06023; GCN2-NEXT: s_waitcnt lgkmcnt(0)6024; GCN2-NEXT: s_add_u32 s0, s0, 326025; GCN2-NEXT: s_addc_u32 s1, s1, 06026; GCN2-NEXT: s_cmp_eq_u32 s1, s66027; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 06028; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]6029; GCN2-NEXT: s_cbranch_vccz .LBB41_26030; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global6031; GCN2-NEXT: v_mov_b32_e32 v0, s06032; GCN2-NEXT: v_mov_b32_e32 v2, s46033; GCN2-NEXT: v_mov_b32_e32 v1, s16034; GCN2-NEXT: v_mov_b32_e32 v3, s56035; GCN2-NEXT: flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc6036; GCN2-NEXT: s_waitcnt lgkmcnt(0)6037; GCN2-NEXT: s_cbranch_execz .LBB41_36038; GCN2-NEXT: s_branch .LBB41_46039; GCN2-NEXT: .LBB41_2:6040; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr16041; GCN2-NEXT: .LBB41_3: ; %atomicrmw.private6042; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 06043; GCN2-NEXT: s_cselect_b32 s0, s0, -16044; GCN2-NEXT: v_mov_b32_e32 v2, s06045; GCN2-NEXT: s_add_i32 s0, s0, 46046; GCN2-NEXT: v_mov_b32_e32 v3, s06047; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen6048; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen6049; GCN2-NEXT: v_mov_b32_e32 v5, s46050; GCN2-NEXT: v_mov_b32_e32 v4, s56051; GCN2-NEXT: s_waitcnt vmcnt(0)6052; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[0:1]6053; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc6054; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc6055; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen6056; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen6057; GCN2-NEXT: .LBB41_4: ; %atomicrmw.end6058; GCN2-NEXT: v_mov_b32_e32 v2, s26059; GCN2-NEXT: v_mov_b32_e32 v3, s36060; GCN2-NEXT: s_waitcnt vmcnt(0)6061; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]6062; GCN2-NEXT: s_endpgm6063;6064; GFX12-LABEL: atomic_min_i64_ret_offset:6065; GFX12: ; %bb.0: ; %entry6066; GFX12-NEXT: s_clause 0x16067; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x246068; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x346069; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base6070; GFX12-NEXT: s_wait_kmcnt 0x06071; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 326072; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)6073; GFX12-NEXT: s_cmp_eq_u32 s1, s76074; GFX12-NEXT: s_cselect_b32 s6, -1, 06075; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s66076; GFX12-NEXT: s_cbranch_vccz .LBB41_26077; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global6078; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16079; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s56080; GFX12-NEXT: flat_atomic_min_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE6081; GFX12-NEXT: s_wait_loadcnt_dscnt 0x06082; GFX12-NEXT: global_inv scope:SCOPE_SE6083; GFX12-NEXT: s_cbranch_execz .LBB41_36084; GFX12-NEXT: s_branch .LBB41_46085; GFX12-NEXT: .LBB41_2:6086; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr16087; GFX12-NEXT: .LBB41_3: ; %atomicrmw.private6088; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 06089; GFX12-NEXT: s_cselect_b32 s0, s0, -16090; GFX12-NEXT: scratch_load_b64 v[0:1], off, s06091; GFX12-NEXT: s_wait_loadcnt 0x06092; GFX12-NEXT: v_cmp_ge_i64_e32 vcc_lo, s[4:5], v[0:1]6093; GFX12-NEXT: v_cndmask_b32_e32 v3, s5, v1, vcc_lo6094; GFX12-NEXT: v_cndmask_b32_e32 v2, s4, v0, vcc_lo6095; GFX12-NEXT: scratch_store_b64 off, v[2:3], s06096; GFX12-NEXT: .LBB41_4: ; %atomicrmw.end6097; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s36098; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]6099; GFX12-NEXT: s_endpgm6100entry:6101 %gep = getelementptr inbounds i64, ptr %out, i64 46102 %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !06103 store i64 %tmp0, ptr %out26104 ret void6105}6106 6107define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {6108; GCN1-LABEL: atomic_min_i64_addr64_offset:6109; GCN1: ; %bb.0: ; %entry6110; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD06111; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD16112; GCN1-NEXT: s_mov_b32 s14, -16113; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd6114; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96115; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f6116; GCN1-NEXT: s_mov_b32 s15, 0xe8f0006117; GCN1-NEXT: s_add_u32 s12, s12, s116118; GCN1-NEXT: s_addc_u32 s13, s13, 06119; GCN1-NEXT: s_waitcnt lgkmcnt(0)6120; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 36121; GCN1-NEXT: s_add_u32 s0, s0, s46122; GCN1-NEXT: s_addc_u32 s1, s1, s56123; GCN1-NEXT: s_add_u32 s0, s0, 326124; GCN1-NEXT: s_addc_u32 s1, s1, 06125; GCN1-NEXT: s_cmp_eq_u32 s1, s86126; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 06127; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]6128; GCN1-NEXT: s_mov_b64 s[4:5], -16129; GCN1-NEXT: s_cbranch_vccnz .LBB42_36130; GCN1-NEXT: ; %bb.1: ; %Flow6131; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]6132; GCN1-NEXT: s_cbranch_vccz .LBB42_46133; GCN1-NEXT: .LBB42_2: ; %atomicrmw.phi6134; GCN1-NEXT: s_endpgm6135; GCN1-NEXT: .LBB42_3: ; %atomicrmw.global6136; GCN1-NEXT: v_mov_b32_e32 v0, s06137; GCN1-NEXT: v_mov_b32_e32 v2, s26138; GCN1-NEXT: v_mov_b32_e32 v1, s16139; GCN1-NEXT: v_mov_b32_e32 v3, s36140; GCN1-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3]6141; GCN1-NEXT: s_waitcnt lgkmcnt(0)6142; GCN1-NEXT: s_cbranch_execnz .LBB42_26143; GCN1-NEXT: .LBB42_4: ; %atomicrmw.private6144; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 06145; GCN1-NEXT: v_mov_b32_e32 v5, s26146; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec6147; GCN1-NEXT: s_cselect_b32 s0, s0, -16148; GCN1-NEXT: v_mov_b32_e32 v2, s06149; GCN1-NEXT: s_add_i32 s0, s0, 46150; GCN1-NEXT: v_mov_b32_e32 v3, s06151; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen6152; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen6153; GCN1-NEXT: v_mov_b32_e32 v4, s36154; GCN1-NEXT: s_waitcnt vmcnt(0)6155; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]6156; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc6157; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc6158; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen6159; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen6160; GCN1-NEXT: s_endpgm6161;6162; GCN2-LABEL: atomic_min_i64_addr64_offset:6163; GCN2: ; %bb.0: ; %entry6164; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD06165; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD16166; GCN2-NEXT: s_mov_b32 s90, -16167; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346168; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246169; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc6170; GCN2-NEXT: s_mov_b32 s91, 0xe800006171; GCN2-NEXT: s_add_u32 s88, s88, s116172; GCN2-NEXT: s_addc_u32 s89, s89, 06173; GCN2-NEXT: s_waitcnt lgkmcnt(0)6174; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 36175; GCN2-NEXT: s_add_u32 s0, s0, s46176; GCN2-NEXT: s_addc_u32 s1, s1, s56177; GCN2-NEXT: s_add_u32 s0, s0, 326178; GCN2-NEXT: s_addc_u32 s1, s1, 06179; GCN2-NEXT: s_cmp_eq_u32 s1, s86180; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 06181; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]6182; GCN2-NEXT: s_mov_b64 s[4:5], -16183; GCN2-NEXT: s_cbranch_vccnz .LBB42_36184; GCN2-NEXT: ; %bb.1: ; %Flow6185; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]6186; GCN2-NEXT: s_cbranch_vccz .LBB42_46187; GCN2-NEXT: .LBB42_2: ; %atomicrmw.phi6188; GCN2-NEXT: s_endpgm6189; GCN2-NEXT: .LBB42_3: ; %atomicrmw.global6190; GCN2-NEXT: v_mov_b32_e32 v0, s06191; GCN2-NEXT: v_mov_b32_e32 v2, s26192; GCN2-NEXT: v_mov_b32_e32 v1, s16193; GCN2-NEXT: v_mov_b32_e32 v3, s36194; GCN2-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3]6195; GCN2-NEXT: s_waitcnt lgkmcnt(0)6196; GCN2-NEXT: s_cbranch_execnz .LBB42_26197; GCN2-NEXT: .LBB42_4: ; %atomicrmw.private6198; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 06199; GCN2-NEXT: s_cselect_b32 s0, s0, -16200; GCN2-NEXT: v_mov_b32_e32 v2, s06201; GCN2-NEXT: s_add_i32 s0, s0, 46202; GCN2-NEXT: v_mov_b32_e32 v3, s06203; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen6204; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen6205; GCN2-NEXT: v_mov_b32_e32 v5, s26206; GCN2-NEXT: v_mov_b32_e32 v4, s36207; GCN2-NEXT: s_waitcnt vmcnt(0)6208; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]6209; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc6210; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc6211; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen6212; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen6213; GCN2-NEXT: s_endpgm6214;6215; GFX12-LABEL: atomic_min_i64_addr64_offset:6216; GFX12: ; %bb.0: ; %entry6217; GFX12-NEXT: s_clause 0x16218; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x346219; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x246220; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base6221; GFX12-NEXT: s_wait_kmcnt 0x06222; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 36223; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)6224; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]6225; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 326226; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)6227; GFX12-NEXT: s_cmp_eq_u32 s1, s76228; GFX12-NEXT: s_cselect_b32 s4, -1, 06229; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s46230; GFX12-NEXT: s_mov_b32 s4, -16231; GFX12-NEXT: s_cbranch_vccnz .LBB42_36232; GFX12-NEXT: ; %bb.1: ; %Flow6233; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s46234; GFX12-NEXT: s_cbranch_vccz .LBB42_46235; GFX12-NEXT: .LBB42_2: ; %atomicrmw.phi6236; GFX12-NEXT: s_endpgm6237; GFX12-NEXT: .LBB42_3: ; %atomicrmw.global6238; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16239; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s36240; GFX12-NEXT: flat_atomic_min_i64 v[0:1], v[2:3] scope:SCOPE_SE6241; GFX12-NEXT: s_wait_storecnt_dscnt 0x06242; GFX12-NEXT: global_inv scope:SCOPE_SE6243; GFX12-NEXT: s_cbranch_execnz .LBB42_26244; GFX12-NEXT: .LBB42_4: ; %atomicrmw.private6245; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 06246; GFX12-NEXT: s_cselect_b32 s0, s0, -16247; GFX12-NEXT: scratch_load_b64 v[0:1], off, s06248; GFX12-NEXT: s_wait_loadcnt 0x06249; GFX12-NEXT: v_cmp_ge_i64_e32 vcc_lo, s[2:3], v[0:1]6250; GFX12-NEXT: v_cndmask_b32_e32 v1, s3, v1, vcc_lo6251; GFX12-NEXT: v_cndmask_b32_e32 v0, s2, v0, vcc_lo6252; GFX12-NEXT: scratch_store_b64 off, v[0:1], s06253; GFX12-NEXT: s_endpgm6254entry:6255 %ptr = getelementptr inbounds i64, ptr %out, i64 %index6256 %gep = getelementptr inbounds i64, ptr %ptr, i64 46257 %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !06258 ret void6259}6260 6261define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {6262; GCN1-LABEL: atomic_min_i64_ret_addr64_offset:6263; GCN1: ; %bb.0: ; %entry6264; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD06265; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD16266; GCN1-NEXT: s_mov_b32 s18, -16267; GCN1-NEXT: s_mov_b32 s19, 0xe8f0006268; GCN1-NEXT: s_add_u32 s16, s16, s116269; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x96270; GCN1-NEXT: s_load_dword s2, s[4:5], 0x416271; GCN1-NEXT: s_addc_u32 s17, s17, 06272; GCN1-NEXT: s_waitcnt lgkmcnt(0)6273; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 36274; GCN1-NEXT: s_add_u32 s0, s8, s06275; GCN1-NEXT: s_addc_u32 s1, s9, s16276; GCN1-NEXT: s_add_u32 s0, s0, 326277; GCN1-NEXT: s_addc_u32 s1, s1, 06278; GCN1-NEXT: s_cmp_eq_u32 s1, s26279; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 06280; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]6281; GCN1-NEXT: s_cbranch_vccz .LBB43_26282; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global6283; GCN1-NEXT: v_mov_b32_e32 v0, s06284; GCN1-NEXT: v_mov_b32_e32 v2, s126285; GCN1-NEXT: v_mov_b32_e32 v1, s16286; GCN1-NEXT: v_mov_b32_e32 v3, s136287; GCN1-NEXT: flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc6288; GCN1-NEXT: s_waitcnt lgkmcnt(0)6289; GCN1-NEXT: s_cbranch_execz .LBB43_36290; GCN1-NEXT: s_branch .LBB43_46291; GCN1-NEXT: .LBB43_2:6292; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr16293; GCN1-NEXT: .LBB43_3: ; %atomicrmw.private6294; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 06295; GCN1-NEXT: v_mov_b32_e32 v5, s126296; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec6297; GCN1-NEXT: s_cselect_b32 s0, s0, -16298; GCN1-NEXT: v_mov_b32_e32 v2, s06299; GCN1-NEXT: s_add_i32 s0, s0, 46300; GCN1-NEXT: v_mov_b32_e32 v3, s06301; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen6302; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen6303; GCN1-NEXT: v_mov_b32_e32 v4, s136304; GCN1-NEXT: s_waitcnt vmcnt(0)6305; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1]6306; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc6307; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc6308; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen6309; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen6310; GCN1-NEXT: .LBB43_4: ; %atomicrmw.end6311; GCN1-NEXT: v_mov_b32_e32 v2, s106312; GCN1-NEXT: v_mov_b32_e32 v3, s116313; GCN1-NEXT: s_waitcnt vmcnt(0)6314; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]6315; GCN1-NEXT: s_endpgm6316;6317; GCN2-LABEL: atomic_min_i64_ret_addr64_offset:6318; GCN2: ; %bb.0: ; %entry6319; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD06320; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD16321; GCN2-NEXT: s_mov_b32 s90, -16322; GCN2-NEXT: s_mov_b32 s91, 0xe800006323; GCN2-NEXT: s_add_u32 s88, s88, s116324; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x246325; GCN2-NEXT: s_load_dword s2, s[4:5], 0x1046326; GCN2-NEXT: s_addc_u32 s89, s89, 06327; GCN2-NEXT: s_waitcnt lgkmcnt(0)6328; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 36329; GCN2-NEXT: s_add_u32 s0, s8, s06330; GCN2-NEXT: s_addc_u32 s1, s9, s16331; GCN2-NEXT: s_add_u32 s0, s0, 326332; GCN2-NEXT: s_addc_u32 s1, s1, 06333; GCN2-NEXT: s_cmp_eq_u32 s1, s26334; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 06335; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]6336; GCN2-NEXT: s_cbranch_vccz .LBB43_26337; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global6338; GCN2-NEXT: v_mov_b32_e32 v0, s06339; GCN2-NEXT: v_mov_b32_e32 v2, s126340; GCN2-NEXT: v_mov_b32_e32 v1, s16341; GCN2-NEXT: v_mov_b32_e32 v3, s136342; GCN2-NEXT: flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc6343; GCN2-NEXT: s_waitcnt lgkmcnt(0)6344; GCN2-NEXT: s_cbranch_execz .LBB43_36345; GCN2-NEXT: s_branch .LBB43_46346; GCN2-NEXT: .LBB43_2:6347; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr16348; GCN2-NEXT: .LBB43_3: ; %atomicrmw.private6349; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 06350; GCN2-NEXT: s_cselect_b32 s0, s0, -16351; GCN2-NEXT: v_mov_b32_e32 v2, s06352; GCN2-NEXT: s_add_i32 s0, s0, 46353; GCN2-NEXT: v_mov_b32_e32 v3, s06354; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen6355; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen6356; GCN2-NEXT: v_mov_b32_e32 v5, s126357; GCN2-NEXT: v_mov_b32_e32 v4, s136358; GCN2-NEXT: s_waitcnt vmcnt(0)6359; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1]6360; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc6361; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc6362; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen6363; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen6364; GCN2-NEXT: .LBB43_4: ; %atomicrmw.end6365; GCN2-NEXT: v_mov_b32_e32 v2, s106366; GCN2-NEXT: v_mov_b32_e32 v3, s116367; GCN2-NEXT: s_waitcnt vmcnt(0)6368; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]6369; GCN2-NEXT: s_endpgm6370;6371; GFX12-LABEL: atomic_min_i64_ret_addr64_offset:6372; GFX12: ; %bb.0: ; %entry6373; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x246374; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base6375; GFX12-NEXT: s_wait_kmcnt 0x06376; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 36377; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)6378; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]6379; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 326380; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)6381; GFX12-NEXT: s_cmp_eq_u32 s1, s96382; GFX12-NEXT: s_cselect_b32 s6, -1, 06383; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s66384; GFX12-NEXT: s_cbranch_vccz .LBB43_26385; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global6386; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16387; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s56388; GFX12-NEXT: flat_atomic_min_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE6389; GFX12-NEXT: s_wait_loadcnt_dscnt 0x06390; GFX12-NEXT: global_inv scope:SCOPE_SE6391; GFX12-NEXT: s_cbranch_execz .LBB43_36392; GFX12-NEXT: s_branch .LBB43_46393; GFX12-NEXT: .LBB43_2:6394; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr16395; GFX12-NEXT: .LBB43_3: ; %atomicrmw.private6396; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 06397; GFX12-NEXT: s_cselect_b32 s0, s0, -16398; GFX12-NEXT: scratch_load_b64 v[0:1], off, s06399; GFX12-NEXT: s_wait_loadcnt 0x06400; GFX12-NEXT: v_cmp_ge_i64_e32 vcc_lo, s[4:5], v[0:1]6401; GFX12-NEXT: v_cndmask_b32_e32 v3, s5, v1, vcc_lo6402; GFX12-NEXT: v_cndmask_b32_e32 v2, s4, v0, vcc_lo6403; GFX12-NEXT: scratch_store_b64 off, v[2:3], s06404; GFX12-NEXT: .LBB43_4: ; %atomicrmw.end6405; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s36406; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]6407; GFX12-NEXT: s_endpgm6408entry:6409 %ptr = getelementptr inbounds i64, ptr %out, i64 %index6410 %gep = getelementptr inbounds i64, ptr %ptr, i64 46411 %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !06412 store i64 %tmp0, ptr %out26413 ret void6414}6415 6416define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {6417; GCN1-LABEL: atomic_min_i64:6418; GCN1: ; %bb.0: ; %entry6419; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD06420; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD16421; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96422; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d6423; GCN1-NEXT: s_mov_b32 s14, -16424; GCN1-NEXT: s_mov_b32 s15, 0xe8f0006425; GCN1-NEXT: s_add_u32 s12, s12, s116426; GCN1-NEXT: s_addc_u32 s13, s13, 06427; GCN1-NEXT: s_waitcnt lgkmcnt(0)6428; GCN1-NEXT: s_cmp_eq_u32 s1, s46429; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 06430; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]6431; GCN1-NEXT: s_mov_b64 s[4:5], -16432; GCN1-NEXT: s_cbranch_vccnz .LBB44_36433; GCN1-NEXT: ; %bb.1: ; %Flow6434; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]6435; GCN1-NEXT: s_cbranch_vccz .LBB44_46436; GCN1-NEXT: .LBB44_2: ; %atomicrmw.phi6437; GCN1-NEXT: s_endpgm6438; GCN1-NEXT: .LBB44_3: ; %atomicrmw.global6439; GCN1-NEXT: v_mov_b32_e32 v0, s06440; GCN1-NEXT: v_mov_b32_e32 v2, s26441; GCN1-NEXT: v_mov_b32_e32 v1, s16442; GCN1-NEXT: v_mov_b32_e32 v3, s36443; GCN1-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3]6444; GCN1-NEXT: s_waitcnt lgkmcnt(0)6445; GCN1-NEXT: s_cbranch_execnz .LBB44_26446; GCN1-NEXT: .LBB44_4: ; %atomicrmw.private6447; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 06448; GCN1-NEXT: v_mov_b32_e32 v5, s26449; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec6450; GCN1-NEXT: s_cselect_b32 s0, s0, -16451; GCN1-NEXT: v_mov_b32_e32 v2, s06452; GCN1-NEXT: s_add_i32 s0, s0, 46453; GCN1-NEXT: v_mov_b32_e32 v3, s06454; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen6455; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen6456; GCN1-NEXT: v_mov_b32_e32 v4, s36457; GCN1-NEXT: s_waitcnt vmcnt(0)6458; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]6459; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc6460; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc6461; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen6462; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen6463; GCN1-NEXT: s_endpgm6464;6465; GCN2-LABEL: atomic_min_i64:6466; GCN2: ; %bb.0: ; %entry6467; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD06468; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD16469; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246470; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf46471; GCN2-NEXT: s_mov_b32 s90, -16472; GCN2-NEXT: s_mov_b32 s91, 0xe800006473; GCN2-NEXT: s_add_u32 s88, s88, s116474; GCN2-NEXT: s_addc_u32 s89, s89, 06475; GCN2-NEXT: s_waitcnt lgkmcnt(0)6476; GCN2-NEXT: s_cmp_eq_u32 s1, s46477; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 06478; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]6479; GCN2-NEXT: s_mov_b64 s[4:5], -16480; GCN2-NEXT: s_cbranch_vccnz .LBB44_36481; GCN2-NEXT: ; %bb.1: ; %Flow6482; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]6483; GCN2-NEXT: s_cbranch_vccz .LBB44_46484; GCN2-NEXT: .LBB44_2: ; %atomicrmw.phi6485; GCN2-NEXT: s_endpgm6486; GCN2-NEXT: .LBB44_3: ; %atomicrmw.global6487; GCN2-NEXT: v_mov_b32_e32 v0, s06488; GCN2-NEXT: v_mov_b32_e32 v2, s26489; GCN2-NEXT: v_mov_b32_e32 v1, s16490; GCN2-NEXT: v_mov_b32_e32 v3, s36491; GCN2-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3]6492; GCN2-NEXT: s_waitcnt lgkmcnt(0)6493; GCN2-NEXT: s_cbranch_execnz .LBB44_26494; GCN2-NEXT: .LBB44_4: ; %atomicrmw.private6495; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 06496; GCN2-NEXT: s_cselect_b32 s0, s0, -16497; GCN2-NEXT: v_mov_b32_e32 v2, s06498; GCN2-NEXT: s_add_i32 s0, s0, 46499; GCN2-NEXT: v_mov_b32_e32 v3, s06500; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen6501; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen6502; GCN2-NEXT: v_mov_b32_e32 v5, s26503; GCN2-NEXT: v_mov_b32_e32 v4, s36504; GCN2-NEXT: s_waitcnt vmcnt(0)6505; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]6506; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc6507; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc6508; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen6509; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen6510; GCN2-NEXT: s_endpgm6511;6512; GFX12-LABEL: atomic_min_i64:6513; GFX12: ; %bb.0: ; %entry6514; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x246515; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base6516; GFX12-NEXT: s_wait_kmcnt 0x06517; GFX12-NEXT: s_cmp_eq_u32 s1, s76518; GFX12-NEXT: s_cselect_b32 s4, -1, 06519; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6520; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s46521; GFX12-NEXT: s_mov_b32 s4, -16522; GFX12-NEXT: s_cbranch_vccnz .LBB44_36523; GFX12-NEXT: ; %bb.1: ; %Flow6524; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s46525; GFX12-NEXT: s_cbranch_vccz .LBB44_46526; GFX12-NEXT: .LBB44_2: ; %atomicrmw.phi6527; GFX12-NEXT: s_endpgm6528; GFX12-NEXT: .LBB44_3: ; %atomicrmw.global6529; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16530; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s36531; GFX12-NEXT: flat_atomic_min_i64 v[0:1], v[2:3] scope:SCOPE_SE6532; GFX12-NEXT: s_wait_storecnt_dscnt 0x06533; GFX12-NEXT: global_inv scope:SCOPE_SE6534; GFX12-NEXT: s_cbranch_execnz .LBB44_26535; GFX12-NEXT: .LBB44_4: ; %atomicrmw.private6536; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 06537; GFX12-NEXT: s_cselect_b32 s0, s0, -16538; GFX12-NEXT: scratch_load_b64 v[0:1], off, s06539; GFX12-NEXT: s_wait_loadcnt 0x06540; GFX12-NEXT: v_cmp_ge_i64_e32 vcc_lo, s[2:3], v[0:1]6541; GFX12-NEXT: v_cndmask_b32_e32 v1, s3, v1, vcc_lo6542; GFX12-NEXT: v_cndmask_b32_e32 v0, s2, v0, vcc_lo6543; GFX12-NEXT: scratch_store_b64 off, v[0:1], s06544; GFX12-NEXT: s_endpgm6545entry:6546 %tmp0 = atomicrmw volatile min ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !06547 ret void6548}6549 6550define amdgpu_kernel void @atomic_min_i64_ret(ptr %out, ptr %out2, i64 %in) {6551; GCN1-LABEL: atomic_min_i64_ret:6552; GCN1: ; %bb.0: ; %entry6553; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD06554; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD16555; GCN1-NEXT: s_mov_b32 s14, -16556; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f6557; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96558; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd6559; GCN1-NEXT: s_mov_b32 s15, 0xe8f0006560; GCN1-NEXT: s_add_u32 s12, s12, s116561; GCN1-NEXT: s_addc_u32 s13, s13, 06562; GCN1-NEXT: s_waitcnt lgkmcnt(0)6563; GCN1-NEXT: s_cmp_eq_u32 s1, s66564; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 06565; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]6566; GCN1-NEXT: s_cbranch_vccz .LBB45_26567; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global6568; GCN1-NEXT: v_mov_b32_e32 v0, s06569; GCN1-NEXT: v_mov_b32_e32 v2, s46570; GCN1-NEXT: v_mov_b32_e32 v1, s16571; GCN1-NEXT: v_mov_b32_e32 v3, s56572; GCN1-NEXT: flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc6573; GCN1-NEXT: s_waitcnt lgkmcnt(0)6574; GCN1-NEXT: s_cbranch_execz .LBB45_36575; GCN1-NEXT: s_branch .LBB45_46576; GCN1-NEXT: .LBB45_2:6577; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr16578; GCN1-NEXT: .LBB45_3: ; %atomicrmw.private6579; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 06580; GCN1-NEXT: v_mov_b32_e32 v5, s46581; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec6582; GCN1-NEXT: s_cselect_b32 s0, s0, -16583; GCN1-NEXT: v_mov_b32_e32 v2, s06584; GCN1-NEXT: s_add_i32 s0, s0, 46585; GCN1-NEXT: v_mov_b32_e32 v3, s06586; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen6587; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen6588; GCN1-NEXT: v_mov_b32_e32 v4, s56589; GCN1-NEXT: s_waitcnt vmcnt(0)6590; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[0:1]6591; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc6592; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc6593; GCN1-NEXT: buffer_store_dword v5, v2, s[12:15], 0 offen6594; GCN1-NEXT: buffer_store_dword v4, v3, s[12:15], 0 offen6595; GCN1-NEXT: .LBB45_4: ; %atomicrmw.end6596; GCN1-NEXT: v_mov_b32_e32 v2, s26597; GCN1-NEXT: v_mov_b32_e32 v3, s36598; GCN1-NEXT: s_waitcnt vmcnt(0)6599; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]6600; GCN1-NEXT: s_endpgm6601;6602; GCN2-LABEL: atomic_min_i64_ret:6603; GCN2: ; %bb.0: ; %entry6604; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD06605; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD16606; GCN2-NEXT: s_mov_b32 s90, -16607; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc6608; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246609; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x346610; GCN2-NEXT: s_mov_b32 s91, 0xe800006611; GCN2-NEXT: s_add_u32 s88, s88, s116612; GCN2-NEXT: s_addc_u32 s89, s89, 06613; GCN2-NEXT: s_waitcnt lgkmcnt(0)6614; GCN2-NEXT: s_cmp_eq_u32 s1, s66615; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 06616; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]6617; GCN2-NEXT: s_cbranch_vccz .LBB45_26618; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global6619; GCN2-NEXT: v_mov_b32_e32 v0, s06620; GCN2-NEXT: v_mov_b32_e32 v2, s46621; GCN2-NEXT: v_mov_b32_e32 v1, s16622; GCN2-NEXT: v_mov_b32_e32 v3, s56623; GCN2-NEXT: flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc6624; GCN2-NEXT: s_waitcnt lgkmcnt(0)6625; GCN2-NEXT: s_cbranch_execz .LBB45_36626; GCN2-NEXT: s_branch .LBB45_46627; GCN2-NEXT: .LBB45_2:6628; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr16629; GCN2-NEXT: .LBB45_3: ; %atomicrmw.private6630; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 06631; GCN2-NEXT: s_cselect_b32 s0, s0, -16632; GCN2-NEXT: v_mov_b32_e32 v2, s06633; GCN2-NEXT: s_add_i32 s0, s0, 46634; GCN2-NEXT: v_mov_b32_e32 v3, s06635; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen6636; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen6637; GCN2-NEXT: v_mov_b32_e32 v5, s46638; GCN2-NEXT: v_mov_b32_e32 v4, s56639; GCN2-NEXT: s_waitcnt vmcnt(0)6640; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[0:1]6641; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc6642; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc6643; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen6644; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen6645; GCN2-NEXT: .LBB45_4: ; %atomicrmw.end6646; GCN2-NEXT: v_mov_b32_e32 v2, s26647; GCN2-NEXT: v_mov_b32_e32 v3, s36648; GCN2-NEXT: s_waitcnt vmcnt(0)6649; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]6650; GCN2-NEXT: s_endpgm6651;6652; GFX12-LABEL: atomic_min_i64_ret:6653; GFX12: ; %bb.0: ; %entry6654; GFX12-NEXT: s_clause 0x16655; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x246656; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x346657; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base6658; GFX12-NEXT: s_wait_kmcnt 0x06659; GFX12-NEXT: s_cmp_eq_u32 s1, s76660; GFX12-NEXT: s_cselect_b32 s6, -1, 06661; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6662; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s66663; GFX12-NEXT: s_cbranch_vccz .LBB45_26664; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global6665; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16666; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s56667; GFX12-NEXT: flat_atomic_min_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE6668; GFX12-NEXT: s_wait_loadcnt_dscnt 0x06669; GFX12-NEXT: global_inv scope:SCOPE_SE6670; GFX12-NEXT: s_cbranch_execz .LBB45_36671; GFX12-NEXT: s_branch .LBB45_46672; GFX12-NEXT: .LBB45_2:6673; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr16674; GFX12-NEXT: .LBB45_3: ; %atomicrmw.private6675; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 06676; GFX12-NEXT: s_cselect_b32 s0, s0, -16677; GFX12-NEXT: scratch_load_b64 v[0:1], off, s06678; GFX12-NEXT: s_wait_loadcnt 0x06679; GFX12-NEXT: v_cmp_ge_i64_e32 vcc_lo, s[4:5], v[0:1]6680; GFX12-NEXT: v_cndmask_b32_e32 v3, s5, v1, vcc_lo6681; GFX12-NEXT: v_cndmask_b32_e32 v2, s4, v0, vcc_lo6682; GFX12-NEXT: scratch_store_b64 off, v[2:3], s06683; GFX12-NEXT: .LBB45_4: ; %atomicrmw.end6684; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s36685; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]6686; GFX12-NEXT: s_endpgm6687entry:6688 %tmp0 = atomicrmw volatile min ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !06689 store i64 %tmp0, ptr %out26690 ret void6691}6692 6693define amdgpu_kernel void @atomic_min_i64_addr64(ptr %out, i64 %in, i64 %index) {6694; GCN1-LABEL: atomic_min_i64_addr64:6695; GCN1: ; %bb.0: ; %entry6696; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD06697; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD16698; GCN1-NEXT: s_mov_b32 s14, -16699; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd6700; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96701; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f6702; GCN1-NEXT: s_mov_b32 s15, 0xe8f0006703; GCN1-NEXT: s_add_u32 s12, s12, s116704; GCN1-NEXT: s_addc_u32 s13, s13, 06705; GCN1-NEXT: s_waitcnt lgkmcnt(0)6706; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 36707; GCN1-NEXT: s_add_u32 s0, s0, s46708; GCN1-NEXT: s_addc_u32 s1, s1, s56709; GCN1-NEXT: s_cmp_eq_u32 s1, s86710; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 06711; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]6712; GCN1-NEXT: s_mov_b64 s[4:5], -16713; GCN1-NEXT: s_cbranch_vccnz .LBB46_36714; GCN1-NEXT: ; %bb.1: ; %Flow6715; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]6716; GCN1-NEXT: s_cbranch_vccz .LBB46_46717; GCN1-NEXT: .LBB46_2: ; %atomicrmw.phi6718; GCN1-NEXT: s_endpgm6719; GCN1-NEXT: .LBB46_3: ; %atomicrmw.global6720; GCN1-NEXT: v_mov_b32_e32 v0, s06721; GCN1-NEXT: v_mov_b32_e32 v2, s26722; GCN1-NEXT: v_mov_b32_e32 v1, s16723; GCN1-NEXT: v_mov_b32_e32 v3, s36724; GCN1-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3]6725; GCN1-NEXT: s_waitcnt lgkmcnt(0)6726; GCN1-NEXT: s_cbranch_execnz .LBB46_26727; GCN1-NEXT: .LBB46_4: ; %atomicrmw.private6728; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 06729; GCN1-NEXT: v_mov_b32_e32 v5, s26730; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec6731; GCN1-NEXT: s_cselect_b32 s0, s0, -16732; GCN1-NEXT: v_mov_b32_e32 v2, s06733; GCN1-NEXT: s_add_i32 s0, s0, 46734; GCN1-NEXT: v_mov_b32_e32 v3, s06735; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen6736; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen6737; GCN1-NEXT: v_mov_b32_e32 v4, s36738; GCN1-NEXT: s_waitcnt vmcnt(0)6739; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]6740; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc6741; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc6742; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen6743; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen6744; GCN1-NEXT: s_endpgm6745;6746; GCN2-LABEL: atomic_min_i64_addr64:6747; GCN2: ; %bb.0: ; %entry6748; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD06749; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD16750; GCN2-NEXT: s_mov_b32 s90, -16751; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346752; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246753; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc6754; GCN2-NEXT: s_mov_b32 s91, 0xe800006755; GCN2-NEXT: s_add_u32 s88, s88, s116756; GCN2-NEXT: s_addc_u32 s89, s89, 06757; GCN2-NEXT: s_waitcnt lgkmcnt(0)6758; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 36759; GCN2-NEXT: s_add_u32 s0, s0, s46760; GCN2-NEXT: s_addc_u32 s1, s1, s56761; GCN2-NEXT: s_cmp_eq_u32 s1, s86762; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 06763; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]6764; GCN2-NEXT: s_mov_b64 s[4:5], -16765; GCN2-NEXT: s_cbranch_vccnz .LBB46_36766; GCN2-NEXT: ; %bb.1: ; %Flow6767; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]6768; GCN2-NEXT: s_cbranch_vccz .LBB46_46769; GCN2-NEXT: .LBB46_2: ; %atomicrmw.phi6770; GCN2-NEXT: s_endpgm6771; GCN2-NEXT: .LBB46_3: ; %atomicrmw.global6772; GCN2-NEXT: v_mov_b32_e32 v0, s06773; GCN2-NEXT: v_mov_b32_e32 v2, s26774; GCN2-NEXT: v_mov_b32_e32 v1, s16775; GCN2-NEXT: v_mov_b32_e32 v3, s36776; GCN2-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3]6777; GCN2-NEXT: s_waitcnt lgkmcnt(0)6778; GCN2-NEXT: s_cbranch_execnz .LBB46_26779; GCN2-NEXT: .LBB46_4: ; %atomicrmw.private6780; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 06781; GCN2-NEXT: s_cselect_b32 s0, s0, -16782; GCN2-NEXT: v_mov_b32_e32 v2, s06783; GCN2-NEXT: s_add_i32 s0, s0, 46784; GCN2-NEXT: v_mov_b32_e32 v3, s06785; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen6786; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen6787; GCN2-NEXT: v_mov_b32_e32 v5, s26788; GCN2-NEXT: v_mov_b32_e32 v4, s36789; GCN2-NEXT: s_waitcnt vmcnt(0)6790; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]6791; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc6792; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc6793; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen6794; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen6795; GCN2-NEXT: s_endpgm6796;6797; GFX12-LABEL: atomic_min_i64_addr64:6798; GFX12: ; %bb.0: ; %entry6799; GFX12-NEXT: s_clause 0x16800; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x346801; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x246802; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base6803; GFX12-NEXT: s_wait_kmcnt 0x06804; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 36805; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)6806; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]6807; GFX12-NEXT: s_cmp_eq_u32 s1, s76808; GFX12-NEXT: s_cselect_b32 s4, -1, 06809; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6810; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s46811; GFX12-NEXT: s_mov_b32 s4, -16812; GFX12-NEXT: s_cbranch_vccnz .LBB46_36813; GFX12-NEXT: ; %bb.1: ; %Flow6814; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s46815; GFX12-NEXT: s_cbranch_vccz .LBB46_46816; GFX12-NEXT: .LBB46_2: ; %atomicrmw.phi6817; GFX12-NEXT: s_endpgm6818; GFX12-NEXT: .LBB46_3: ; %atomicrmw.global6819; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16820; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s36821; GFX12-NEXT: flat_atomic_min_i64 v[0:1], v[2:3] scope:SCOPE_SE6822; GFX12-NEXT: s_wait_storecnt_dscnt 0x06823; GFX12-NEXT: global_inv scope:SCOPE_SE6824; GFX12-NEXT: s_cbranch_execnz .LBB46_26825; GFX12-NEXT: .LBB46_4: ; %atomicrmw.private6826; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 06827; GFX12-NEXT: s_cselect_b32 s0, s0, -16828; GFX12-NEXT: scratch_load_b64 v[0:1], off, s06829; GFX12-NEXT: s_wait_loadcnt 0x06830; GFX12-NEXT: v_cmp_ge_i64_e32 vcc_lo, s[2:3], v[0:1]6831; GFX12-NEXT: v_cndmask_b32_e32 v1, s3, v1, vcc_lo6832; GFX12-NEXT: v_cndmask_b32_e32 v0, s2, v0, vcc_lo6833; GFX12-NEXT: scratch_store_b64 off, v[0:1], s06834; GFX12-NEXT: s_endpgm6835entry:6836 %ptr = getelementptr inbounds i64, ptr %out, i64 %index6837 %tmp0 = atomicrmw volatile min ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !06838 ret void6839}6840 6841define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {6842; GCN1-LABEL: atomic_min_i64_ret_addr64:6843; GCN1: ; %bb.0: ; %entry6844; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD06845; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD16846; GCN1-NEXT: s_mov_b32 s18, -16847; GCN1-NEXT: s_mov_b32 s19, 0xe8f0006848; GCN1-NEXT: s_add_u32 s16, s16, s116849; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x96850; GCN1-NEXT: s_load_dword s2, s[4:5], 0x416851; GCN1-NEXT: s_addc_u32 s17, s17, 06852; GCN1-NEXT: s_waitcnt lgkmcnt(0)6853; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 36854; GCN1-NEXT: s_add_u32 s0, s8, s06855; GCN1-NEXT: s_addc_u32 s1, s9, s16856; GCN1-NEXT: s_cmp_eq_u32 s1, s26857; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 06858; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]6859; GCN1-NEXT: s_cbranch_vccz .LBB47_26860; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global6861; GCN1-NEXT: v_mov_b32_e32 v0, s06862; GCN1-NEXT: v_mov_b32_e32 v2, s126863; GCN1-NEXT: v_mov_b32_e32 v1, s16864; GCN1-NEXT: v_mov_b32_e32 v3, s136865; GCN1-NEXT: flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc6866; GCN1-NEXT: s_waitcnt lgkmcnt(0)6867; GCN1-NEXT: s_cbranch_execz .LBB47_36868; GCN1-NEXT: s_branch .LBB47_46869; GCN1-NEXT: .LBB47_2:6870; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr16871; GCN1-NEXT: .LBB47_3: ; %atomicrmw.private6872; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 06873; GCN1-NEXT: v_mov_b32_e32 v5, s126874; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec6875; GCN1-NEXT: s_cselect_b32 s0, s0, -16876; GCN1-NEXT: v_mov_b32_e32 v2, s06877; GCN1-NEXT: s_add_i32 s0, s0, 46878; GCN1-NEXT: v_mov_b32_e32 v3, s06879; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen6880; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen6881; GCN1-NEXT: v_mov_b32_e32 v4, s136882; GCN1-NEXT: s_waitcnt vmcnt(0)6883; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1]6884; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc6885; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc6886; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen6887; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen6888; GCN1-NEXT: .LBB47_4: ; %atomicrmw.end6889; GCN1-NEXT: v_mov_b32_e32 v2, s106890; GCN1-NEXT: v_mov_b32_e32 v3, s116891; GCN1-NEXT: s_waitcnt vmcnt(0)6892; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]6893; GCN1-NEXT: s_endpgm6894;6895; GCN2-LABEL: atomic_min_i64_ret_addr64:6896; GCN2: ; %bb.0: ; %entry6897; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD06898; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD16899; GCN2-NEXT: s_mov_b32 s90, -16900; GCN2-NEXT: s_mov_b32 s91, 0xe800006901; GCN2-NEXT: s_add_u32 s88, s88, s116902; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x246903; GCN2-NEXT: s_load_dword s2, s[4:5], 0x1046904; GCN2-NEXT: s_addc_u32 s89, s89, 06905; GCN2-NEXT: s_waitcnt lgkmcnt(0)6906; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 36907; GCN2-NEXT: s_add_u32 s0, s8, s06908; GCN2-NEXT: s_addc_u32 s1, s9, s16909; GCN2-NEXT: s_cmp_eq_u32 s1, s26910; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 06911; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]6912; GCN2-NEXT: s_cbranch_vccz .LBB47_26913; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global6914; GCN2-NEXT: v_mov_b32_e32 v0, s06915; GCN2-NEXT: v_mov_b32_e32 v2, s126916; GCN2-NEXT: v_mov_b32_e32 v1, s16917; GCN2-NEXT: v_mov_b32_e32 v3, s136918; GCN2-NEXT: flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc6919; GCN2-NEXT: s_waitcnt lgkmcnt(0)6920; GCN2-NEXT: s_cbranch_execz .LBB47_36921; GCN2-NEXT: s_branch .LBB47_46922; GCN2-NEXT: .LBB47_2:6923; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr16924; GCN2-NEXT: .LBB47_3: ; %atomicrmw.private6925; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 06926; GCN2-NEXT: s_cselect_b32 s0, s0, -16927; GCN2-NEXT: v_mov_b32_e32 v2, s06928; GCN2-NEXT: s_add_i32 s0, s0, 46929; GCN2-NEXT: v_mov_b32_e32 v3, s06930; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen6931; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen6932; GCN2-NEXT: v_mov_b32_e32 v5, s126933; GCN2-NEXT: v_mov_b32_e32 v4, s136934; GCN2-NEXT: s_waitcnt vmcnt(0)6935; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1]6936; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc6937; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc6938; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen6939; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen6940; GCN2-NEXT: .LBB47_4: ; %atomicrmw.end6941; GCN2-NEXT: v_mov_b32_e32 v2, s106942; GCN2-NEXT: v_mov_b32_e32 v3, s116943; GCN2-NEXT: s_waitcnt vmcnt(0)6944; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]6945; GCN2-NEXT: s_endpgm6946;6947; GFX12-LABEL: atomic_min_i64_ret_addr64:6948; GFX12: ; %bb.0: ; %entry6949; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x246950; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base6951; GFX12-NEXT: s_wait_kmcnt 0x06952; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 36953; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)6954; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]6955; GFX12-NEXT: s_cmp_eq_u32 s1, s96956; GFX12-NEXT: s_cselect_b32 s6, -1, 06957; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6958; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s66959; GFX12-NEXT: s_cbranch_vccz .LBB47_26960; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global6961; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16962; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s56963; GFX12-NEXT: flat_atomic_min_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE6964; GFX12-NEXT: s_wait_loadcnt_dscnt 0x06965; GFX12-NEXT: global_inv scope:SCOPE_SE6966; GFX12-NEXT: s_cbranch_execz .LBB47_36967; GFX12-NEXT: s_branch .LBB47_46968; GFX12-NEXT: .LBB47_2:6969; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr16970; GFX12-NEXT: .LBB47_3: ; %atomicrmw.private6971; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 06972; GFX12-NEXT: s_cselect_b32 s0, s0, -16973; GFX12-NEXT: scratch_load_b64 v[0:1], off, s06974; GFX12-NEXT: s_wait_loadcnt 0x06975; GFX12-NEXT: v_cmp_ge_i64_e32 vcc_lo, s[4:5], v[0:1]6976; GFX12-NEXT: v_cndmask_b32_e32 v3, s5, v1, vcc_lo6977; GFX12-NEXT: v_cndmask_b32_e32 v2, s4, v0, vcc_lo6978; GFX12-NEXT: scratch_store_b64 off, v[2:3], s06979; GFX12-NEXT: .LBB47_4: ; %atomicrmw.end6980; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s36981; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]6982; GFX12-NEXT: s_endpgm6983entry:6984 %ptr = getelementptr inbounds i64, ptr %out, i64 %index6985 %tmp0 = atomicrmw volatile min ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !06986 store i64 %tmp0, ptr %out26987 ret void6988}6989 6990define amdgpu_kernel void @atomic_umin_i64_offset(ptr %out, i64 %in) {6991; GCN1-LABEL: atomic_umin_i64_offset:6992; GCN1: ; %bb.0: ; %entry6993; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD06994; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD16995; GCN1-NEXT: s_mov_b32 s14, -16996; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96997; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d6998; GCN1-NEXT: s_mov_b32 s15, 0xe8f0006999; GCN1-NEXT: s_add_u32 s12, s12, s117000; GCN1-NEXT: s_addc_u32 s13, s13, 07001; GCN1-NEXT: s_waitcnt lgkmcnt(0)7002; GCN1-NEXT: s_add_u32 s0, s0, 327003; GCN1-NEXT: s_addc_u32 s1, s1, 07004; GCN1-NEXT: s_cmp_eq_u32 s1, s47005; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 07006; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]7007; GCN1-NEXT: s_mov_b64 s[4:5], -17008; GCN1-NEXT: s_cbranch_vccnz .LBB48_37009; GCN1-NEXT: ; %bb.1: ; %Flow7010; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]7011; GCN1-NEXT: s_cbranch_vccz .LBB48_47012; GCN1-NEXT: .LBB48_2: ; %atomicrmw.phi7013; GCN1-NEXT: s_endpgm7014; GCN1-NEXT: .LBB48_3: ; %atomicrmw.global7015; GCN1-NEXT: v_mov_b32_e32 v0, s07016; GCN1-NEXT: v_mov_b32_e32 v2, s27017; GCN1-NEXT: v_mov_b32_e32 v1, s17018; GCN1-NEXT: v_mov_b32_e32 v3, s37019; GCN1-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3]7020; GCN1-NEXT: s_waitcnt lgkmcnt(0)7021; GCN1-NEXT: s_cbranch_execnz .LBB48_27022; GCN1-NEXT: .LBB48_4: ; %atomicrmw.private7023; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 07024; GCN1-NEXT: v_mov_b32_e32 v5, s27025; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec7026; GCN1-NEXT: s_cselect_b32 s0, s0, -17027; GCN1-NEXT: v_mov_b32_e32 v2, s07028; GCN1-NEXT: s_add_i32 s0, s0, 47029; GCN1-NEXT: v_mov_b32_e32 v3, s07030; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen7031; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen7032; GCN1-NEXT: v_mov_b32_e32 v4, s37033; GCN1-NEXT: s_waitcnt vmcnt(0)7034; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]7035; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc7036; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc7037; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen7038; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen7039; GCN1-NEXT: s_endpgm7040;7041; GCN2-LABEL: atomic_umin_i64_offset:7042; GCN2: ; %bb.0: ; %entry7043; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD07044; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD17045; GCN2-NEXT: s_mov_b32 s90, -17046; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247047; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf47048; GCN2-NEXT: s_mov_b32 s91, 0xe800007049; GCN2-NEXT: s_add_u32 s88, s88, s117050; GCN2-NEXT: s_addc_u32 s89, s89, 07051; GCN2-NEXT: s_waitcnt lgkmcnt(0)7052; GCN2-NEXT: s_add_u32 s0, s0, 327053; GCN2-NEXT: s_addc_u32 s1, s1, 07054; GCN2-NEXT: s_cmp_eq_u32 s1, s47055; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 07056; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]7057; GCN2-NEXT: s_mov_b64 s[4:5], -17058; GCN2-NEXT: s_cbranch_vccnz .LBB48_37059; GCN2-NEXT: ; %bb.1: ; %Flow7060; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]7061; GCN2-NEXT: s_cbranch_vccz .LBB48_47062; GCN2-NEXT: .LBB48_2: ; %atomicrmw.phi7063; GCN2-NEXT: s_endpgm7064; GCN2-NEXT: .LBB48_3: ; %atomicrmw.global7065; GCN2-NEXT: v_mov_b32_e32 v0, s07066; GCN2-NEXT: v_mov_b32_e32 v2, s27067; GCN2-NEXT: v_mov_b32_e32 v1, s17068; GCN2-NEXT: v_mov_b32_e32 v3, s37069; GCN2-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3]7070; GCN2-NEXT: s_waitcnt lgkmcnt(0)7071; GCN2-NEXT: s_cbranch_execnz .LBB48_27072; GCN2-NEXT: .LBB48_4: ; %atomicrmw.private7073; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 07074; GCN2-NEXT: s_cselect_b32 s0, s0, -17075; GCN2-NEXT: v_mov_b32_e32 v2, s07076; GCN2-NEXT: s_add_i32 s0, s0, 47077; GCN2-NEXT: v_mov_b32_e32 v3, s07078; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen7079; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen7080; GCN2-NEXT: v_mov_b32_e32 v5, s27081; GCN2-NEXT: v_mov_b32_e32 v4, s37082; GCN2-NEXT: s_waitcnt vmcnt(0)7083; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]7084; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc7085; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc7086; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen7087; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen7088; GCN2-NEXT: s_endpgm7089;7090; GFX12-LABEL: atomic_umin_i64_offset:7091; GFX12: ; %bb.0: ; %entry7092; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x247093; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base7094; GFX12-NEXT: s_wait_kmcnt 0x07095; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 327096; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)7097; GFX12-NEXT: s_cmp_eq_u32 s1, s77098; GFX12-NEXT: s_cselect_b32 s4, -1, 07099; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s47100; GFX12-NEXT: s_mov_b32 s4, -17101; GFX12-NEXT: s_cbranch_vccnz .LBB48_37102; GFX12-NEXT: ; %bb.1: ; %Flow7103; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s47104; GFX12-NEXT: s_cbranch_vccz .LBB48_47105; GFX12-NEXT: .LBB48_2: ; %atomicrmw.phi7106; GFX12-NEXT: s_endpgm7107; GFX12-NEXT: .LBB48_3: ; %atomicrmw.global7108; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17109; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s37110; GFX12-NEXT: flat_atomic_min_u64 v[0:1], v[2:3] scope:SCOPE_SE7111; GFX12-NEXT: s_wait_storecnt_dscnt 0x07112; GFX12-NEXT: global_inv scope:SCOPE_SE7113; GFX12-NEXT: s_cbranch_execnz .LBB48_27114; GFX12-NEXT: .LBB48_4: ; %atomicrmw.private7115; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 07116; GFX12-NEXT: s_cselect_b32 s0, s0, -17117; GFX12-NEXT: scratch_load_b64 v[0:1], off, s07118; GFX12-NEXT: s_wait_loadcnt 0x07119; GFX12-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[2:3], v[0:1]7120; GFX12-NEXT: v_cndmask_b32_e32 v1, s3, v1, vcc_lo7121; GFX12-NEXT: v_cndmask_b32_e32 v0, s2, v0, vcc_lo7122; GFX12-NEXT: scratch_store_b64 off, v[0:1], s07123; GFX12-NEXT: s_endpgm7124entry:7125 %gep = getelementptr inbounds i64, ptr %out, i64 47126 %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !07127 ret void7128}7129 7130define amdgpu_kernel void @atomic_umin_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {7131; GCN1-LABEL: atomic_umin_i64_ret_offset:7132; GCN1: ; %bb.0: ; %entry7133; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD07134; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD17135; GCN1-NEXT: s_mov_b32 s14, -17136; GCN1-NEXT: s_mov_b32 s15, 0xe8f0007137; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x97138; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f7139; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd7140; GCN1-NEXT: s_add_u32 s12, s12, s117141; GCN1-NEXT: s_addc_u32 s13, s13, 07142; GCN1-NEXT: s_waitcnt lgkmcnt(0)7143; GCN1-NEXT: s_add_u32 s0, s0, 327144; GCN1-NEXT: s_addc_u32 s1, s1, 07145; GCN1-NEXT: s_cmp_eq_u32 s1, s67146; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 07147; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]7148; GCN1-NEXT: s_cbranch_vccz .LBB49_27149; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global7150; GCN1-NEXT: v_mov_b32_e32 v0, s07151; GCN1-NEXT: v_mov_b32_e32 v2, s47152; GCN1-NEXT: v_mov_b32_e32 v1, s17153; GCN1-NEXT: v_mov_b32_e32 v3, s57154; GCN1-NEXT: flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc7155; GCN1-NEXT: s_waitcnt lgkmcnt(0)7156; GCN1-NEXT: s_cbranch_execz .LBB49_37157; GCN1-NEXT: s_branch .LBB49_47158; GCN1-NEXT: .LBB49_2:7159; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr17160; GCN1-NEXT: .LBB49_3: ; %atomicrmw.private7161; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 07162; GCN1-NEXT: v_mov_b32_e32 v5, s47163; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec7164; GCN1-NEXT: s_cselect_b32 s0, s0, -17165; GCN1-NEXT: v_mov_b32_e32 v2, s07166; GCN1-NEXT: s_add_i32 s0, s0, 47167; GCN1-NEXT: v_mov_b32_e32 v3, s07168; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen7169; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen7170; GCN1-NEXT: v_mov_b32_e32 v4, s57171; GCN1-NEXT: s_waitcnt vmcnt(0)7172; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[0:1]7173; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc7174; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc7175; GCN1-NEXT: buffer_store_dword v5, v2, s[12:15], 0 offen7176; GCN1-NEXT: buffer_store_dword v4, v3, s[12:15], 0 offen7177; GCN1-NEXT: .LBB49_4: ; %atomicrmw.end7178; GCN1-NEXT: v_mov_b32_e32 v2, s27179; GCN1-NEXT: v_mov_b32_e32 v3, s37180; GCN1-NEXT: s_waitcnt vmcnt(0)7181; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]7182; GCN1-NEXT: s_endpgm7183;7184; GCN2-LABEL: atomic_umin_i64_ret_offset:7185; GCN2: ; %bb.0: ; %entry7186; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD07187; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD17188; GCN2-NEXT: s_mov_b32 s90, -17189; GCN2-NEXT: s_mov_b32 s91, 0xe800007190; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247191; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc7192; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x347193; GCN2-NEXT: s_add_u32 s88, s88, s117194; GCN2-NEXT: s_addc_u32 s89, s89, 07195; GCN2-NEXT: s_waitcnt lgkmcnt(0)7196; GCN2-NEXT: s_add_u32 s0, s0, 327197; GCN2-NEXT: s_addc_u32 s1, s1, 07198; GCN2-NEXT: s_cmp_eq_u32 s1, s67199; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 07200; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]7201; GCN2-NEXT: s_cbranch_vccz .LBB49_27202; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global7203; GCN2-NEXT: v_mov_b32_e32 v0, s07204; GCN2-NEXT: v_mov_b32_e32 v2, s47205; GCN2-NEXT: v_mov_b32_e32 v1, s17206; GCN2-NEXT: v_mov_b32_e32 v3, s57207; GCN2-NEXT: flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc7208; GCN2-NEXT: s_waitcnt lgkmcnt(0)7209; GCN2-NEXT: s_cbranch_execz .LBB49_37210; GCN2-NEXT: s_branch .LBB49_47211; GCN2-NEXT: .LBB49_2:7212; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr17213; GCN2-NEXT: .LBB49_3: ; %atomicrmw.private7214; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 07215; GCN2-NEXT: s_cselect_b32 s0, s0, -17216; GCN2-NEXT: v_mov_b32_e32 v2, s07217; GCN2-NEXT: s_add_i32 s0, s0, 47218; GCN2-NEXT: v_mov_b32_e32 v3, s07219; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen7220; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen7221; GCN2-NEXT: v_mov_b32_e32 v5, s47222; GCN2-NEXT: v_mov_b32_e32 v4, s57223; GCN2-NEXT: s_waitcnt vmcnt(0)7224; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[0:1]7225; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc7226; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc7227; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen7228; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen7229; GCN2-NEXT: .LBB49_4: ; %atomicrmw.end7230; GCN2-NEXT: v_mov_b32_e32 v2, s27231; GCN2-NEXT: v_mov_b32_e32 v3, s37232; GCN2-NEXT: s_waitcnt vmcnt(0)7233; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]7234; GCN2-NEXT: s_endpgm7235;7236; GFX12-LABEL: atomic_umin_i64_ret_offset:7237; GFX12: ; %bb.0: ; %entry7238; GFX12-NEXT: s_clause 0x17239; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x247240; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x347241; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base7242; GFX12-NEXT: s_wait_kmcnt 0x07243; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 327244; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)7245; GFX12-NEXT: s_cmp_eq_u32 s1, s77246; GFX12-NEXT: s_cselect_b32 s6, -1, 07247; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s67248; GFX12-NEXT: s_cbranch_vccz .LBB49_27249; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global7250; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17251; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s57252; GFX12-NEXT: flat_atomic_min_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE7253; GFX12-NEXT: s_wait_loadcnt_dscnt 0x07254; GFX12-NEXT: global_inv scope:SCOPE_SE7255; GFX12-NEXT: s_cbranch_execz .LBB49_37256; GFX12-NEXT: s_branch .LBB49_47257; GFX12-NEXT: .LBB49_2:7258; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr17259; GFX12-NEXT: .LBB49_3: ; %atomicrmw.private7260; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 07261; GFX12-NEXT: s_cselect_b32 s0, s0, -17262; GFX12-NEXT: scratch_load_b64 v[0:1], off, s07263; GFX12-NEXT: s_wait_loadcnt 0x07264; GFX12-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[0:1]7265; GFX12-NEXT: v_cndmask_b32_e32 v3, s5, v1, vcc_lo7266; GFX12-NEXT: v_cndmask_b32_e32 v2, s4, v0, vcc_lo7267; GFX12-NEXT: scratch_store_b64 off, v[2:3], s07268; GFX12-NEXT: .LBB49_4: ; %atomicrmw.end7269; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s37270; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]7271; GFX12-NEXT: s_endpgm7272entry:7273 %gep = getelementptr inbounds i64, ptr %out, i64 47274 %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !07275 store i64 %tmp0, ptr %out27276 ret void7277}7278 7279define amdgpu_kernel void @atomic_umin_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {7280; GCN1-LABEL: atomic_umin_i64_addr64_offset:7281; GCN1: ; %bb.0: ; %entry7282; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD07283; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD17284; GCN1-NEXT: s_mov_b32 s14, -17285; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd7286; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x97287; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f7288; GCN1-NEXT: s_mov_b32 s15, 0xe8f0007289; GCN1-NEXT: s_add_u32 s12, s12, s117290; GCN1-NEXT: s_addc_u32 s13, s13, 07291; GCN1-NEXT: s_waitcnt lgkmcnt(0)7292; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 37293; GCN1-NEXT: s_add_u32 s0, s0, s47294; GCN1-NEXT: s_addc_u32 s1, s1, s57295; GCN1-NEXT: s_add_u32 s0, s0, 327296; GCN1-NEXT: s_addc_u32 s1, s1, 07297; GCN1-NEXT: s_cmp_eq_u32 s1, s87298; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 07299; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]7300; GCN1-NEXT: s_mov_b64 s[4:5], -17301; GCN1-NEXT: s_cbranch_vccnz .LBB50_37302; GCN1-NEXT: ; %bb.1: ; %Flow7303; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]7304; GCN1-NEXT: s_cbranch_vccz .LBB50_47305; GCN1-NEXT: .LBB50_2: ; %atomicrmw.phi7306; GCN1-NEXT: s_endpgm7307; GCN1-NEXT: .LBB50_3: ; %atomicrmw.global7308; GCN1-NEXT: v_mov_b32_e32 v0, s07309; GCN1-NEXT: v_mov_b32_e32 v2, s27310; GCN1-NEXT: v_mov_b32_e32 v1, s17311; GCN1-NEXT: v_mov_b32_e32 v3, s37312; GCN1-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3]7313; GCN1-NEXT: s_waitcnt lgkmcnt(0)7314; GCN1-NEXT: s_cbranch_execnz .LBB50_27315; GCN1-NEXT: .LBB50_4: ; %atomicrmw.private7316; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 07317; GCN1-NEXT: v_mov_b32_e32 v5, s27318; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec7319; GCN1-NEXT: s_cselect_b32 s0, s0, -17320; GCN1-NEXT: v_mov_b32_e32 v2, s07321; GCN1-NEXT: s_add_i32 s0, s0, 47322; GCN1-NEXT: v_mov_b32_e32 v3, s07323; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen7324; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen7325; GCN1-NEXT: v_mov_b32_e32 v4, s37326; GCN1-NEXT: s_waitcnt vmcnt(0)7327; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]7328; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc7329; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc7330; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen7331; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen7332; GCN1-NEXT: s_endpgm7333;7334; GCN2-LABEL: atomic_umin_i64_addr64_offset:7335; GCN2: ; %bb.0: ; %entry7336; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD07337; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD17338; GCN2-NEXT: s_mov_b32 s90, -17339; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x347340; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247341; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc7342; GCN2-NEXT: s_mov_b32 s91, 0xe800007343; GCN2-NEXT: s_add_u32 s88, s88, s117344; GCN2-NEXT: s_addc_u32 s89, s89, 07345; GCN2-NEXT: s_waitcnt lgkmcnt(0)7346; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 37347; GCN2-NEXT: s_add_u32 s0, s0, s47348; GCN2-NEXT: s_addc_u32 s1, s1, s57349; GCN2-NEXT: s_add_u32 s0, s0, 327350; GCN2-NEXT: s_addc_u32 s1, s1, 07351; GCN2-NEXT: s_cmp_eq_u32 s1, s87352; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 07353; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]7354; GCN2-NEXT: s_mov_b64 s[4:5], -17355; GCN2-NEXT: s_cbranch_vccnz .LBB50_37356; GCN2-NEXT: ; %bb.1: ; %Flow7357; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]7358; GCN2-NEXT: s_cbranch_vccz .LBB50_47359; GCN2-NEXT: .LBB50_2: ; %atomicrmw.phi7360; GCN2-NEXT: s_endpgm7361; GCN2-NEXT: .LBB50_3: ; %atomicrmw.global7362; GCN2-NEXT: v_mov_b32_e32 v0, s07363; GCN2-NEXT: v_mov_b32_e32 v2, s27364; GCN2-NEXT: v_mov_b32_e32 v1, s17365; GCN2-NEXT: v_mov_b32_e32 v3, s37366; GCN2-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3]7367; GCN2-NEXT: s_waitcnt lgkmcnt(0)7368; GCN2-NEXT: s_cbranch_execnz .LBB50_27369; GCN2-NEXT: .LBB50_4: ; %atomicrmw.private7370; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 07371; GCN2-NEXT: s_cselect_b32 s0, s0, -17372; GCN2-NEXT: v_mov_b32_e32 v2, s07373; GCN2-NEXT: s_add_i32 s0, s0, 47374; GCN2-NEXT: v_mov_b32_e32 v3, s07375; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen7376; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen7377; GCN2-NEXT: v_mov_b32_e32 v5, s27378; GCN2-NEXT: v_mov_b32_e32 v4, s37379; GCN2-NEXT: s_waitcnt vmcnt(0)7380; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]7381; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc7382; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc7383; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen7384; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen7385; GCN2-NEXT: s_endpgm7386;7387; GFX12-LABEL: atomic_umin_i64_addr64_offset:7388; GFX12: ; %bb.0: ; %entry7389; GFX12-NEXT: s_clause 0x17390; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x347391; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x247392; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base7393; GFX12-NEXT: s_wait_kmcnt 0x07394; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 37395; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)7396; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]7397; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 327398; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)7399; GFX12-NEXT: s_cmp_eq_u32 s1, s77400; GFX12-NEXT: s_cselect_b32 s4, -1, 07401; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s47402; GFX12-NEXT: s_mov_b32 s4, -17403; GFX12-NEXT: s_cbranch_vccnz .LBB50_37404; GFX12-NEXT: ; %bb.1: ; %Flow7405; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s47406; GFX12-NEXT: s_cbranch_vccz .LBB50_47407; GFX12-NEXT: .LBB50_2: ; %atomicrmw.phi7408; GFX12-NEXT: s_endpgm7409; GFX12-NEXT: .LBB50_3: ; %atomicrmw.global7410; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17411; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s37412; GFX12-NEXT: flat_atomic_min_u64 v[0:1], v[2:3] scope:SCOPE_SE7413; GFX12-NEXT: s_wait_storecnt_dscnt 0x07414; GFX12-NEXT: global_inv scope:SCOPE_SE7415; GFX12-NEXT: s_cbranch_execnz .LBB50_27416; GFX12-NEXT: .LBB50_4: ; %atomicrmw.private7417; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 07418; GFX12-NEXT: s_cselect_b32 s0, s0, -17419; GFX12-NEXT: scratch_load_b64 v[0:1], off, s07420; GFX12-NEXT: s_wait_loadcnt 0x07421; GFX12-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[2:3], v[0:1]7422; GFX12-NEXT: v_cndmask_b32_e32 v1, s3, v1, vcc_lo7423; GFX12-NEXT: v_cndmask_b32_e32 v0, s2, v0, vcc_lo7424; GFX12-NEXT: scratch_store_b64 off, v[0:1], s07425; GFX12-NEXT: s_endpgm7426entry:7427 %ptr = getelementptr inbounds i64, ptr %out, i64 %index7428 %gep = getelementptr inbounds i64, ptr %ptr, i64 47429 %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !07430 ret void7431}7432 7433define amdgpu_kernel void @atomic_umin_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {7434; GCN1-LABEL: atomic_umin_i64_ret_addr64_offset:7435; GCN1: ; %bb.0: ; %entry7436; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD07437; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD17438; GCN1-NEXT: s_mov_b32 s18, -17439; GCN1-NEXT: s_mov_b32 s19, 0xe8f0007440; GCN1-NEXT: s_add_u32 s16, s16, s117441; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x97442; GCN1-NEXT: s_load_dword s2, s[4:5], 0x417443; GCN1-NEXT: s_addc_u32 s17, s17, 07444; GCN1-NEXT: s_waitcnt lgkmcnt(0)7445; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 37446; GCN1-NEXT: s_add_u32 s0, s8, s07447; GCN1-NEXT: s_addc_u32 s1, s9, s17448; GCN1-NEXT: s_add_u32 s0, s0, 327449; GCN1-NEXT: s_addc_u32 s1, s1, 07450; GCN1-NEXT: s_cmp_eq_u32 s1, s27451; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 07452; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]7453; GCN1-NEXT: s_cbranch_vccz .LBB51_27454; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global7455; GCN1-NEXT: v_mov_b32_e32 v0, s07456; GCN1-NEXT: v_mov_b32_e32 v2, s127457; GCN1-NEXT: v_mov_b32_e32 v1, s17458; GCN1-NEXT: v_mov_b32_e32 v3, s137459; GCN1-NEXT: flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc7460; GCN1-NEXT: s_waitcnt lgkmcnt(0)7461; GCN1-NEXT: s_cbranch_execz .LBB51_37462; GCN1-NEXT: s_branch .LBB51_47463; GCN1-NEXT: .LBB51_2:7464; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr17465; GCN1-NEXT: .LBB51_3: ; %atomicrmw.private7466; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 07467; GCN1-NEXT: v_mov_b32_e32 v5, s127468; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec7469; GCN1-NEXT: s_cselect_b32 s0, s0, -17470; GCN1-NEXT: v_mov_b32_e32 v2, s07471; GCN1-NEXT: s_add_i32 s0, s0, 47472; GCN1-NEXT: v_mov_b32_e32 v3, s07473; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen7474; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen7475; GCN1-NEXT: v_mov_b32_e32 v4, s137476; GCN1-NEXT: s_waitcnt vmcnt(0)7477; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[12:13], v[0:1]7478; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc7479; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc7480; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen7481; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen7482; GCN1-NEXT: .LBB51_4: ; %atomicrmw.end7483; GCN1-NEXT: v_mov_b32_e32 v2, s107484; GCN1-NEXT: v_mov_b32_e32 v3, s117485; GCN1-NEXT: s_waitcnt vmcnt(0)7486; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]7487; GCN1-NEXT: s_endpgm7488;7489; GCN2-LABEL: atomic_umin_i64_ret_addr64_offset:7490; GCN2: ; %bb.0: ; %entry7491; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD07492; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD17493; GCN2-NEXT: s_mov_b32 s90, -17494; GCN2-NEXT: s_mov_b32 s91, 0xe800007495; GCN2-NEXT: s_add_u32 s88, s88, s117496; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x247497; GCN2-NEXT: s_load_dword s2, s[4:5], 0x1047498; GCN2-NEXT: s_addc_u32 s89, s89, 07499; GCN2-NEXT: s_waitcnt lgkmcnt(0)7500; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 37501; GCN2-NEXT: s_add_u32 s0, s8, s07502; GCN2-NEXT: s_addc_u32 s1, s9, s17503; GCN2-NEXT: s_add_u32 s0, s0, 327504; GCN2-NEXT: s_addc_u32 s1, s1, 07505; GCN2-NEXT: s_cmp_eq_u32 s1, s27506; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 07507; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]7508; GCN2-NEXT: s_cbranch_vccz .LBB51_27509; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global7510; GCN2-NEXT: v_mov_b32_e32 v0, s07511; GCN2-NEXT: v_mov_b32_e32 v2, s127512; GCN2-NEXT: v_mov_b32_e32 v1, s17513; GCN2-NEXT: v_mov_b32_e32 v3, s137514; GCN2-NEXT: flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc7515; GCN2-NEXT: s_waitcnt lgkmcnt(0)7516; GCN2-NEXT: s_cbranch_execz .LBB51_37517; GCN2-NEXT: s_branch .LBB51_47518; GCN2-NEXT: .LBB51_2:7519; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr17520; GCN2-NEXT: .LBB51_3: ; %atomicrmw.private7521; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 07522; GCN2-NEXT: s_cselect_b32 s0, s0, -17523; GCN2-NEXT: v_mov_b32_e32 v2, s07524; GCN2-NEXT: s_add_i32 s0, s0, 47525; GCN2-NEXT: v_mov_b32_e32 v3, s07526; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen7527; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen7528; GCN2-NEXT: v_mov_b32_e32 v5, s127529; GCN2-NEXT: v_mov_b32_e32 v4, s137530; GCN2-NEXT: s_waitcnt vmcnt(0)7531; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[12:13], v[0:1]7532; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc7533; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc7534; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen7535; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen7536; GCN2-NEXT: .LBB51_4: ; %atomicrmw.end7537; GCN2-NEXT: v_mov_b32_e32 v2, s107538; GCN2-NEXT: v_mov_b32_e32 v3, s117539; GCN2-NEXT: s_waitcnt vmcnt(0)7540; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]7541; GCN2-NEXT: s_endpgm7542;7543; GFX12-LABEL: atomic_umin_i64_ret_addr64_offset:7544; GFX12: ; %bb.0: ; %entry7545; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x247546; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base7547; GFX12-NEXT: s_wait_kmcnt 0x07548; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 37549; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)7550; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]7551; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 327552; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)7553; GFX12-NEXT: s_cmp_eq_u32 s1, s97554; GFX12-NEXT: s_cselect_b32 s6, -1, 07555; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s67556; GFX12-NEXT: s_cbranch_vccz .LBB51_27557; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global7558; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17559; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s57560; GFX12-NEXT: flat_atomic_min_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE7561; GFX12-NEXT: s_wait_loadcnt_dscnt 0x07562; GFX12-NEXT: global_inv scope:SCOPE_SE7563; GFX12-NEXT: s_cbranch_execz .LBB51_37564; GFX12-NEXT: s_branch .LBB51_47565; GFX12-NEXT: .LBB51_2:7566; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr17567; GFX12-NEXT: .LBB51_3: ; %atomicrmw.private7568; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 07569; GFX12-NEXT: s_cselect_b32 s0, s0, -17570; GFX12-NEXT: scratch_load_b64 v[0:1], off, s07571; GFX12-NEXT: s_wait_loadcnt 0x07572; GFX12-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[0:1]7573; GFX12-NEXT: v_cndmask_b32_e32 v3, s5, v1, vcc_lo7574; GFX12-NEXT: v_cndmask_b32_e32 v2, s4, v0, vcc_lo7575; GFX12-NEXT: scratch_store_b64 off, v[2:3], s07576; GFX12-NEXT: .LBB51_4: ; %atomicrmw.end7577; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s37578; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]7579; GFX12-NEXT: s_endpgm7580entry:7581 %ptr = getelementptr inbounds i64, ptr %out, i64 %index7582 %gep = getelementptr inbounds i64, ptr %ptr, i64 47583 %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !07584 store i64 %tmp0, ptr %out27585 ret void7586}7587 7588define amdgpu_kernel void @atomic_umin_i64(ptr %out, i64 %in) {7589; GCN1-LABEL: atomic_umin_i64:7590; GCN1: ; %bb.0: ; %entry7591; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD07592; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD17593; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x97594; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d7595; GCN1-NEXT: s_mov_b32 s14, -17596; GCN1-NEXT: s_mov_b32 s15, 0xe8f0007597; GCN1-NEXT: s_add_u32 s12, s12, s117598; GCN1-NEXT: s_addc_u32 s13, s13, 07599; GCN1-NEXT: s_waitcnt lgkmcnt(0)7600; GCN1-NEXT: s_cmp_eq_u32 s1, s47601; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 07602; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]7603; GCN1-NEXT: s_mov_b64 s[4:5], -17604; GCN1-NEXT: s_cbranch_vccnz .LBB52_37605; GCN1-NEXT: ; %bb.1: ; %Flow7606; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]7607; GCN1-NEXT: s_cbranch_vccz .LBB52_47608; GCN1-NEXT: .LBB52_2: ; %atomicrmw.phi7609; GCN1-NEXT: s_endpgm7610; GCN1-NEXT: .LBB52_3: ; %atomicrmw.global7611; GCN1-NEXT: v_mov_b32_e32 v0, s07612; GCN1-NEXT: v_mov_b32_e32 v2, s27613; GCN1-NEXT: v_mov_b32_e32 v1, s17614; GCN1-NEXT: v_mov_b32_e32 v3, s37615; GCN1-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3]7616; GCN1-NEXT: s_waitcnt lgkmcnt(0)7617; GCN1-NEXT: s_cbranch_execnz .LBB52_27618; GCN1-NEXT: .LBB52_4: ; %atomicrmw.private7619; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 07620; GCN1-NEXT: v_mov_b32_e32 v5, s27621; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec7622; GCN1-NEXT: s_cselect_b32 s0, s0, -17623; GCN1-NEXT: v_mov_b32_e32 v2, s07624; GCN1-NEXT: s_add_i32 s0, s0, 47625; GCN1-NEXT: v_mov_b32_e32 v3, s07626; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen7627; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen7628; GCN1-NEXT: v_mov_b32_e32 v4, s37629; GCN1-NEXT: s_waitcnt vmcnt(0)7630; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]7631; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc7632; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc7633; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen7634; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen7635; GCN1-NEXT: s_endpgm7636;7637; GCN2-LABEL: atomic_umin_i64:7638; GCN2: ; %bb.0: ; %entry7639; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD07640; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD17641; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247642; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf47643; GCN2-NEXT: s_mov_b32 s90, -17644; GCN2-NEXT: s_mov_b32 s91, 0xe800007645; GCN2-NEXT: s_add_u32 s88, s88, s117646; GCN2-NEXT: s_addc_u32 s89, s89, 07647; GCN2-NEXT: s_waitcnt lgkmcnt(0)7648; GCN2-NEXT: s_cmp_eq_u32 s1, s47649; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 07650; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]7651; GCN2-NEXT: s_mov_b64 s[4:5], -17652; GCN2-NEXT: s_cbranch_vccnz .LBB52_37653; GCN2-NEXT: ; %bb.1: ; %Flow7654; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]7655; GCN2-NEXT: s_cbranch_vccz .LBB52_47656; GCN2-NEXT: .LBB52_2: ; %atomicrmw.phi7657; GCN2-NEXT: s_endpgm7658; GCN2-NEXT: .LBB52_3: ; %atomicrmw.global7659; GCN2-NEXT: v_mov_b32_e32 v0, s07660; GCN2-NEXT: v_mov_b32_e32 v2, s27661; GCN2-NEXT: v_mov_b32_e32 v1, s17662; GCN2-NEXT: v_mov_b32_e32 v3, s37663; GCN2-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3]7664; GCN2-NEXT: s_waitcnt lgkmcnt(0)7665; GCN2-NEXT: s_cbranch_execnz .LBB52_27666; GCN2-NEXT: .LBB52_4: ; %atomicrmw.private7667; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 07668; GCN2-NEXT: s_cselect_b32 s0, s0, -17669; GCN2-NEXT: v_mov_b32_e32 v2, s07670; GCN2-NEXT: s_add_i32 s0, s0, 47671; GCN2-NEXT: v_mov_b32_e32 v3, s07672; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen7673; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen7674; GCN2-NEXT: v_mov_b32_e32 v5, s27675; GCN2-NEXT: v_mov_b32_e32 v4, s37676; GCN2-NEXT: s_waitcnt vmcnt(0)7677; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]7678; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc7679; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc7680; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen7681; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen7682; GCN2-NEXT: s_endpgm7683;7684; GFX12-LABEL: atomic_umin_i64:7685; GFX12: ; %bb.0: ; %entry7686; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x247687; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base7688; GFX12-NEXT: s_wait_kmcnt 0x07689; GFX12-NEXT: s_cmp_eq_u32 s1, s77690; GFX12-NEXT: s_cselect_b32 s4, -1, 07691; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7692; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s47693; GFX12-NEXT: s_mov_b32 s4, -17694; GFX12-NEXT: s_cbranch_vccnz .LBB52_37695; GFX12-NEXT: ; %bb.1: ; %Flow7696; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s47697; GFX12-NEXT: s_cbranch_vccz .LBB52_47698; GFX12-NEXT: .LBB52_2: ; %atomicrmw.phi7699; GFX12-NEXT: s_endpgm7700; GFX12-NEXT: .LBB52_3: ; %atomicrmw.global7701; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17702; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s37703; GFX12-NEXT: flat_atomic_min_u64 v[0:1], v[2:3] scope:SCOPE_SE7704; GFX12-NEXT: s_wait_storecnt_dscnt 0x07705; GFX12-NEXT: global_inv scope:SCOPE_SE7706; GFX12-NEXT: s_cbranch_execnz .LBB52_27707; GFX12-NEXT: .LBB52_4: ; %atomicrmw.private7708; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 07709; GFX12-NEXT: s_cselect_b32 s0, s0, -17710; GFX12-NEXT: scratch_load_b64 v[0:1], off, s07711; GFX12-NEXT: s_wait_loadcnt 0x07712; GFX12-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[2:3], v[0:1]7713; GFX12-NEXT: v_cndmask_b32_e32 v1, s3, v1, vcc_lo7714; GFX12-NEXT: v_cndmask_b32_e32 v0, s2, v0, vcc_lo7715; GFX12-NEXT: scratch_store_b64 off, v[0:1], s07716; GFX12-NEXT: s_endpgm7717entry:7718 %tmp0 = atomicrmw volatile umin ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !07719 ret void7720}7721 7722define amdgpu_kernel void @atomic_umin_i64_ret(ptr %out, ptr %out2, i64 %in) {7723; GCN1-LABEL: atomic_umin_i64_ret:7724; GCN1: ; %bb.0: ; %entry7725; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD07726; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD17727; GCN1-NEXT: s_mov_b32 s14, -17728; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f7729; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x97730; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd7731; GCN1-NEXT: s_mov_b32 s15, 0xe8f0007732; GCN1-NEXT: s_add_u32 s12, s12, s117733; GCN1-NEXT: s_addc_u32 s13, s13, 07734; GCN1-NEXT: s_waitcnt lgkmcnt(0)7735; GCN1-NEXT: s_cmp_eq_u32 s1, s67736; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 07737; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]7738; GCN1-NEXT: s_cbranch_vccz .LBB53_27739; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global7740; GCN1-NEXT: v_mov_b32_e32 v0, s07741; GCN1-NEXT: v_mov_b32_e32 v2, s47742; GCN1-NEXT: v_mov_b32_e32 v1, s17743; GCN1-NEXT: v_mov_b32_e32 v3, s57744; GCN1-NEXT: flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc7745; GCN1-NEXT: s_waitcnt lgkmcnt(0)7746; GCN1-NEXT: s_cbranch_execz .LBB53_37747; GCN1-NEXT: s_branch .LBB53_47748; GCN1-NEXT: .LBB53_2:7749; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr17750; GCN1-NEXT: .LBB53_3: ; %atomicrmw.private7751; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 07752; GCN1-NEXT: v_mov_b32_e32 v5, s47753; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec7754; GCN1-NEXT: s_cselect_b32 s0, s0, -17755; GCN1-NEXT: v_mov_b32_e32 v2, s07756; GCN1-NEXT: s_add_i32 s0, s0, 47757; GCN1-NEXT: v_mov_b32_e32 v3, s07758; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen7759; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen7760; GCN1-NEXT: v_mov_b32_e32 v4, s57761; GCN1-NEXT: s_waitcnt vmcnt(0)7762; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[0:1]7763; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc7764; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc7765; GCN1-NEXT: buffer_store_dword v5, v2, s[12:15], 0 offen7766; GCN1-NEXT: buffer_store_dword v4, v3, s[12:15], 0 offen7767; GCN1-NEXT: .LBB53_4: ; %atomicrmw.end7768; GCN1-NEXT: v_mov_b32_e32 v2, s27769; GCN1-NEXT: v_mov_b32_e32 v3, s37770; GCN1-NEXT: s_waitcnt vmcnt(0)7771; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]7772; GCN1-NEXT: s_endpgm7773;7774; GCN2-LABEL: atomic_umin_i64_ret:7775; GCN2: ; %bb.0: ; %entry7776; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD07777; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD17778; GCN2-NEXT: s_mov_b32 s90, -17779; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc7780; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247781; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x347782; GCN2-NEXT: s_mov_b32 s91, 0xe800007783; GCN2-NEXT: s_add_u32 s88, s88, s117784; GCN2-NEXT: s_addc_u32 s89, s89, 07785; GCN2-NEXT: s_waitcnt lgkmcnt(0)7786; GCN2-NEXT: s_cmp_eq_u32 s1, s67787; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 07788; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]7789; GCN2-NEXT: s_cbranch_vccz .LBB53_27790; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global7791; GCN2-NEXT: v_mov_b32_e32 v0, s07792; GCN2-NEXT: v_mov_b32_e32 v2, s47793; GCN2-NEXT: v_mov_b32_e32 v1, s17794; GCN2-NEXT: v_mov_b32_e32 v3, s57795; GCN2-NEXT: flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc7796; GCN2-NEXT: s_waitcnt lgkmcnt(0)7797; GCN2-NEXT: s_cbranch_execz .LBB53_37798; GCN2-NEXT: s_branch .LBB53_47799; GCN2-NEXT: .LBB53_2:7800; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr17801; GCN2-NEXT: .LBB53_3: ; %atomicrmw.private7802; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 07803; GCN2-NEXT: s_cselect_b32 s0, s0, -17804; GCN2-NEXT: v_mov_b32_e32 v2, s07805; GCN2-NEXT: s_add_i32 s0, s0, 47806; GCN2-NEXT: v_mov_b32_e32 v3, s07807; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen7808; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen7809; GCN2-NEXT: v_mov_b32_e32 v5, s47810; GCN2-NEXT: v_mov_b32_e32 v4, s57811; GCN2-NEXT: s_waitcnt vmcnt(0)7812; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[0:1]7813; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc7814; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc7815; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen7816; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen7817; GCN2-NEXT: .LBB53_4: ; %atomicrmw.end7818; GCN2-NEXT: v_mov_b32_e32 v2, s27819; GCN2-NEXT: v_mov_b32_e32 v3, s37820; GCN2-NEXT: s_waitcnt vmcnt(0)7821; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]7822; GCN2-NEXT: s_endpgm7823;7824; GFX12-LABEL: atomic_umin_i64_ret:7825; GFX12: ; %bb.0: ; %entry7826; GFX12-NEXT: s_clause 0x17827; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x247828; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x347829; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base7830; GFX12-NEXT: s_wait_kmcnt 0x07831; GFX12-NEXT: s_cmp_eq_u32 s1, s77832; GFX12-NEXT: s_cselect_b32 s6, -1, 07833; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7834; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s67835; GFX12-NEXT: s_cbranch_vccz .LBB53_27836; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global7837; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17838; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s57839; GFX12-NEXT: flat_atomic_min_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE7840; GFX12-NEXT: s_wait_loadcnt_dscnt 0x07841; GFX12-NEXT: global_inv scope:SCOPE_SE7842; GFX12-NEXT: s_cbranch_execz .LBB53_37843; GFX12-NEXT: s_branch .LBB53_47844; GFX12-NEXT: .LBB53_2:7845; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr17846; GFX12-NEXT: .LBB53_3: ; %atomicrmw.private7847; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 07848; GFX12-NEXT: s_cselect_b32 s0, s0, -17849; GFX12-NEXT: scratch_load_b64 v[0:1], off, s07850; GFX12-NEXT: s_wait_loadcnt 0x07851; GFX12-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[0:1]7852; GFX12-NEXT: v_cndmask_b32_e32 v3, s5, v1, vcc_lo7853; GFX12-NEXT: v_cndmask_b32_e32 v2, s4, v0, vcc_lo7854; GFX12-NEXT: scratch_store_b64 off, v[2:3], s07855; GFX12-NEXT: .LBB53_4: ; %atomicrmw.end7856; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s37857; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]7858; GFX12-NEXT: s_endpgm7859entry:7860 %tmp0 = atomicrmw volatile umin ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !07861 store i64 %tmp0, ptr %out27862 ret void7863}7864 7865define amdgpu_kernel void @atomic_umin_i64_addr64(ptr %out, i64 %in, i64 %index) {7866; GCN1-LABEL: atomic_umin_i64_addr64:7867; GCN1: ; %bb.0: ; %entry7868; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD07869; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD17870; GCN1-NEXT: s_mov_b32 s14, -17871; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd7872; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x97873; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f7874; GCN1-NEXT: s_mov_b32 s15, 0xe8f0007875; GCN1-NEXT: s_add_u32 s12, s12, s117876; GCN1-NEXT: s_addc_u32 s13, s13, 07877; GCN1-NEXT: s_waitcnt lgkmcnt(0)7878; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 37879; GCN1-NEXT: s_add_u32 s0, s0, s47880; GCN1-NEXT: s_addc_u32 s1, s1, s57881; GCN1-NEXT: s_cmp_eq_u32 s1, s87882; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 07883; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]7884; GCN1-NEXT: s_mov_b64 s[4:5], -17885; GCN1-NEXT: s_cbranch_vccnz .LBB54_37886; GCN1-NEXT: ; %bb.1: ; %Flow7887; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]7888; GCN1-NEXT: s_cbranch_vccz .LBB54_47889; GCN1-NEXT: .LBB54_2: ; %atomicrmw.phi7890; GCN1-NEXT: s_endpgm7891; GCN1-NEXT: .LBB54_3: ; %atomicrmw.global7892; GCN1-NEXT: v_mov_b32_e32 v0, s07893; GCN1-NEXT: v_mov_b32_e32 v2, s27894; GCN1-NEXT: v_mov_b32_e32 v1, s17895; GCN1-NEXT: v_mov_b32_e32 v3, s37896; GCN1-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3]7897; GCN1-NEXT: s_waitcnt lgkmcnt(0)7898; GCN1-NEXT: s_cbranch_execnz .LBB54_27899; GCN1-NEXT: .LBB54_4: ; %atomicrmw.private7900; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 07901; GCN1-NEXT: v_mov_b32_e32 v5, s27902; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec7903; GCN1-NEXT: s_cselect_b32 s0, s0, -17904; GCN1-NEXT: v_mov_b32_e32 v2, s07905; GCN1-NEXT: s_add_i32 s0, s0, 47906; GCN1-NEXT: v_mov_b32_e32 v3, s07907; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen7908; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen7909; GCN1-NEXT: v_mov_b32_e32 v4, s37910; GCN1-NEXT: s_waitcnt vmcnt(0)7911; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]7912; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc7913; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc7914; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen7915; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen7916; GCN1-NEXT: s_endpgm7917;7918; GCN2-LABEL: atomic_umin_i64_addr64:7919; GCN2: ; %bb.0: ; %entry7920; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD07921; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD17922; GCN2-NEXT: s_mov_b32 s90, -17923; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x347924; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247925; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc7926; GCN2-NEXT: s_mov_b32 s91, 0xe800007927; GCN2-NEXT: s_add_u32 s88, s88, s117928; GCN2-NEXT: s_addc_u32 s89, s89, 07929; GCN2-NEXT: s_waitcnt lgkmcnt(0)7930; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 37931; GCN2-NEXT: s_add_u32 s0, s0, s47932; GCN2-NEXT: s_addc_u32 s1, s1, s57933; GCN2-NEXT: s_cmp_eq_u32 s1, s87934; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 07935; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]7936; GCN2-NEXT: s_mov_b64 s[4:5], -17937; GCN2-NEXT: s_cbranch_vccnz .LBB54_37938; GCN2-NEXT: ; %bb.1: ; %Flow7939; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]7940; GCN2-NEXT: s_cbranch_vccz .LBB54_47941; GCN2-NEXT: .LBB54_2: ; %atomicrmw.phi7942; GCN2-NEXT: s_endpgm7943; GCN2-NEXT: .LBB54_3: ; %atomicrmw.global7944; GCN2-NEXT: v_mov_b32_e32 v0, s07945; GCN2-NEXT: v_mov_b32_e32 v2, s27946; GCN2-NEXT: v_mov_b32_e32 v1, s17947; GCN2-NEXT: v_mov_b32_e32 v3, s37948; GCN2-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3]7949; GCN2-NEXT: s_waitcnt lgkmcnt(0)7950; GCN2-NEXT: s_cbranch_execnz .LBB54_27951; GCN2-NEXT: .LBB54_4: ; %atomicrmw.private7952; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 07953; GCN2-NEXT: s_cselect_b32 s0, s0, -17954; GCN2-NEXT: v_mov_b32_e32 v2, s07955; GCN2-NEXT: s_add_i32 s0, s0, 47956; GCN2-NEXT: v_mov_b32_e32 v3, s07957; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen7958; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen7959; GCN2-NEXT: v_mov_b32_e32 v5, s27960; GCN2-NEXT: v_mov_b32_e32 v4, s37961; GCN2-NEXT: s_waitcnt vmcnt(0)7962; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]7963; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc7964; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc7965; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen7966; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen7967; GCN2-NEXT: s_endpgm7968;7969; GFX12-LABEL: atomic_umin_i64_addr64:7970; GFX12: ; %bb.0: ; %entry7971; GFX12-NEXT: s_clause 0x17972; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x347973; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x247974; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base7975; GFX12-NEXT: s_wait_kmcnt 0x07976; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 37977; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)7978; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]7979; GFX12-NEXT: s_cmp_eq_u32 s1, s77980; GFX12-NEXT: s_cselect_b32 s4, -1, 07981; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7982; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s47983; GFX12-NEXT: s_mov_b32 s4, -17984; GFX12-NEXT: s_cbranch_vccnz .LBB54_37985; GFX12-NEXT: ; %bb.1: ; %Flow7986; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s47987; GFX12-NEXT: s_cbranch_vccz .LBB54_47988; GFX12-NEXT: .LBB54_2: ; %atomicrmw.phi7989; GFX12-NEXT: s_endpgm7990; GFX12-NEXT: .LBB54_3: ; %atomicrmw.global7991; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17992; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s37993; GFX12-NEXT: flat_atomic_min_u64 v[0:1], v[2:3] scope:SCOPE_SE7994; GFX12-NEXT: s_wait_storecnt_dscnt 0x07995; GFX12-NEXT: global_inv scope:SCOPE_SE7996; GFX12-NEXT: s_cbranch_execnz .LBB54_27997; GFX12-NEXT: .LBB54_4: ; %atomicrmw.private7998; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 07999; GFX12-NEXT: s_cselect_b32 s0, s0, -18000; GFX12-NEXT: scratch_load_b64 v[0:1], off, s08001; GFX12-NEXT: s_wait_loadcnt 0x08002; GFX12-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[2:3], v[0:1]8003; GFX12-NEXT: v_cndmask_b32_e32 v1, s3, v1, vcc_lo8004; GFX12-NEXT: v_cndmask_b32_e32 v0, s2, v0, vcc_lo8005; GFX12-NEXT: scratch_store_b64 off, v[0:1], s08006; GFX12-NEXT: s_endpgm8007entry:8008 %ptr = getelementptr inbounds i64, ptr %out, i64 %index8009 %tmp0 = atomicrmw volatile umin ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !08010 ret void8011}8012 8013define amdgpu_kernel void @atomic_umin_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {8014; GCN1-LABEL: atomic_umin_i64_ret_addr64:8015; GCN1: ; %bb.0: ; %entry8016; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD08017; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD18018; GCN1-NEXT: s_mov_b32 s18, -18019; GCN1-NEXT: s_mov_b32 s19, 0xe8f0008020; GCN1-NEXT: s_add_u32 s16, s16, s118021; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x98022; GCN1-NEXT: s_load_dword s2, s[4:5], 0x418023; GCN1-NEXT: s_addc_u32 s17, s17, 08024; GCN1-NEXT: s_waitcnt lgkmcnt(0)8025; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 38026; GCN1-NEXT: s_add_u32 s0, s8, s08027; GCN1-NEXT: s_addc_u32 s1, s9, s18028; GCN1-NEXT: s_cmp_eq_u32 s1, s28029; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 08030; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]8031; GCN1-NEXT: s_cbranch_vccz .LBB55_28032; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global8033; GCN1-NEXT: v_mov_b32_e32 v0, s08034; GCN1-NEXT: v_mov_b32_e32 v2, s128035; GCN1-NEXT: v_mov_b32_e32 v1, s18036; GCN1-NEXT: v_mov_b32_e32 v3, s138037; GCN1-NEXT: flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc8038; GCN1-NEXT: s_waitcnt lgkmcnt(0)8039; GCN1-NEXT: s_cbranch_execz .LBB55_38040; GCN1-NEXT: s_branch .LBB55_48041; GCN1-NEXT: .LBB55_2:8042; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr18043; GCN1-NEXT: .LBB55_3: ; %atomicrmw.private8044; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 08045; GCN1-NEXT: v_mov_b32_e32 v5, s128046; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec8047; GCN1-NEXT: s_cselect_b32 s0, s0, -18048; GCN1-NEXT: v_mov_b32_e32 v2, s08049; GCN1-NEXT: s_add_i32 s0, s0, 48050; GCN1-NEXT: v_mov_b32_e32 v3, s08051; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen8052; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen8053; GCN1-NEXT: v_mov_b32_e32 v4, s138054; GCN1-NEXT: s_waitcnt vmcnt(0)8055; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[12:13], v[0:1]8056; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc8057; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc8058; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen8059; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen8060; GCN1-NEXT: .LBB55_4: ; %atomicrmw.end8061; GCN1-NEXT: v_mov_b32_e32 v2, s108062; GCN1-NEXT: v_mov_b32_e32 v3, s118063; GCN1-NEXT: s_waitcnt vmcnt(0)8064; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]8065; GCN1-NEXT: s_endpgm8066;8067; GCN2-LABEL: atomic_umin_i64_ret_addr64:8068; GCN2: ; %bb.0: ; %entry8069; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD08070; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD18071; GCN2-NEXT: s_mov_b32 s90, -18072; GCN2-NEXT: s_mov_b32 s91, 0xe800008073; GCN2-NEXT: s_add_u32 s88, s88, s118074; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x248075; GCN2-NEXT: s_load_dword s2, s[4:5], 0x1048076; GCN2-NEXT: s_addc_u32 s89, s89, 08077; GCN2-NEXT: s_waitcnt lgkmcnt(0)8078; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 38079; GCN2-NEXT: s_add_u32 s0, s8, s08080; GCN2-NEXT: s_addc_u32 s1, s9, s18081; GCN2-NEXT: s_cmp_eq_u32 s1, s28082; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 08083; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]8084; GCN2-NEXT: s_cbranch_vccz .LBB55_28085; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global8086; GCN2-NEXT: v_mov_b32_e32 v0, s08087; GCN2-NEXT: v_mov_b32_e32 v2, s128088; GCN2-NEXT: v_mov_b32_e32 v1, s18089; GCN2-NEXT: v_mov_b32_e32 v3, s138090; GCN2-NEXT: flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc8091; GCN2-NEXT: s_waitcnt lgkmcnt(0)8092; GCN2-NEXT: s_cbranch_execz .LBB55_38093; GCN2-NEXT: s_branch .LBB55_48094; GCN2-NEXT: .LBB55_2:8095; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr18096; GCN2-NEXT: .LBB55_3: ; %atomicrmw.private8097; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 08098; GCN2-NEXT: s_cselect_b32 s0, s0, -18099; GCN2-NEXT: v_mov_b32_e32 v2, s08100; GCN2-NEXT: s_add_i32 s0, s0, 48101; GCN2-NEXT: v_mov_b32_e32 v3, s08102; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen8103; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen8104; GCN2-NEXT: v_mov_b32_e32 v5, s128105; GCN2-NEXT: v_mov_b32_e32 v4, s138106; GCN2-NEXT: s_waitcnt vmcnt(0)8107; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[12:13], v[0:1]8108; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc8109; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc8110; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen8111; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen8112; GCN2-NEXT: .LBB55_4: ; %atomicrmw.end8113; GCN2-NEXT: v_mov_b32_e32 v2, s108114; GCN2-NEXT: v_mov_b32_e32 v3, s118115; GCN2-NEXT: s_waitcnt vmcnt(0)8116; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]8117; GCN2-NEXT: s_endpgm8118;8119; GFX12-LABEL: atomic_umin_i64_ret_addr64:8120; GFX12: ; %bb.0: ; %entry8121; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x248122; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base8123; GFX12-NEXT: s_wait_kmcnt 0x08124; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 38125; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)8126; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]8127; GFX12-NEXT: s_cmp_eq_u32 s1, s98128; GFX12-NEXT: s_cselect_b32 s6, -1, 08129; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8130; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s68131; GFX12-NEXT: s_cbranch_vccz .LBB55_28132; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global8133; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18134; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s58135; GFX12-NEXT: flat_atomic_min_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE8136; GFX12-NEXT: s_wait_loadcnt_dscnt 0x08137; GFX12-NEXT: global_inv scope:SCOPE_SE8138; GFX12-NEXT: s_cbranch_execz .LBB55_38139; GFX12-NEXT: s_branch .LBB55_48140; GFX12-NEXT: .LBB55_2:8141; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr18142; GFX12-NEXT: .LBB55_3: ; %atomicrmw.private8143; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 08144; GFX12-NEXT: s_cselect_b32 s0, s0, -18145; GFX12-NEXT: scratch_load_b64 v[0:1], off, s08146; GFX12-NEXT: s_wait_loadcnt 0x08147; GFX12-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[0:1]8148; GFX12-NEXT: v_cndmask_b32_e32 v3, s5, v1, vcc_lo8149; GFX12-NEXT: v_cndmask_b32_e32 v2, s4, v0, vcc_lo8150; GFX12-NEXT: scratch_store_b64 off, v[2:3], s08151; GFX12-NEXT: .LBB55_4: ; %atomicrmw.end8152; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s38153; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]8154; GFX12-NEXT: s_endpgm8155entry:8156 %ptr = getelementptr inbounds i64, ptr %out, i64 %index8157 %tmp0 = atomicrmw volatile umin ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !08158 store i64 %tmp0, ptr %out28159 ret void8160}8161 8162define amdgpu_kernel void @atomic_or_i64_offset(ptr %out, i64 %in) {8163; GCN1-LABEL: atomic_or_i64_offset:8164; GCN1: ; %bb.0: ; %entry8165; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD08166; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD18167; GCN1-NEXT: s_mov_b32 s14, -18168; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x98169; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d8170; GCN1-NEXT: s_mov_b32 s15, 0xe8f0008171; GCN1-NEXT: s_add_u32 s12, s12, s118172; GCN1-NEXT: s_addc_u32 s13, s13, 08173; GCN1-NEXT: s_waitcnt lgkmcnt(0)8174; GCN1-NEXT: s_add_u32 s0, s0, 328175; GCN1-NEXT: s_addc_u32 s1, s1, 08176; GCN1-NEXT: s_cmp_eq_u32 s1, s48177; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 08178; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]8179; GCN1-NEXT: s_mov_b64 s[4:5], -18180; GCN1-NEXT: s_cbranch_vccnz .LBB56_38181; GCN1-NEXT: ; %bb.1: ; %Flow8182; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]8183; GCN1-NEXT: s_cbranch_vccz .LBB56_48184; GCN1-NEXT: .LBB56_2: ; %atomicrmw.phi8185; GCN1-NEXT: s_endpgm8186; GCN1-NEXT: .LBB56_3: ; %atomicrmw.global8187; GCN1-NEXT: v_mov_b32_e32 v0, s08188; GCN1-NEXT: v_mov_b32_e32 v2, s28189; GCN1-NEXT: v_mov_b32_e32 v1, s18190; GCN1-NEXT: v_mov_b32_e32 v3, s38191; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]8192; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8193; GCN1-NEXT: buffer_wbinvl1_vol8194; GCN1-NEXT: s_cbranch_execnz .LBB56_28195; GCN1-NEXT: .LBB56_4: ; %atomicrmw.private8196; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 08197; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec8198; GCN1-NEXT: s_cselect_b32 s0, s0, -18199; GCN1-NEXT: v_mov_b32_e32 v0, s08200; GCN1-NEXT: s_add_i32 s0, s0, 48201; GCN1-NEXT: v_mov_b32_e32 v1, s08202; GCN1-NEXT: buffer_load_dword v2, v0, s[12:15], 0 offen8203; GCN1-NEXT: buffer_load_dword v3, v1, s[12:15], 0 offen8204; GCN1-NEXT: s_waitcnt vmcnt(1)8205; GCN1-NEXT: v_or_b32_e32 v2, s2, v28206; GCN1-NEXT: s_waitcnt vmcnt(0)8207; GCN1-NEXT: v_or_b32_e32 v3, s3, v38208; GCN1-NEXT: buffer_store_dword v2, v0, s[12:15], 0 offen8209; GCN1-NEXT: buffer_store_dword v3, v1, s[12:15], 0 offen8210; GCN1-NEXT: s_endpgm8211;8212; GCN2-LABEL: atomic_or_i64_offset:8213; GCN2: ; %bb.0: ; %entry8214; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD08215; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD18216; GCN2-NEXT: s_mov_b32 s90, -18217; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x248218; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf48219; GCN2-NEXT: s_mov_b32 s91, 0xe800008220; GCN2-NEXT: s_add_u32 s88, s88, s118221; GCN2-NEXT: s_addc_u32 s89, s89, 08222; GCN2-NEXT: s_waitcnt lgkmcnt(0)8223; GCN2-NEXT: s_add_u32 s0, s0, 328224; GCN2-NEXT: s_addc_u32 s1, s1, 08225; GCN2-NEXT: s_cmp_eq_u32 s1, s48226; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 08227; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]8228; GCN2-NEXT: s_mov_b64 s[4:5], -18229; GCN2-NEXT: s_cbranch_vccnz .LBB56_38230; GCN2-NEXT: ; %bb.1: ; %Flow8231; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]8232; GCN2-NEXT: s_cbranch_vccz .LBB56_48233; GCN2-NEXT: .LBB56_2: ; %atomicrmw.phi8234; GCN2-NEXT: s_endpgm8235; GCN2-NEXT: .LBB56_3: ; %atomicrmw.global8236; GCN2-NEXT: v_mov_b32_e32 v0, s08237; GCN2-NEXT: v_mov_b32_e32 v2, s28238; GCN2-NEXT: v_mov_b32_e32 v1, s18239; GCN2-NEXT: v_mov_b32_e32 v3, s38240; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]8241; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8242; GCN2-NEXT: buffer_wbinvl1_vol8243; GCN2-NEXT: s_cbranch_execnz .LBB56_28244; GCN2-NEXT: .LBB56_4: ; %atomicrmw.private8245; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 08246; GCN2-NEXT: s_cselect_b32 s0, s0, -18247; GCN2-NEXT: v_mov_b32_e32 v0, s08248; GCN2-NEXT: s_add_i32 s0, s0, 48249; GCN2-NEXT: v_mov_b32_e32 v1, s08250; GCN2-NEXT: buffer_load_dword v2, v0, s[88:91], 0 offen8251; GCN2-NEXT: buffer_load_dword v3, v1, s[88:91], 0 offen8252; GCN2-NEXT: s_waitcnt vmcnt(1)8253; GCN2-NEXT: v_or_b32_e32 v2, s2, v28254; GCN2-NEXT: s_waitcnt vmcnt(0)8255; GCN2-NEXT: v_or_b32_e32 v3, s3, v38256; GCN2-NEXT: buffer_store_dword v2, v0, s[88:91], 0 offen8257; GCN2-NEXT: buffer_store_dword v3, v1, s[88:91], 0 offen8258; GCN2-NEXT: s_endpgm8259;8260; GFX12-LABEL: atomic_or_i64_offset:8261; GFX12: ; %bb.0: ; %entry8262; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x248263; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base8264; GFX12-NEXT: s_wait_kmcnt 0x08265; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 328266; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)8267; GFX12-NEXT: s_cmp_eq_u32 s1, s78268; GFX12-NEXT: s_cselect_b32 s4, -1, 08269; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s48270; GFX12-NEXT: s_mov_b32 s4, -18271; GFX12-NEXT: s_cbranch_vccnz .LBB56_38272; GFX12-NEXT: ; %bb.1: ; %Flow8273; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s48274; GFX12-NEXT: s_cbranch_vccz .LBB56_48275; GFX12-NEXT: .LBB56_2: ; %atomicrmw.phi8276; GFX12-NEXT: s_endpgm8277; GFX12-NEXT: .LBB56_3: ; %atomicrmw.global8278; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18279; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s38280; GFX12-NEXT: flat_atomic_or_b64 v[0:1], v[2:3] scope:SCOPE_DEV8281; GFX12-NEXT: s_wait_storecnt_dscnt 0x08282; GFX12-NEXT: global_inv scope:SCOPE_DEV8283; GFX12-NEXT: s_cbranch_execnz .LBB56_28284; GFX12-NEXT: .LBB56_4: ; %atomicrmw.private8285; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 08286; GFX12-NEXT: s_cselect_b32 s0, s0, -18287; GFX12-NEXT: scratch_load_b64 v[0:1], off, s08288; GFX12-NEXT: s_wait_loadcnt 0x08289; GFX12-NEXT: v_or_b32_e32 v1, s3, v18290; GFX12-NEXT: v_or_b32_e32 v0, s2, v08291; GFX12-NEXT: scratch_store_b64 off, v[0:1], s08292; GFX12-NEXT: s_endpgm8293entry:8294 %gep = getelementptr inbounds i64, ptr %out, i64 48295 %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !08296 ret void8297}8298 8299define amdgpu_kernel void @atomic_or_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {8300; GCN1-LABEL: atomic_or_i64_ret_offset:8301; GCN1: ; %bb.0: ; %entry8302; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD08303; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD18304; GCN1-NEXT: s_mov_b32 s14, -18305; GCN1-NEXT: s_mov_b32 s15, 0xe8f0008306; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x98307; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f8308; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd8309; GCN1-NEXT: s_add_u32 s12, s12, s118310; GCN1-NEXT: s_addc_u32 s13, s13, 08311; GCN1-NEXT: s_waitcnt lgkmcnt(0)8312; GCN1-NEXT: s_add_u32 s0, s0, 328313; GCN1-NEXT: s_addc_u32 s1, s1, 08314; GCN1-NEXT: s_cmp_eq_u32 s1, s68315; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 08316; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]8317; GCN1-NEXT: s_cbranch_vccz .LBB57_28318; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global8319; GCN1-NEXT: v_mov_b32_e32 v0, s08320; GCN1-NEXT: v_mov_b32_e32 v2, s48321; GCN1-NEXT: v_mov_b32_e32 v1, s18322; GCN1-NEXT: v_mov_b32_e32 v3, s58323; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc8324; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8325; GCN1-NEXT: buffer_wbinvl1_vol8326; GCN1-NEXT: s_cbranch_execz .LBB57_38327; GCN1-NEXT: s_branch .LBB57_48328; GCN1-NEXT: .LBB57_2:8329; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr18330; GCN1-NEXT: .LBB57_3: ; %atomicrmw.private8331; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 08332; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec8333; GCN1-NEXT: s_cselect_b32 s0, s0, -18334; GCN1-NEXT: v_mov_b32_e32 v2, s08335; GCN1-NEXT: s_add_i32 s0, s0, 48336; GCN1-NEXT: v_mov_b32_e32 v3, s08337; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen8338; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen8339; GCN1-NEXT: s_waitcnt vmcnt(1)8340; GCN1-NEXT: v_or_b32_e32 v4, s4, v08341; GCN1-NEXT: s_waitcnt vmcnt(0)8342; GCN1-NEXT: v_or_b32_e32 v5, s5, v18343; GCN1-NEXT: buffer_store_dword v4, v2, s[12:15], 0 offen8344; GCN1-NEXT: buffer_store_dword v5, v3, s[12:15], 0 offen8345; GCN1-NEXT: .LBB57_4: ; %atomicrmw.end8346; GCN1-NEXT: v_mov_b32_e32 v2, s28347; GCN1-NEXT: v_mov_b32_e32 v3, s38348; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]8349; GCN1-NEXT: s_endpgm8350;8351; GCN2-LABEL: atomic_or_i64_ret_offset:8352; GCN2: ; %bb.0: ; %entry8353; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD08354; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD18355; GCN2-NEXT: s_mov_b32 s90, -18356; GCN2-NEXT: s_mov_b32 s91, 0xe800008357; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x248358; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc8359; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x348360; GCN2-NEXT: s_add_u32 s88, s88, s118361; GCN2-NEXT: s_addc_u32 s89, s89, 08362; GCN2-NEXT: s_waitcnt lgkmcnt(0)8363; GCN2-NEXT: s_add_u32 s0, s0, 328364; GCN2-NEXT: s_addc_u32 s1, s1, 08365; GCN2-NEXT: s_cmp_eq_u32 s1, s68366; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 08367; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]8368; GCN2-NEXT: s_cbranch_vccz .LBB57_28369; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global8370; GCN2-NEXT: v_mov_b32_e32 v0, s08371; GCN2-NEXT: v_mov_b32_e32 v2, s48372; GCN2-NEXT: v_mov_b32_e32 v1, s18373; GCN2-NEXT: v_mov_b32_e32 v3, s58374; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc8375; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8376; GCN2-NEXT: buffer_wbinvl1_vol8377; GCN2-NEXT: s_cbranch_execz .LBB57_38378; GCN2-NEXT: s_branch .LBB57_48379; GCN2-NEXT: .LBB57_2:8380; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr18381; GCN2-NEXT: .LBB57_3: ; %atomicrmw.private8382; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 08383; GCN2-NEXT: s_cselect_b32 s0, s0, -18384; GCN2-NEXT: v_mov_b32_e32 v2, s08385; GCN2-NEXT: s_add_i32 s0, s0, 48386; GCN2-NEXT: v_mov_b32_e32 v3, s08387; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen8388; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen8389; GCN2-NEXT: s_waitcnt vmcnt(1)8390; GCN2-NEXT: v_or_b32_e32 v4, s4, v08391; GCN2-NEXT: s_waitcnt vmcnt(0)8392; GCN2-NEXT: v_or_b32_e32 v5, s5, v18393; GCN2-NEXT: buffer_store_dword v4, v2, s[88:91], 0 offen8394; GCN2-NEXT: buffer_store_dword v5, v3, s[88:91], 0 offen8395; GCN2-NEXT: .LBB57_4: ; %atomicrmw.end8396; GCN2-NEXT: v_mov_b32_e32 v2, s28397; GCN2-NEXT: v_mov_b32_e32 v3, s38398; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]8399; GCN2-NEXT: s_endpgm8400;8401; GFX12-LABEL: atomic_or_i64_ret_offset:8402; GFX12: ; %bb.0: ; %entry8403; GFX12-NEXT: s_clause 0x18404; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x248405; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x348406; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base8407; GFX12-NEXT: s_wait_kmcnt 0x08408; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 328409; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)8410; GFX12-NEXT: s_cmp_eq_u32 s1, s78411; GFX12-NEXT: s_cselect_b32 s6, -1, 08412; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s68413; GFX12-NEXT: s_cbranch_vccz .LBB57_28414; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global8415; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18416; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s58417; GFX12-NEXT: flat_atomic_or_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV8418; GFX12-NEXT: s_wait_loadcnt_dscnt 0x08419; GFX12-NEXT: global_inv scope:SCOPE_DEV8420; GFX12-NEXT: s_cbranch_execz .LBB57_38421; GFX12-NEXT: s_branch .LBB57_48422; GFX12-NEXT: .LBB57_2:8423; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr18424; GFX12-NEXT: .LBB57_3: ; %atomicrmw.private8425; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 08426; GFX12-NEXT: s_cselect_b32 s0, s0, -18427; GFX12-NEXT: scratch_load_b64 v[0:1], off, s08428; GFX12-NEXT: s_wait_loadcnt 0x08429; GFX12-NEXT: v_or_b32_e32 v3, s5, v18430; GFX12-NEXT: v_or_b32_e32 v2, s4, v08431; GFX12-NEXT: scratch_store_b64 off, v[2:3], s08432; GFX12-NEXT: .LBB57_4: ; %atomicrmw.end8433; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s38434; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]8435; GFX12-NEXT: s_endpgm8436entry:8437 %gep = getelementptr inbounds i64, ptr %out, i64 48438 %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !08439 store i64 %tmp0, ptr %out28440 ret void8441}8442 8443define amdgpu_kernel void @atomic_or_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {8444; GCN1-LABEL: atomic_or_i64_addr64_offset:8445; GCN1: ; %bb.0: ; %entry8446; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD08447; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD18448; GCN1-NEXT: s_mov_b32 s14, -18449; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd8450; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x98451; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f8452; GCN1-NEXT: s_mov_b32 s15, 0xe8f0008453; GCN1-NEXT: s_add_u32 s12, s12, s118454; GCN1-NEXT: s_addc_u32 s13, s13, 08455; GCN1-NEXT: s_waitcnt lgkmcnt(0)8456; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 38457; GCN1-NEXT: s_add_u32 s0, s0, s48458; GCN1-NEXT: s_addc_u32 s1, s1, s58459; GCN1-NEXT: s_add_u32 s0, s0, 328460; GCN1-NEXT: s_addc_u32 s1, s1, 08461; GCN1-NEXT: s_cmp_eq_u32 s1, s88462; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 08463; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]8464; GCN1-NEXT: s_mov_b64 s[4:5], -18465; GCN1-NEXT: s_cbranch_vccnz .LBB58_38466; GCN1-NEXT: ; %bb.1: ; %Flow8467; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]8468; GCN1-NEXT: s_cbranch_vccz .LBB58_48469; GCN1-NEXT: .LBB58_2: ; %atomicrmw.phi8470; GCN1-NEXT: s_endpgm8471; GCN1-NEXT: .LBB58_3: ; %atomicrmw.global8472; GCN1-NEXT: v_mov_b32_e32 v0, s08473; GCN1-NEXT: v_mov_b32_e32 v2, s28474; GCN1-NEXT: v_mov_b32_e32 v1, s18475; GCN1-NEXT: v_mov_b32_e32 v3, s38476; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]8477; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8478; GCN1-NEXT: buffer_wbinvl1_vol8479; GCN1-NEXT: s_cbranch_execnz .LBB58_28480; GCN1-NEXT: .LBB58_4: ; %atomicrmw.private8481; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 08482; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec8483; GCN1-NEXT: s_cselect_b32 s0, s0, -18484; GCN1-NEXT: v_mov_b32_e32 v0, s08485; GCN1-NEXT: s_add_i32 s0, s0, 48486; GCN1-NEXT: v_mov_b32_e32 v1, s08487; GCN1-NEXT: buffer_load_dword v2, v0, s[12:15], 0 offen8488; GCN1-NEXT: buffer_load_dword v3, v1, s[12:15], 0 offen8489; GCN1-NEXT: s_waitcnt vmcnt(1)8490; GCN1-NEXT: v_or_b32_e32 v2, s2, v28491; GCN1-NEXT: s_waitcnt vmcnt(0)8492; GCN1-NEXT: v_or_b32_e32 v3, s3, v38493; GCN1-NEXT: buffer_store_dword v2, v0, s[12:15], 0 offen8494; GCN1-NEXT: buffer_store_dword v3, v1, s[12:15], 0 offen8495; GCN1-NEXT: s_endpgm8496;8497; GCN2-LABEL: atomic_or_i64_addr64_offset:8498; GCN2: ; %bb.0: ; %entry8499; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD08500; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD18501; GCN2-NEXT: s_mov_b32 s90, -18502; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x348503; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x248504; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc8505; GCN2-NEXT: s_mov_b32 s91, 0xe800008506; GCN2-NEXT: s_add_u32 s88, s88, s118507; GCN2-NEXT: s_addc_u32 s89, s89, 08508; GCN2-NEXT: s_waitcnt lgkmcnt(0)8509; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 38510; GCN2-NEXT: s_add_u32 s0, s0, s48511; GCN2-NEXT: s_addc_u32 s1, s1, s58512; GCN2-NEXT: s_add_u32 s0, s0, 328513; GCN2-NEXT: s_addc_u32 s1, s1, 08514; GCN2-NEXT: s_cmp_eq_u32 s1, s88515; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 08516; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]8517; GCN2-NEXT: s_mov_b64 s[4:5], -18518; GCN2-NEXT: s_cbranch_vccnz .LBB58_38519; GCN2-NEXT: ; %bb.1: ; %Flow8520; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]8521; GCN2-NEXT: s_cbranch_vccz .LBB58_48522; GCN2-NEXT: .LBB58_2: ; %atomicrmw.phi8523; GCN2-NEXT: s_endpgm8524; GCN2-NEXT: .LBB58_3: ; %atomicrmw.global8525; GCN2-NEXT: v_mov_b32_e32 v0, s08526; GCN2-NEXT: v_mov_b32_e32 v2, s28527; GCN2-NEXT: v_mov_b32_e32 v1, s18528; GCN2-NEXT: v_mov_b32_e32 v3, s38529; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]8530; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8531; GCN2-NEXT: buffer_wbinvl1_vol8532; GCN2-NEXT: s_cbranch_execnz .LBB58_28533; GCN2-NEXT: .LBB58_4: ; %atomicrmw.private8534; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 08535; GCN2-NEXT: s_cselect_b32 s0, s0, -18536; GCN2-NEXT: v_mov_b32_e32 v0, s08537; GCN2-NEXT: s_add_i32 s0, s0, 48538; GCN2-NEXT: v_mov_b32_e32 v1, s08539; GCN2-NEXT: buffer_load_dword v2, v0, s[88:91], 0 offen8540; GCN2-NEXT: buffer_load_dword v3, v1, s[88:91], 0 offen8541; GCN2-NEXT: s_waitcnt vmcnt(1)8542; GCN2-NEXT: v_or_b32_e32 v2, s2, v28543; GCN2-NEXT: s_waitcnt vmcnt(0)8544; GCN2-NEXT: v_or_b32_e32 v3, s3, v38545; GCN2-NEXT: buffer_store_dword v2, v0, s[88:91], 0 offen8546; GCN2-NEXT: buffer_store_dword v3, v1, s[88:91], 0 offen8547; GCN2-NEXT: s_endpgm8548;8549; GFX12-LABEL: atomic_or_i64_addr64_offset:8550; GFX12: ; %bb.0: ; %entry8551; GFX12-NEXT: s_clause 0x18552; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x348553; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x248554; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base8555; GFX12-NEXT: s_wait_kmcnt 0x08556; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 38557; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)8558; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]8559; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 328560; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)8561; GFX12-NEXT: s_cmp_eq_u32 s1, s78562; GFX12-NEXT: s_cselect_b32 s4, -1, 08563; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s48564; GFX12-NEXT: s_mov_b32 s4, -18565; GFX12-NEXT: s_cbranch_vccnz .LBB58_38566; GFX12-NEXT: ; %bb.1: ; %Flow8567; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s48568; GFX12-NEXT: s_cbranch_vccz .LBB58_48569; GFX12-NEXT: .LBB58_2: ; %atomicrmw.phi8570; GFX12-NEXT: s_endpgm8571; GFX12-NEXT: .LBB58_3: ; %atomicrmw.global8572; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18573; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s38574; GFX12-NEXT: flat_atomic_or_b64 v[0:1], v[2:3] scope:SCOPE_DEV8575; GFX12-NEXT: s_wait_storecnt_dscnt 0x08576; GFX12-NEXT: global_inv scope:SCOPE_DEV8577; GFX12-NEXT: s_cbranch_execnz .LBB58_28578; GFX12-NEXT: .LBB58_4: ; %atomicrmw.private8579; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 08580; GFX12-NEXT: s_cselect_b32 s0, s0, -18581; GFX12-NEXT: scratch_load_b64 v[0:1], off, s08582; GFX12-NEXT: s_wait_loadcnt 0x08583; GFX12-NEXT: v_or_b32_e32 v1, s3, v18584; GFX12-NEXT: v_or_b32_e32 v0, s2, v08585; GFX12-NEXT: scratch_store_b64 off, v[0:1], s08586; GFX12-NEXT: s_endpgm8587entry:8588 %ptr = getelementptr inbounds i64, ptr %out, i64 %index8589 %gep = getelementptr inbounds i64, ptr %ptr, i64 48590 %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !08591 ret void8592}8593 8594define amdgpu_kernel void @atomic_or_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {8595; GCN1-LABEL: atomic_or_i64_ret_addr64_offset:8596; GCN1: ; %bb.0: ; %entry8597; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD08598; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD18599; GCN1-NEXT: s_mov_b32 s18, -18600; GCN1-NEXT: s_mov_b32 s19, 0xe8f0008601; GCN1-NEXT: s_add_u32 s16, s16, s118602; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x98603; GCN1-NEXT: s_load_dword s2, s[4:5], 0x418604; GCN1-NEXT: s_addc_u32 s17, s17, 08605; GCN1-NEXT: s_waitcnt lgkmcnt(0)8606; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 38607; GCN1-NEXT: s_add_u32 s0, s8, s08608; GCN1-NEXT: s_addc_u32 s1, s9, s18609; GCN1-NEXT: s_add_u32 s0, s0, 328610; GCN1-NEXT: s_addc_u32 s1, s1, 08611; GCN1-NEXT: s_cmp_eq_u32 s1, s28612; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 08613; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]8614; GCN1-NEXT: s_cbranch_vccz .LBB59_28615; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global8616; GCN1-NEXT: v_mov_b32_e32 v0, s08617; GCN1-NEXT: v_mov_b32_e32 v2, s128618; GCN1-NEXT: v_mov_b32_e32 v1, s18619; GCN1-NEXT: v_mov_b32_e32 v3, s138620; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc8621; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8622; GCN1-NEXT: buffer_wbinvl1_vol8623; GCN1-NEXT: s_cbranch_execz .LBB59_38624; GCN1-NEXT: s_branch .LBB59_48625; GCN1-NEXT: .LBB59_2:8626; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr18627; GCN1-NEXT: .LBB59_3: ; %atomicrmw.private8628; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 08629; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec8630; GCN1-NEXT: s_cselect_b32 s0, s0, -18631; GCN1-NEXT: v_mov_b32_e32 v2, s08632; GCN1-NEXT: s_add_i32 s0, s0, 48633; GCN1-NEXT: v_mov_b32_e32 v3, s08634; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen8635; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen8636; GCN1-NEXT: s_waitcnt vmcnt(1)8637; GCN1-NEXT: v_or_b32_e32 v4, s12, v08638; GCN1-NEXT: s_waitcnt vmcnt(0)8639; GCN1-NEXT: v_or_b32_e32 v5, s13, v18640; GCN1-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen8641; GCN1-NEXT: buffer_store_dword v5, v3, s[16:19], 0 offen8642; GCN1-NEXT: .LBB59_4: ; %atomicrmw.end8643; GCN1-NEXT: v_mov_b32_e32 v2, s108644; GCN1-NEXT: v_mov_b32_e32 v3, s118645; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]8646; GCN1-NEXT: s_endpgm8647;8648; GCN2-LABEL: atomic_or_i64_ret_addr64_offset:8649; GCN2: ; %bb.0: ; %entry8650; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD08651; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD18652; GCN2-NEXT: s_mov_b32 s90, -18653; GCN2-NEXT: s_mov_b32 s91, 0xe800008654; GCN2-NEXT: s_add_u32 s88, s88, s118655; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x248656; GCN2-NEXT: s_load_dword s2, s[4:5], 0x1048657; GCN2-NEXT: s_addc_u32 s89, s89, 08658; GCN2-NEXT: s_waitcnt lgkmcnt(0)8659; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 38660; GCN2-NEXT: s_add_u32 s0, s8, s08661; GCN2-NEXT: s_addc_u32 s1, s9, s18662; GCN2-NEXT: s_add_u32 s0, s0, 328663; GCN2-NEXT: s_addc_u32 s1, s1, 08664; GCN2-NEXT: s_cmp_eq_u32 s1, s28665; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 08666; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]8667; GCN2-NEXT: s_cbranch_vccz .LBB59_28668; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global8669; GCN2-NEXT: v_mov_b32_e32 v0, s08670; GCN2-NEXT: v_mov_b32_e32 v2, s128671; GCN2-NEXT: v_mov_b32_e32 v1, s18672; GCN2-NEXT: v_mov_b32_e32 v3, s138673; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc8674; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8675; GCN2-NEXT: buffer_wbinvl1_vol8676; GCN2-NEXT: s_cbranch_execz .LBB59_38677; GCN2-NEXT: s_branch .LBB59_48678; GCN2-NEXT: .LBB59_2:8679; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr18680; GCN2-NEXT: .LBB59_3: ; %atomicrmw.private8681; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 08682; GCN2-NEXT: s_cselect_b32 s0, s0, -18683; GCN2-NEXT: v_mov_b32_e32 v2, s08684; GCN2-NEXT: s_add_i32 s0, s0, 48685; GCN2-NEXT: v_mov_b32_e32 v3, s08686; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen8687; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen8688; GCN2-NEXT: s_waitcnt vmcnt(1)8689; GCN2-NEXT: v_or_b32_e32 v4, s12, v08690; GCN2-NEXT: s_waitcnt vmcnt(0)8691; GCN2-NEXT: v_or_b32_e32 v5, s13, v18692; GCN2-NEXT: buffer_store_dword v4, v2, s[88:91], 0 offen8693; GCN2-NEXT: buffer_store_dword v5, v3, s[88:91], 0 offen8694; GCN2-NEXT: .LBB59_4: ; %atomicrmw.end8695; GCN2-NEXT: v_mov_b32_e32 v2, s108696; GCN2-NEXT: v_mov_b32_e32 v3, s118697; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]8698; GCN2-NEXT: s_endpgm8699;8700; GFX12-LABEL: atomic_or_i64_ret_addr64_offset:8701; GFX12: ; %bb.0: ; %entry8702; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x248703; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base8704; GFX12-NEXT: s_wait_kmcnt 0x08705; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 38706; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)8707; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]8708; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 328709; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)8710; GFX12-NEXT: s_cmp_eq_u32 s1, s98711; GFX12-NEXT: s_cselect_b32 s6, -1, 08712; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s68713; GFX12-NEXT: s_cbranch_vccz .LBB59_28714; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global8715; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18716; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s58717; GFX12-NEXT: flat_atomic_or_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV8718; GFX12-NEXT: s_wait_loadcnt_dscnt 0x08719; GFX12-NEXT: global_inv scope:SCOPE_DEV8720; GFX12-NEXT: s_cbranch_execz .LBB59_38721; GFX12-NEXT: s_branch .LBB59_48722; GFX12-NEXT: .LBB59_2:8723; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr18724; GFX12-NEXT: .LBB59_3: ; %atomicrmw.private8725; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 08726; GFX12-NEXT: s_cselect_b32 s0, s0, -18727; GFX12-NEXT: scratch_load_b64 v[0:1], off, s08728; GFX12-NEXT: s_wait_loadcnt 0x08729; GFX12-NEXT: v_or_b32_e32 v3, s5, v18730; GFX12-NEXT: v_or_b32_e32 v2, s4, v08731; GFX12-NEXT: scratch_store_b64 off, v[2:3], s08732; GFX12-NEXT: .LBB59_4: ; %atomicrmw.end8733; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s38734; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]8735; GFX12-NEXT: s_endpgm8736entry:8737 %ptr = getelementptr inbounds i64, ptr %out, i64 %index8738 %gep = getelementptr inbounds i64, ptr %ptr, i64 48739 %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !08740 store i64 %tmp0, ptr %out28741 ret void8742}8743 8744define amdgpu_kernel void @atomic_or_i64(ptr %out, i64 %in) {8745; GCN1-LABEL: atomic_or_i64:8746; GCN1: ; %bb.0: ; %entry8747; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD08748; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD18749; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x98750; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d8751; GCN1-NEXT: s_mov_b32 s14, -18752; GCN1-NEXT: s_mov_b32 s15, 0xe8f0008753; GCN1-NEXT: s_add_u32 s12, s12, s118754; GCN1-NEXT: s_addc_u32 s13, s13, 08755; GCN1-NEXT: s_waitcnt lgkmcnt(0)8756; GCN1-NEXT: s_cmp_eq_u32 s1, s48757; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 08758; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]8759; GCN1-NEXT: s_mov_b64 s[4:5], -18760; GCN1-NEXT: s_cbranch_vccnz .LBB60_38761; GCN1-NEXT: ; %bb.1: ; %Flow8762; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]8763; GCN1-NEXT: s_cbranch_vccz .LBB60_48764; GCN1-NEXT: .LBB60_2: ; %atomicrmw.phi8765; GCN1-NEXT: s_endpgm8766; GCN1-NEXT: .LBB60_3: ; %atomicrmw.global8767; GCN1-NEXT: v_mov_b32_e32 v0, s08768; GCN1-NEXT: v_mov_b32_e32 v2, s28769; GCN1-NEXT: v_mov_b32_e32 v1, s18770; GCN1-NEXT: v_mov_b32_e32 v3, s38771; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]8772; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8773; GCN1-NEXT: buffer_wbinvl1_vol8774; GCN1-NEXT: s_cbranch_execnz .LBB60_28775; GCN1-NEXT: .LBB60_4: ; %atomicrmw.private8776; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 08777; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec8778; GCN1-NEXT: s_cselect_b32 s0, s0, -18779; GCN1-NEXT: v_mov_b32_e32 v0, s08780; GCN1-NEXT: s_add_i32 s0, s0, 48781; GCN1-NEXT: v_mov_b32_e32 v1, s08782; GCN1-NEXT: buffer_load_dword v2, v0, s[12:15], 0 offen8783; GCN1-NEXT: buffer_load_dword v3, v1, s[12:15], 0 offen8784; GCN1-NEXT: s_waitcnt vmcnt(1)8785; GCN1-NEXT: v_or_b32_e32 v2, s2, v28786; GCN1-NEXT: s_waitcnt vmcnt(0)8787; GCN1-NEXT: v_or_b32_e32 v3, s3, v38788; GCN1-NEXT: buffer_store_dword v2, v0, s[12:15], 0 offen8789; GCN1-NEXT: buffer_store_dword v3, v1, s[12:15], 0 offen8790; GCN1-NEXT: s_endpgm8791;8792; GCN2-LABEL: atomic_or_i64:8793; GCN2: ; %bb.0: ; %entry8794; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD08795; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD18796; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x248797; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf48798; GCN2-NEXT: s_mov_b32 s90, -18799; GCN2-NEXT: s_mov_b32 s91, 0xe800008800; GCN2-NEXT: s_add_u32 s88, s88, s118801; GCN2-NEXT: s_addc_u32 s89, s89, 08802; GCN2-NEXT: s_waitcnt lgkmcnt(0)8803; GCN2-NEXT: s_cmp_eq_u32 s1, s48804; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 08805; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]8806; GCN2-NEXT: s_mov_b64 s[4:5], -18807; GCN2-NEXT: s_cbranch_vccnz .LBB60_38808; GCN2-NEXT: ; %bb.1: ; %Flow8809; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]8810; GCN2-NEXT: s_cbranch_vccz .LBB60_48811; GCN2-NEXT: .LBB60_2: ; %atomicrmw.phi8812; GCN2-NEXT: s_endpgm8813; GCN2-NEXT: .LBB60_3: ; %atomicrmw.global8814; GCN2-NEXT: v_mov_b32_e32 v0, s08815; GCN2-NEXT: v_mov_b32_e32 v2, s28816; GCN2-NEXT: v_mov_b32_e32 v1, s18817; GCN2-NEXT: v_mov_b32_e32 v3, s38818; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]8819; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8820; GCN2-NEXT: buffer_wbinvl1_vol8821; GCN2-NEXT: s_cbranch_execnz .LBB60_28822; GCN2-NEXT: .LBB60_4: ; %atomicrmw.private8823; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 08824; GCN2-NEXT: s_cselect_b32 s0, s0, -18825; GCN2-NEXT: v_mov_b32_e32 v0, s08826; GCN2-NEXT: s_add_i32 s0, s0, 48827; GCN2-NEXT: v_mov_b32_e32 v1, s08828; GCN2-NEXT: buffer_load_dword v2, v0, s[88:91], 0 offen8829; GCN2-NEXT: buffer_load_dword v3, v1, s[88:91], 0 offen8830; GCN2-NEXT: s_waitcnt vmcnt(1)8831; GCN2-NEXT: v_or_b32_e32 v2, s2, v28832; GCN2-NEXT: s_waitcnt vmcnt(0)8833; GCN2-NEXT: v_or_b32_e32 v3, s3, v38834; GCN2-NEXT: buffer_store_dword v2, v0, s[88:91], 0 offen8835; GCN2-NEXT: buffer_store_dword v3, v1, s[88:91], 0 offen8836; GCN2-NEXT: s_endpgm8837;8838; GFX12-LABEL: atomic_or_i64:8839; GFX12: ; %bb.0: ; %entry8840; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x248841; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base8842; GFX12-NEXT: s_wait_kmcnt 0x08843; GFX12-NEXT: s_cmp_eq_u32 s1, s78844; GFX12-NEXT: s_cselect_b32 s4, -1, 08845; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8846; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s48847; GFX12-NEXT: s_mov_b32 s4, -18848; GFX12-NEXT: s_cbranch_vccnz .LBB60_38849; GFX12-NEXT: ; %bb.1: ; %Flow8850; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s48851; GFX12-NEXT: s_cbranch_vccz .LBB60_48852; GFX12-NEXT: .LBB60_2: ; %atomicrmw.phi8853; GFX12-NEXT: s_endpgm8854; GFX12-NEXT: .LBB60_3: ; %atomicrmw.global8855; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18856; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s38857; GFX12-NEXT: flat_atomic_or_b64 v[0:1], v[2:3] scope:SCOPE_DEV8858; GFX12-NEXT: s_wait_storecnt_dscnt 0x08859; GFX12-NEXT: global_inv scope:SCOPE_DEV8860; GFX12-NEXT: s_cbranch_execnz .LBB60_28861; GFX12-NEXT: .LBB60_4: ; %atomicrmw.private8862; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 08863; GFX12-NEXT: s_cselect_b32 s0, s0, -18864; GFX12-NEXT: scratch_load_b64 v[0:1], off, s08865; GFX12-NEXT: s_wait_loadcnt 0x08866; GFX12-NEXT: v_or_b32_e32 v1, s3, v18867; GFX12-NEXT: v_or_b32_e32 v0, s2, v08868; GFX12-NEXT: scratch_store_b64 off, v[0:1], s08869; GFX12-NEXT: s_endpgm8870entry:8871 %tmp0 = atomicrmw volatile or ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !08872 ret void8873}8874 8875define amdgpu_kernel void @atomic_or_i64_ret(ptr %out, ptr %out2, i64 %in) {8876; GCN1-LABEL: atomic_or_i64_ret:8877; GCN1: ; %bb.0: ; %entry8878; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD08879; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD18880; GCN1-NEXT: s_mov_b32 s14, -18881; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f8882; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x98883; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd8884; GCN1-NEXT: s_mov_b32 s15, 0xe8f0008885; GCN1-NEXT: s_add_u32 s12, s12, s118886; GCN1-NEXT: s_addc_u32 s13, s13, 08887; GCN1-NEXT: s_waitcnt lgkmcnt(0)8888; GCN1-NEXT: s_cmp_eq_u32 s1, s68889; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 08890; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]8891; GCN1-NEXT: s_cbranch_vccz .LBB61_28892; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global8893; GCN1-NEXT: v_mov_b32_e32 v0, s08894; GCN1-NEXT: v_mov_b32_e32 v2, s48895; GCN1-NEXT: v_mov_b32_e32 v1, s18896; GCN1-NEXT: v_mov_b32_e32 v3, s58897; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc8898; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8899; GCN1-NEXT: buffer_wbinvl1_vol8900; GCN1-NEXT: s_cbranch_execz .LBB61_38901; GCN1-NEXT: s_branch .LBB61_48902; GCN1-NEXT: .LBB61_2:8903; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr18904; GCN1-NEXT: .LBB61_3: ; %atomicrmw.private8905; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 08906; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec8907; GCN1-NEXT: s_cselect_b32 s0, s0, -18908; GCN1-NEXT: v_mov_b32_e32 v2, s08909; GCN1-NEXT: s_add_i32 s0, s0, 48910; GCN1-NEXT: v_mov_b32_e32 v3, s08911; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen8912; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen8913; GCN1-NEXT: s_waitcnt vmcnt(1)8914; GCN1-NEXT: v_or_b32_e32 v4, s4, v08915; GCN1-NEXT: s_waitcnt vmcnt(0)8916; GCN1-NEXT: v_or_b32_e32 v5, s5, v18917; GCN1-NEXT: buffer_store_dword v4, v2, s[12:15], 0 offen8918; GCN1-NEXT: buffer_store_dword v5, v3, s[12:15], 0 offen8919; GCN1-NEXT: .LBB61_4: ; %atomicrmw.end8920; GCN1-NEXT: v_mov_b32_e32 v2, s28921; GCN1-NEXT: v_mov_b32_e32 v3, s38922; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]8923; GCN1-NEXT: s_endpgm8924;8925; GCN2-LABEL: atomic_or_i64_ret:8926; GCN2: ; %bb.0: ; %entry8927; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD08928; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD18929; GCN2-NEXT: s_mov_b32 s90, -18930; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc8931; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x248932; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x348933; GCN2-NEXT: s_mov_b32 s91, 0xe800008934; GCN2-NEXT: s_add_u32 s88, s88, s118935; GCN2-NEXT: s_addc_u32 s89, s89, 08936; GCN2-NEXT: s_waitcnt lgkmcnt(0)8937; GCN2-NEXT: s_cmp_eq_u32 s1, s68938; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 08939; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]8940; GCN2-NEXT: s_cbranch_vccz .LBB61_28941; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global8942; GCN2-NEXT: v_mov_b32_e32 v0, s08943; GCN2-NEXT: v_mov_b32_e32 v2, s48944; GCN2-NEXT: v_mov_b32_e32 v1, s18945; GCN2-NEXT: v_mov_b32_e32 v3, s58946; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc8947; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8948; GCN2-NEXT: buffer_wbinvl1_vol8949; GCN2-NEXT: s_cbranch_execz .LBB61_38950; GCN2-NEXT: s_branch .LBB61_48951; GCN2-NEXT: .LBB61_2:8952; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr18953; GCN2-NEXT: .LBB61_3: ; %atomicrmw.private8954; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 08955; GCN2-NEXT: s_cselect_b32 s0, s0, -18956; GCN2-NEXT: v_mov_b32_e32 v2, s08957; GCN2-NEXT: s_add_i32 s0, s0, 48958; GCN2-NEXT: v_mov_b32_e32 v3, s08959; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen8960; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen8961; GCN2-NEXT: s_waitcnt vmcnt(1)8962; GCN2-NEXT: v_or_b32_e32 v4, s4, v08963; GCN2-NEXT: s_waitcnt vmcnt(0)8964; GCN2-NEXT: v_or_b32_e32 v5, s5, v18965; GCN2-NEXT: buffer_store_dword v4, v2, s[88:91], 0 offen8966; GCN2-NEXT: buffer_store_dword v5, v3, s[88:91], 0 offen8967; GCN2-NEXT: .LBB61_4: ; %atomicrmw.end8968; GCN2-NEXT: v_mov_b32_e32 v2, s28969; GCN2-NEXT: v_mov_b32_e32 v3, s38970; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]8971; GCN2-NEXT: s_endpgm8972;8973; GFX12-LABEL: atomic_or_i64_ret:8974; GFX12: ; %bb.0: ; %entry8975; GFX12-NEXT: s_clause 0x18976; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x248977; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x348978; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base8979; GFX12-NEXT: s_wait_kmcnt 0x08980; GFX12-NEXT: s_cmp_eq_u32 s1, s78981; GFX12-NEXT: s_cselect_b32 s6, -1, 08982; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8983; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s68984; GFX12-NEXT: s_cbranch_vccz .LBB61_28985; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global8986; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18987; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s58988; GFX12-NEXT: flat_atomic_or_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV8989; GFX12-NEXT: s_wait_loadcnt_dscnt 0x08990; GFX12-NEXT: global_inv scope:SCOPE_DEV8991; GFX12-NEXT: s_cbranch_execz .LBB61_38992; GFX12-NEXT: s_branch .LBB61_48993; GFX12-NEXT: .LBB61_2:8994; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr18995; GFX12-NEXT: .LBB61_3: ; %atomicrmw.private8996; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 08997; GFX12-NEXT: s_cselect_b32 s0, s0, -18998; GFX12-NEXT: scratch_load_b64 v[0:1], off, s08999; GFX12-NEXT: s_wait_loadcnt 0x09000; GFX12-NEXT: v_or_b32_e32 v3, s5, v19001; GFX12-NEXT: v_or_b32_e32 v2, s4, v09002; GFX12-NEXT: scratch_store_b64 off, v[2:3], s09003; GFX12-NEXT: .LBB61_4: ; %atomicrmw.end9004; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s39005; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]9006; GFX12-NEXT: s_endpgm9007entry:9008 %tmp0 = atomicrmw volatile or ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !09009 store i64 %tmp0, ptr %out29010 ret void9011}9012 9013define amdgpu_kernel void @atomic_or_i64_addr64(ptr %out, i64 %in, i64 %index) {9014; GCN1-LABEL: atomic_or_i64_addr64:9015; GCN1: ; %bb.0: ; %entry9016; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD09017; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD19018; GCN1-NEXT: s_mov_b32 s14, -19019; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd9020; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x99021; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f9022; GCN1-NEXT: s_mov_b32 s15, 0xe8f0009023; GCN1-NEXT: s_add_u32 s12, s12, s119024; GCN1-NEXT: s_addc_u32 s13, s13, 09025; GCN1-NEXT: s_waitcnt lgkmcnt(0)9026; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 39027; GCN1-NEXT: s_add_u32 s0, s0, s49028; GCN1-NEXT: s_addc_u32 s1, s1, s59029; GCN1-NEXT: s_cmp_eq_u32 s1, s89030; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 09031; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]9032; GCN1-NEXT: s_mov_b64 s[4:5], -19033; GCN1-NEXT: s_cbranch_vccnz .LBB62_39034; GCN1-NEXT: ; %bb.1: ; %Flow9035; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]9036; GCN1-NEXT: s_cbranch_vccz .LBB62_49037; GCN1-NEXT: .LBB62_2: ; %atomicrmw.phi9038; GCN1-NEXT: s_endpgm9039; GCN1-NEXT: .LBB62_3: ; %atomicrmw.global9040; GCN1-NEXT: v_mov_b32_e32 v0, s09041; GCN1-NEXT: v_mov_b32_e32 v2, s29042; GCN1-NEXT: v_mov_b32_e32 v1, s19043; GCN1-NEXT: v_mov_b32_e32 v3, s39044; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]9045; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9046; GCN1-NEXT: buffer_wbinvl1_vol9047; GCN1-NEXT: s_cbranch_execnz .LBB62_29048; GCN1-NEXT: .LBB62_4: ; %atomicrmw.private9049; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 09050; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec9051; GCN1-NEXT: s_cselect_b32 s0, s0, -19052; GCN1-NEXT: v_mov_b32_e32 v0, s09053; GCN1-NEXT: s_add_i32 s0, s0, 49054; GCN1-NEXT: v_mov_b32_e32 v1, s09055; GCN1-NEXT: buffer_load_dword v2, v0, s[12:15], 0 offen9056; GCN1-NEXT: buffer_load_dword v3, v1, s[12:15], 0 offen9057; GCN1-NEXT: s_waitcnt vmcnt(1)9058; GCN1-NEXT: v_or_b32_e32 v2, s2, v29059; GCN1-NEXT: s_waitcnt vmcnt(0)9060; GCN1-NEXT: v_or_b32_e32 v3, s3, v39061; GCN1-NEXT: buffer_store_dword v2, v0, s[12:15], 0 offen9062; GCN1-NEXT: buffer_store_dword v3, v1, s[12:15], 0 offen9063; GCN1-NEXT: s_endpgm9064;9065; GCN2-LABEL: atomic_or_i64_addr64:9066; GCN2: ; %bb.0: ; %entry9067; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD09068; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD19069; GCN2-NEXT: s_mov_b32 s90, -19070; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x349071; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249072; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc9073; GCN2-NEXT: s_mov_b32 s91, 0xe800009074; GCN2-NEXT: s_add_u32 s88, s88, s119075; GCN2-NEXT: s_addc_u32 s89, s89, 09076; GCN2-NEXT: s_waitcnt lgkmcnt(0)9077; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 39078; GCN2-NEXT: s_add_u32 s0, s0, s49079; GCN2-NEXT: s_addc_u32 s1, s1, s59080; GCN2-NEXT: s_cmp_eq_u32 s1, s89081; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 09082; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]9083; GCN2-NEXT: s_mov_b64 s[4:5], -19084; GCN2-NEXT: s_cbranch_vccnz .LBB62_39085; GCN2-NEXT: ; %bb.1: ; %Flow9086; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]9087; GCN2-NEXT: s_cbranch_vccz .LBB62_49088; GCN2-NEXT: .LBB62_2: ; %atomicrmw.phi9089; GCN2-NEXT: s_endpgm9090; GCN2-NEXT: .LBB62_3: ; %atomicrmw.global9091; GCN2-NEXT: v_mov_b32_e32 v0, s09092; GCN2-NEXT: v_mov_b32_e32 v2, s29093; GCN2-NEXT: v_mov_b32_e32 v1, s19094; GCN2-NEXT: v_mov_b32_e32 v3, s39095; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]9096; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9097; GCN2-NEXT: buffer_wbinvl1_vol9098; GCN2-NEXT: s_cbranch_execnz .LBB62_29099; GCN2-NEXT: .LBB62_4: ; %atomicrmw.private9100; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 09101; GCN2-NEXT: s_cselect_b32 s0, s0, -19102; GCN2-NEXT: v_mov_b32_e32 v0, s09103; GCN2-NEXT: s_add_i32 s0, s0, 49104; GCN2-NEXT: v_mov_b32_e32 v1, s09105; GCN2-NEXT: buffer_load_dword v2, v0, s[88:91], 0 offen9106; GCN2-NEXT: buffer_load_dword v3, v1, s[88:91], 0 offen9107; GCN2-NEXT: s_waitcnt vmcnt(1)9108; GCN2-NEXT: v_or_b32_e32 v2, s2, v29109; GCN2-NEXT: s_waitcnt vmcnt(0)9110; GCN2-NEXT: v_or_b32_e32 v3, s3, v39111; GCN2-NEXT: buffer_store_dword v2, v0, s[88:91], 0 offen9112; GCN2-NEXT: buffer_store_dword v3, v1, s[88:91], 0 offen9113; GCN2-NEXT: s_endpgm9114;9115; GFX12-LABEL: atomic_or_i64_addr64:9116; GFX12: ; %bb.0: ; %entry9117; GFX12-NEXT: s_clause 0x19118; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x349119; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x249120; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base9121; GFX12-NEXT: s_wait_kmcnt 0x09122; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 39123; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)9124; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]9125; GFX12-NEXT: s_cmp_eq_u32 s1, s79126; GFX12-NEXT: s_cselect_b32 s4, -1, 09127; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)9128; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s49129; GFX12-NEXT: s_mov_b32 s4, -19130; GFX12-NEXT: s_cbranch_vccnz .LBB62_39131; GFX12-NEXT: ; %bb.1: ; %Flow9132; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s49133; GFX12-NEXT: s_cbranch_vccz .LBB62_49134; GFX12-NEXT: .LBB62_2: ; %atomicrmw.phi9135; GFX12-NEXT: s_endpgm9136; GFX12-NEXT: .LBB62_3: ; %atomicrmw.global9137; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s19138; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s39139; GFX12-NEXT: flat_atomic_or_b64 v[0:1], v[2:3] scope:SCOPE_DEV9140; GFX12-NEXT: s_wait_storecnt_dscnt 0x09141; GFX12-NEXT: global_inv scope:SCOPE_DEV9142; GFX12-NEXT: s_cbranch_execnz .LBB62_29143; GFX12-NEXT: .LBB62_4: ; %atomicrmw.private9144; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 09145; GFX12-NEXT: s_cselect_b32 s0, s0, -19146; GFX12-NEXT: scratch_load_b64 v[0:1], off, s09147; GFX12-NEXT: s_wait_loadcnt 0x09148; GFX12-NEXT: v_or_b32_e32 v1, s3, v19149; GFX12-NEXT: v_or_b32_e32 v0, s2, v09150; GFX12-NEXT: scratch_store_b64 off, v[0:1], s09151; GFX12-NEXT: s_endpgm9152entry:9153 %ptr = getelementptr inbounds i64, ptr %out, i64 %index9154 %tmp0 = atomicrmw volatile or ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !09155 ret void9156}9157 9158define amdgpu_kernel void @atomic_or_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {9159; GCN1-LABEL: atomic_or_i64_ret_addr64:9160; GCN1: ; %bb.0: ; %entry9161; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD09162; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD19163; GCN1-NEXT: s_mov_b32 s18, -19164; GCN1-NEXT: s_mov_b32 s19, 0xe8f0009165; GCN1-NEXT: s_add_u32 s16, s16, s119166; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x99167; GCN1-NEXT: s_load_dword s2, s[4:5], 0x419168; GCN1-NEXT: s_addc_u32 s17, s17, 09169; GCN1-NEXT: s_waitcnt lgkmcnt(0)9170; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 39171; GCN1-NEXT: s_add_u32 s0, s8, s09172; GCN1-NEXT: s_addc_u32 s1, s9, s19173; GCN1-NEXT: s_cmp_eq_u32 s1, s29174; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 09175; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]9176; GCN1-NEXT: s_cbranch_vccz .LBB63_29177; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global9178; GCN1-NEXT: v_mov_b32_e32 v0, s09179; GCN1-NEXT: v_mov_b32_e32 v2, s129180; GCN1-NEXT: v_mov_b32_e32 v1, s19181; GCN1-NEXT: v_mov_b32_e32 v3, s139182; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc9183; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9184; GCN1-NEXT: buffer_wbinvl1_vol9185; GCN1-NEXT: s_cbranch_execz .LBB63_39186; GCN1-NEXT: s_branch .LBB63_49187; GCN1-NEXT: .LBB63_2:9188; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr19189; GCN1-NEXT: .LBB63_3: ; %atomicrmw.private9190; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 09191; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec9192; GCN1-NEXT: s_cselect_b32 s0, s0, -19193; GCN1-NEXT: v_mov_b32_e32 v2, s09194; GCN1-NEXT: s_add_i32 s0, s0, 49195; GCN1-NEXT: v_mov_b32_e32 v3, s09196; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen9197; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen9198; GCN1-NEXT: s_waitcnt vmcnt(1)9199; GCN1-NEXT: v_or_b32_e32 v4, s12, v09200; GCN1-NEXT: s_waitcnt vmcnt(0)9201; GCN1-NEXT: v_or_b32_e32 v5, s13, v19202; GCN1-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen9203; GCN1-NEXT: buffer_store_dword v5, v3, s[16:19], 0 offen9204; GCN1-NEXT: .LBB63_4: ; %atomicrmw.end9205; GCN1-NEXT: v_mov_b32_e32 v2, s109206; GCN1-NEXT: v_mov_b32_e32 v3, s119207; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]9208; GCN1-NEXT: s_endpgm9209;9210; GCN2-LABEL: atomic_or_i64_ret_addr64:9211; GCN2: ; %bb.0: ; %entry9212; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD09213; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD19214; GCN2-NEXT: s_mov_b32 s90, -19215; GCN2-NEXT: s_mov_b32 s91, 0xe800009216; GCN2-NEXT: s_add_u32 s88, s88, s119217; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x249218; GCN2-NEXT: s_load_dword s2, s[4:5], 0x1049219; GCN2-NEXT: s_addc_u32 s89, s89, 09220; GCN2-NEXT: s_waitcnt lgkmcnt(0)9221; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 39222; GCN2-NEXT: s_add_u32 s0, s8, s09223; GCN2-NEXT: s_addc_u32 s1, s9, s19224; GCN2-NEXT: s_cmp_eq_u32 s1, s29225; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 09226; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]9227; GCN2-NEXT: s_cbranch_vccz .LBB63_29228; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global9229; GCN2-NEXT: v_mov_b32_e32 v0, s09230; GCN2-NEXT: v_mov_b32_e32 v2, s129231; GCN2-NEXT: v_mov_b32_e32 v1, s19232; GCN2-NEXT: v_mov_b32_e32 v3, s139233; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc9234; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9235; GCN2-NEXT: buffer_wbinvl1_vol9236; GCN2-NEXT: s_cbranch_execz .LBB63_39237; GCN2-NEXT: s_branch .LBB63_49238; GCN2-NEXT: .LBB63_2:9239; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr19240; GCN2-NEXT: .LBB63_3: ; %atomicrmw.private9241; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 09242; GCN2-NEXT: s_cselect_b32 s0, s0, -19243; GCN2-NEXT: v_mov_b32_e32 v2, s09244; GCN2-NEXT: s_add_i32 s0, s0, 49245; GCN2-NEXT: v_mov_b32_e32 v3, s09246; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen9247; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen9248; GCN2-NEXT: s_waitcnt vmcnt(1)9249; GCN2-NEXT: v_or_b32_e32 v4, s12, v09250; GCN2-NEXT: s_waitcnt vmcnt(0)9251; GCN2-NEXT: v_or_b32_e32 v5, s13, v19252; GCN2-NEXT: buffer_store_dword v4, v2, s[88:91], 0 offen9253; GCN2-NEXT: buffer_store_dword v5, v3, s[88:91], 0 offen9254; GCN2-NEXT: .LBB63_4: ; %atomicrmw.end9255; GCN2-NEXT: v_mov_b32_e32 v2, s109256; GCN2-NEXT: v_mov_b32_e32 v3, s119257; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]9258; GCN2-NEXT: s_endpgm9259;9260; GFX12-LABEL: atomic_or_i64_ret_addr64:9261; GFX12: ; %bb.0: ; %entry9262; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x249263; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base9264; GFX12-NEXT: s_wait_kmcnt 0x09265; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 39266; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)9267; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]9268; GFX12-NEXT: s_cmp_eq_u32 s1, s99269; GFX12-NEXT: s_cselect_b32 s6, -1, 09270; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)9271; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s69272; GFX12-NEXT: s_cbranch_vccz .LBB63_29273; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global9274; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s19275; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s59276; GFX12-NEXT: flat_atomic_or_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV9277; GFX12-NEXT: s_wait_loadcnt_dscnt 0x09278; GFX12-NEXT: global_inv scope:SCOPE_DEV9279; GFX12-NEXT: s_cbranch_execz .LBB63_39280; GFX12-NEXT: s_branch .LBB63_49281; GFX12-NEXT: .LBB63_2:9282; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr19283; GFX12-NEXT: .LBB63_3: ; %atomicrmw.private9284; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 09285; GFX12-NEXT: s_cselect_b32 s0, s0, -19286; GFX12-NEXT: scratch_load_b64 v[0:1], off, s09287; GFX12-NEXT: s_wait_loadcnt 0x09288; GFX12-NEXT: v_or_b32_e32 v3, s5, v19289; GFX12-NEXT: v_or_b32_e32 v2, s4, v09290; GFX12-NEXT: scratch_store_b64 off, v[2:3], s09291; GFX12-NEXT: .LBB63_4: ; %atomicrmw.end9292; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s39293; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]9294; GFX12-NEXT: s_endpgm9295entry:9296 %ptr = getelementptr inbounds i64, ptr %out, i64 %index9297 %tmp0 = atomicrmw volatile or ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !09298 store i64 %tmp0, ptr %out29299 ret void9300}9301 9302define amdgpu_kernel void @atomic_xchg_i64_offset(ptr %out, i64 %in) {9303; GCN1-LABEL: atomic_xchg_i64_offset:9304; GCN1: ; %bb.0: ; %entry9305; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD09306; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD19307; GCN1-NEXT: s_mov_b32 s14, -19308; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x99309; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d9310; GCN1-NEXT: s_mov_b32 s15, 0xe8f0009311; GCN1-NEXT: s_add_u32 s12, s12, s119312; GCN1-NEXT: s_addc_u32 s13, s13, 09313; GCN1-NEXT: s_waitcnt lgkmcnt(0)9314; GCN1-NEXT: s_add_u32 s0, s0, 329315; GCN1-NEXT: s_addc_u32 s1, s1, 09316; GCN1-NEXT: s_cmp_eq_u32 s1, s49317; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 09318; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]9319; GCN1-NEXT: s_mov_b64 s[4:5], -19320; GCN1-NEXT: s_cbranch_vccnz .LBB64_39321; GCN1-NEXT: ; %bb.1: ; %Flow9322; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]9323; GCN1-NEXT: s_cbranch_vccz .LBB64_49324; GCN1-NEXT: .LBB64_2: ; %atomicrmw.phi9325; GCN1-NEXT: s_endpgm9326; GCN1-NEXT: .LBB64_3: ; %atomicrmw.global9327; GCN1-NEXT: v_mov_b32_e32 v0, s09328; GCN1-NEXT: v_mov_b32_e32 v2, s29329; GCN1-NEXT: v_mov_b32_e32 v1, s19330; GCN1-NEXT: v_mov_b32_e32 v3, s39331; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]9332; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9333; GCN1-NEXT: buffer_wbinvl1_vol9334; GCN1-NEXT: s_cbranch_execnz .LBB64_29335; GCN1-NEXT: .LBB64_4: ; %atomicrmw.private9336; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 09337; GCN1-NEXT: v_mov_b32_e32 v0, s29338; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec9339; GCN1-NEXT: s_cselect_b32 s0, s0, -19340; GCN1-NEXT: v_mov_b32_e32 v1, s09341; GCN1-NEXT: s_add_i32 s0, s0, 49342; GCN1-NEXT: buffer_store_dword v0, v1, s[12:15], 0 offen9343; GCN1-NEXT: v_mov_b32_e32 v0, s39344; GCN1-NEXT: v_mov_b32_e32 v1, s09345; GCN1-NEXT: buffer_store_dword v0, v1, s[12:15], 0 offen9346; GCN1-NEXT: s_endpgm9347;9348; GCN2-LABEL: atomic_xchg_i64_offset:9349; GCN2: ; %bb.0: ; %entry9350; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD09351; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD19352; GCN2-NEXT: s_mov_b32 s90, -19353; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249354; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf49355; GCN2-NEXT: s_mov_b32 s91, 0xe800009356; GCN2-NEXT: s_add_u32 s88, s88, s119357; GCN2-NEXT: s_addc_u32 s89, s89, 09358; GCN2-NEXT: s_waitcnt lgkmcnt(0)9359; GCN2-NEXT: s_add_u32 s0, s0, 329360; GCN2-NEXT: s_addc_u32 s1, s1, 09361; GCN2-NEXT: s_cmp_eq_u32 s1, s49362; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 09363; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]9364; GCN2-NEXT: s_mov_b64 s[4:5], -19365; GCN2-NEXT: s_cbranch_vccnz .LBB64_39366; GCN2-NEXT: ; %bb.1: ; %Flow9367; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]9368; GCN2-NEXT: s_cbranch_vccz .LBB64_49369; GCN2-NEXT: .LBB64_2: ; %atomicrmw.phi9370; GCN2-NEXT: s_endpgm9371; GCN2-NEXT: .LBB64_3: ; %atomicrmw.global9372; GCN2-NEXT: v_mov_b32_e32 v0, s09373; GCN2-NEXT: v_mov_b32_e32 v2, s29374; GCN2-NEXT: v_mov_b32_e32 v1, s19375; GCN2-NEXT: v_mov_b32_e32 v3, s39376; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]9377; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9378; GCN2-NEXT: buffer_wbinvl1_vol9379; GCN2-NEXT: s_cbranch_execnz .LBB64_29380; GCN2-NEXT: .LBB64_4: ; %atomicrmw.private9381; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 09382; GCN2-NEXT: s_cselect_b32 s0, s0, -19383; GCN2-NEXT: v_mov_b32_e32 v0, s29384; GCN2-NEXT: v_mov_b32_e32 v1, s09385; GCN2-NEXT: s_add_i32 s0, s0, 49386; GCN2-NEXT: buffer_store_dword v0, v1, s[88:91], 0 offen9387; GCN2-NEXT: v_mov_b32_e32 v0, s39388; GCN2-NEXT: v_mov_b32_e32 v1, s09389; GCN2-NEXT: buffer_store_dword v0, v1, s[88:91], 0 offen9390; GCN2-NEXT: s_endpgm9391;9392; GFX12-LABEL: atomic_xchg_i64_offset:9393; GFX12: ; %bb.0: ; %entry9394; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x249395; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base9396; GFX12-NEXT: s_wait_kmcnt 0x09397; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 329398; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)9399; GFX12-NEXT: s_cmp_eq_u32 s1, s79400; GFX12-NEXT: s_cselect_b32 s4, -1, 09401; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s49402; GFX12-NEXT: s_mov_b32 s4, -19403; GFX12-NEXT: s_cbranch_vccnz .LBB64_39404; GFX12-NEXT: ; %bb.1: ; %Flow9405; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s49406; GFX12-NEXT: s_cbranch_vccz .LBB64_49407; GFX12-NEXT: .LBB64_2: ; %atomicrmw.phi9408; GFX12-NEXT: s_endpgm9409; GFX12-NEXT: .LBB64_3: ; %atomicrmw.global9410; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s19411; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s39412; GFX12-NEXT: flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV9413; GFX12-NEXT: s_wait_storecnt_dscnt 0x09414; GFX12-NEXT: global_inv scope:SCOPE_DEV9415; GFX12-NEXT: s_cbranch_execnz .LBB64_29416; GFX12-NEXT: .LBB64_4: ; %atomicrmw.private9417; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s39418; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 09419; GFX12-NEXT: s_cselect_b32 s0, s0, -19420; GFX12-NEXT: scratch_store_b64 off, v[0:1], s09421; GFX12-NEXT: s_endpgm9422entry:9423 %gep = getelementptr inbounds i64, ptr %out, i64 49424 %tmp0 = atomicrmw volatile xchg ptr %gep, i64 %in syncscope("agent") seq_cst9425 ret void9426}9427 9428define amdgpu_kernel void @atomic_xchg_f64_offset(ptr %out, double %in) {9429; GCN1-LABEL: atomic_xchg_f64_offset:9430; GCN1: ; %bb.0: ; %entry9431; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD09432; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD19433; GCN1-NEXT: s_mov_b32 s14, -19434; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x99435; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d9436; GCN1-NEXT: s_mov_b32 s15, 0xe8f0009437; GCN1-NEXT: s_add_u32 s12, s12, s119438; GCN1-NEXT: s_addc_u32 s13, s13, 09439; GCN1-NEXT: s_waitcnt lgkmcnt(0)9440; GCN1-NEXT: s_add_u32 s0, s0, 329441; GCN1-NEXT: s_addc_u32 s1, s1, 09442; GCN1-NEXT: s_cmp_eq_u32 s1, s49443; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 09444; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]9445; GCN1-NEXT: s_mov_b64 s[4:5], -19446; GCN1-NEXT: s_cbranch_vccnz .LBB65_39447; GCN1-NEXT: ; %bb.1: ; %Flow9448; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]9449; GCN1-NEXT: s_cbranch_vccz .LBB65_49450; GCN1-NEXT: .LBB65_2: ; %atomicrmw.phi9451; GCN1-NEXT: s_endpgm9452; GCN1-NEXT: .LBB65_3: ; %atomicrmw.global9453; GCN1-NEXT: v_mov_b32_e32 v0, s09454; GCN1-NEXT: v_mov_b32_e32 v2, s29455; GCN1-NEXT: v_mov_b32_e32 v1, s19456; GCN1-NEXT: v_mov_b32_e32 v3, s39457; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]9458; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9459; GCN1-NEXT: buffer_wbinvl1_vol9460; GCN1-NEXT: s_cbranch_execnz .LBB65_29461; GCN1-NEXT: .LBB65_4: ; %atomicrmw.private9462; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 09463; GCN1-NEXT: v_mov_b32_e32 v0, s29464; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec9465; GCN1-NEXT: s_cselect_b32 s0, s0, -19466; GCN1-NEXT: v_mov_b32_e32 v1, s09467; GCN1-NEXT: s_add_i32 s0, s0, 49468; GCN1-NEXT: buffer_store_dword v0, v1, s[12:15], 0 offen9469; GCN1-NEXT: v_mov_b32_e32 v0, s39470; GCN1-NEXT: v_mov_b32_e32 v1, s09471; GCN1-NEXT: buffer_store_dword v0, v1, s[12:15], 0 offen9472; GCN1-NEXT: s_endpgm9473;9474; GCN2-LABEL: atomic_xchg_f64_offset:9475; GCN2: ; %bb.0: ; %entry9476; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD09477; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD19478; GCN2-NEXT: s_mov_b32 s90, -19479; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249480; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf49481; GCN2-NEXT: s_mov_b32 s91, 0xe800009482; GCN2-NEXT: s_add_u32 s88, s88, s119483; GCN2-NEXT: s_addc_u32 s89, s89, 09484; GCN2-NEXT: s_waitcnt lgkmcnt(0)9485; GCN2-NEXT: s_add_u32 s0, s0, 329486; GCN2-NEXT: s_addc_u32 s1, s1, 09487; GCN2-NEXT: s_cmp_eq_u32 s1, s49488; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 09489; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]9490; GCN2-NEXT: s_mov_b64 s[4:5], -19491; GCN2-NEXT: s_cbranch_vccnz .LBB65_39492; GCN2-NEXT: ; %bb.1: ; %Flow9493; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]9494; GCN2-NEXT: s_cbranch_vccz .LBB65_49495; GCN2-NEXT: .LBB65_2: ; %atomicrmw.phi9496; GCN2-NEXT: s_endpgm9497; GCN2-NEXT: .LBB65_3: ; %atomicrmw.global9498; GCN2-NEXT: v_mov_b32_e32 v0, s09499; GCN2-NEXT: v_mov_b32_e32 v2, s29500; GCN2-NEXT: v_mov_b32_e32 v1, s19501; GCN2-NEXT: v_mov_b32_e32 v3, s39502; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]9503; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9504; GCN2-NEXT: buffer_wbinvl1_vol9505; GCN2-NEXT: s_cbranch_execnz .LBB65_29506; GCN2-NEXT: .LBB65_4: ; %atomicrmw.private9507; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 09508; GCN2-NEXT: s_cselect_b32 s0, s0, -19509; GCN2-NEXT: v_mov_b32_e32 v0, s29510; GCN2-NEXT: v_mov_b32_e32 v1, s09511; GCN2-NEXT: s_add_i32 s0, s0, 49512; GCN2-NEXT: buffer_store_dword v0, v1, s[88:91], 0 offen9513; GCN2-NEXT: v_mov_b32_e32 v0, s39514; GCN2-NEXT: v_mov_b32_e32 v1, s09515; GCN2-NEXT: buffer_store_dword v0, v1, s[88:91], 0 offen9516; GCN2-NEXT: s_endpgm9517;9518; GFX12-LABEL: atomic_xchg_f64_offset:9519; GFX12: ; %bb.0: ; %entry9520; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x249521; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base9522; GFX12-NEXT: s_wait_kmcnt 0x09523; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 329524; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)9525; GFX12-NEXT: s_cmp_eq_u32 s1, s79526; GFX12-NEXT: s_cselect_b32 s4, -1, 09527; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s49528; GFX12-NEXT: s_mov_b32 s4, -19529; GFX12-NEXT: s_cbranch_vccnz .LBB65_39530; GFX12-NEXT: ; %bb.1: ; %Flow9531; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s49532; GFX12-NEXT: s_cbranch_vccz .LBB65_49533; GFX12-NEXT: .LBB65_2: ; %atomicrmw.phi9534; GFX12-NEXT: s_endpgm9535; GFX12-NEXT: .LBB65_3: ; %atomicrmw.global9536; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s19537; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s39538; GFX12-NEXT: flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV9539; GFX12-NEXT: s_wait_storecnt_dscnt 0x09540; GFX12-NEXT: global_inv scope:SCOPE_DEV9541; GFX12-NEXT: s_cbranch_execnz .LBB65_29542; GFX12-NEXT: .LBB65_4: ; %atomicrmw.private9543; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s39544; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 09545; GFX12-NEXT: s_cselect_b32 s0, s0, -19546; GFX12-NEXT: scratch_store_b64 off, v[0:1], s09547; GFX12-NEXT: s_endpgm9548entry:9549 %gep = getelementptr inbounds double, ptr %out, i64 49550 %tmp0 = atomicrmw volatile xchg ptr %gep, double %in syncscope("agent") seq_cst9551 ret void9552}9553 9554define amdgpu_kernel void @atomic_xchg_pointer_offset(ptr %out, ptr %in) {9555; GCN1-LABEL: atomic_xchg_pointer_offset:9556; GCN1: ; %bb.0: ; %entry9557; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD09558; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD19559; GCN1-NEXT: s_mov_b32 s14, -19560; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x99561; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d9562; GCN1-NEXT: s_mov_b32 s15, 0xe8f0009563; GCN1-NEXT: s_add_u32 s12, s12, s119564; GCN1-NEXT: s_addc_u32 s13, s13, 09565; GCN1-NEXT: s_waitcnt lgkmcnt(0)9566; GCN1-NEXT: s_add_u32 s0, s0, 329567; GCN1-NEXT: s_addc_u32 s1, s1, 09568; GCN1-NEXT: s_cmp_eq_u32 s1, s49569; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 09570; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]9571; GCN1-NEXT: s_mov_b64 s[4:5], -19572; GCN1-NEXT: s_cbranch_vccnz .LBB66_39573; GCN1-NEXT: ; %bb.1: ; %Flow9574; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]9575; GCN1-NEXT: s_cbranch_vccz .LBB66_49576; GCN1-NEXT: .LBB66_2: ; %atomicrmw.phi9577; GCN1-NEXT: s_endpgm9578; GCN1-NEXT: .LBB66_3: ; %atomicrmw.global9579; GCN1-NEXT: v_mov_b32_e32 v0, s09580; GCN1-NEXT: v_mov_b32_e32 v2, s29581; GCN1-NEXT: v_mov_b32_e32 v1, s19582; GCN1-NEXT: v_mov_b32_e32 v3, s39583; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]9584; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9585; GCN1-NEXT: buffer_wbinvl1_vol9586; GCN1-NEXT: s_cbranch_execnz .LBB66_29587; GCN1-NEXT: .LBB66_4: ; %atomicrmw.private9588; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 09589; GCN1-NEXT: v_mov_b32_e32 v0, s29590; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec9591; GCN1-NEXT: s_cselect_b32 s0, s0, -19592; GCN1-NEXT: v_mov_b32_e32 v1, s09593; GCN1-NEXT: s_add_i32 s0, s0, 49594; GCN1-NEXT: buffer_store_dword v0, v1, s[12:15], 0 offen9595; GCN1-NEXT: v_mov_b32_e32 v0, s39596; GCN1-NEXT: v_mov_b32_e32 v1, s09597; GCN1-NEXT: buffer_store_dword v0, v1, s[12:15], 0 offen9598; GCN1-NEXT: s_endpgm9599;9600; GCN2-LABEL: atomic_xchg_pointer_offset:9601; GCN2: ; %bb.0: ; %entry9602; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD09603; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD19604; GCN2-NEXT: s_mov_b32 s90, -19605; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249606; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf49607; GCN2-NEXT: s_mov_b32 s91, 0xe800009608; GCN2-NEXT: s_add_u32 s88, s88, s119609; GCN2-NEXT: s_addc_u32 s89, s89, 09610; GCN2-NEXT: s_waitcnt lgkmcnt(0)9611; GCN2-NEXT: s_add_u32 s0, s0, 329612; GCN2-NEXT: s_addc_u32 s1, s1, 09613; GCN2-NEXT: s_cmp_eq_u32 s1, s49614; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 09615; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]9616; GCN2-NEXT: s_mov_b64 s[4:5], -19617; GCN2-NEXT: s_cbranch_vccnz .LBB66_39618; GCN2-NEXT: ; %bb.1: ; %Flow9619; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]9620; GCN2-NEXT: s_cbranch_vccz .LBB66_49621; GCN2-NEXT: .LBB66_2: ; %atomicrmw.phi9622; GCN2-NEXT: s_endpgm9623; GCN2-NEXT: .LBB66_3: ; %atomicrmw.global9624; GCN2-NEXT: v_mov_b32_e32 v0, s09625; GCN2-NEXT: v_mov_b32_e32 v2, s29626; GCN2-NEXT: v_mov_b32_e32 v1, s19627; GCN2-NEXT: v_mov_b32_e32 v3, s39628; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]9629; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9630; GCN2-NEXT: buffer_wbinvl1_vol9631; GCN2-NEXT: s_cbranch_execnz .LBB66_29632; GCN2-NEXT: .LBB66_4: ; %atomicrmw.private9633; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 09634; GCN2-NEXT: s_cselect_b32 s0, s0, -19635; GCN2-NEXT: v_mov_b32_e32 v0, s29636; GCN2-NEXT: v_mov_b32_e32 v1, s09637; GCN2-NEXT: s_add_i32 s0, s0, 49638; GCN2-NEXT: buffer_store_dword v0, v1, s[88:91], 0 offen9639; GCN2-NEXT: v_mov_b32_e32 v0, s39640; GCN2-NEXT: v_mov_b32_e32 v1, s09641; GCN2-NEXT: buffer_store_dword v0, v1, s[88:91], 0 offen9642; GCN2-NEXT: s_endpgm9643;9644; GFX12-LABEL: atomic_xchg_pointer_offset:9645; GFX12: ; %bb.0: ; %entry9646; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x249647; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base9648; GFX12-NEXT: s_wait_kmcnt 0x09649; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 329650; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)9651; GFX12-NEXT: s_cmp_eq_u32 s1, s79652; GFX12-NEXT: s_cselect_b32 s4, -1, 09653; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s49654; GFX12-NEXT: s_mov_b32 s4, -19655; GFX12-NEXT: s_cbranch_vccnz .LBB66_39656; GFX12-NEXT: ; %bb.1: ; %Flow9657; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s49658; GFX12-NEXT: s_cbranch_vccz .LBB66_49659; GFX12-NEXT: .LBB66_2: ; %atomicrmw.phi9660; GFX12-NEXT: s_endpgm9661; GFX12-NEXT: .LBB66_3: ; %atomicrmw.global9662; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s19663; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s39664; GFX12-NEXT: flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV9665; GFX12-NEXT: s_wait_storecnt_dscnt 0x09666; GFX12-NEXT: global_inv scope:SCOPE_DEV9667; GFX12-NEXT: s_cbranch_execnz .LBB66_29668; GFX12-NEXT: .LBB66_4: ; %atomicrmw.private9669; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s39670; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 09671; GFX12-NEXT: s_cselect_b32 s0, s0, -19672; GFX12-NEXT: scratch_store_b64 off, v[0:1], s09673; GFX12-NEXT: s_endpgm9674entry:9675 %gep = getelementptr inbounds ptr, ptr %out, i32 49676 %val = atomicrmw volatile xchg ptr %gep, ptr %in syncscope("agent") seq_cst9677 ret void9678}9679 9680define amdgpu_kernel void @atomic_xchg_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {9681; GCN1-LABEL: atomic_xchg_i64_ret_offset:9682; GCN1: ; %bb.0: ; %entry9683; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD09684; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD19685; GCN1-NEXT: s_mov_b32 s14, -19686; GCN1-NEXT: s_mov_b32 s15, 0xe8f0009687; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x99688; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f9689; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd9690; GCN1-NEXT: s_add_u32 s12, s12, s119691; GCN1-NEXT: s_addc_u32 s13, s13, 09692; GCN1-NEXT: s_waitcnt lgkmcnt(0)9693; GCN1-NEXT: s_add_u32 s0, s0, 329694; GCN1-NEXT: s_addc_u32 s1, s1, 09695; GCN1-NEXT: s_cmp_eq_u32 s1, s69696; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 09697; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]9698; GCN1-NEXT: s_cbranch_vccz .LBB67_29699; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global9700; GCN1-NEXT: v_mov_b32_e32 v0, s09701; GCN1-NEXT: v_mov_b32_e32 v2, s49702; GCN1-NEXT: v_mov_b32_e32 v1, s19703; GCN1-NEXT: v_mov_b32_e32 v3, s59704; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc9705; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9706; GCN1-NEXT: buffer_wbinvl1_vol9707; GCN1-NEXT: s_cbranch_execz .LBB67_39708; GCN1-NEXT: s_branch .LBB67_49709; GCN1-NEXT: .LBB67_2:9710; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr19711; GCN1-NEXT: .LBB67_3: ; %atomicrmw.private9712; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 09713; GCN1-NEXT: v_mov_b32_e32 v4, s49714; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec9715; GCN1-NEXT: s_cselect_b32 s0, s0, -19716; GCN1-NEXT: v_mov_b32_e32 v2, s09717; GCN1-NEXT: s_add_i32 s0, s0, 49718; GCN1-NEXT: v_mov_b32_e32 v3, s09719; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen9720; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen9721; GCN1-NEXT: buffer_store_dword v4, v2, s[12:15], 0 offen9722; GCN1-NEXT: v_mov_b32_e32 v2, s59723; GCN1-NEXT: buffer_store_dword v2, v3, s[12:15], 0 offen9724; GCN1-NEXT: .LBB67_4: ; %atomicrmw.end9725; GCN1-NEXT: v_mov_b32_e32 v2, s29726; GCN1-NEXT: v_mov_b32_e32 v3, s39727; GCN1-NEXT: s_waitcnt vmcnt(2)9728; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]9729; GCN1-NEXT: s_endpgm9730;9731; GCN2-LABEL: atomic_xchg_i64_ret_offset:9732; GCN2: ; %bb.0: ; %entry9733; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD09734; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD19735; GCN2-NEXT: s_mov_b32 s90, -19736; GCN2-NEXT: s_mov_b32 s91, 0xe800009737; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249738; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc9739; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x349740; GCN2-NEXT: s_add_u32 s88, s88, s119741; GCN2-NEXT: s_addc_u32 s89, s89, 09742; GCN2-NEXT: s_waitcnt lgkmcnt(0)9743; GCN2-NEXT: s_add_u32 s0, s0, 329744; GCN2-NEXT: s_addc_u32 s1, s1, 09745; GCN2-NEXT: s_cmp_eq_u32 s1, s69746; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 09747; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]9748; GCN2-NEXT: s_cbranch_vccz .LBB67_29749; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global9750; GCN2-NEXT: v_mov_b32_e32 v0, s09751; GCN2-NEXT: v_mov_b32_e32 v2, s49752; GCN2-NEXT: v_mov_b32_e32 v1, s19753; GCN2-NEXT: v_mov_b32_e32 v3, s59754; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc9755; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9756; GCN2-NEXT: buffer_wbinvl1_vol9757; GCN2-NEXT: s_cbranch_execz .LBB67_39758; GCN2-NEXT: s_branch .LBB67_49759; GCN2-NEXT: .LBB67_2:9760; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr19761; GCN2-NEXT: .LBB67_3: ; %atomicrmw.private9762; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 09763; GCN2-NEXT: s_cselect_b32 s0, s0, -19764; GCN2-NEXT: v_mov_b32_e32 v2, s09765; GCN2-NEXT: s_add_i32 s0, s0, 49766; GCN2-NEXT: v_mov_b32_e32 v3, s09767; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen9768; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen9769; GCN2-NEXT: v_mov_b32_e32 v4, s49770; GCN2-NEXT: buffer_store_dword v4, v2, s[88:91], 0 offen9771; GCN2-NEXT: v_mov_b32_e32 v2, s59772; GCN2-NEXT: buffer_store_dword v2, v3, s[88:91], 0 offen9773; GCN2-NEXT: .LBB67_4: ; %atomicrmw.end9774; GCN2-NEXT: v_mov_b32_e32 v2, s29775; GCN2-NEXT: v_mov_b32_e32 v3, s39776; GCN2-NEXT: s_waitcnt vmcnt(2)9777; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]9778; GCN2-NEXT: s_endpgm9779;9780; GFX12-LABEL: atomic_xchg_i64_ret_offset:9781; GFX12: ; %bb.0: ; %entry9782; GFX12-NEXT: s_clause 0x19783; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x249784; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x349785; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base9786; GFX12-NEXT: s_wait_kmcnt 0x09787; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 329788; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)9789; GFX12-NEXT: s_cmp_eq_u32 s1, s79790; GFX12-NEXT: s_cselect_b32 s6, -1, 09791; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s69792; GFX12-NEXT: s_cbranch_vccz .LBB67_29793; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global9794; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s19795; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s59796; GFX12-NEXT: flat_atomic_swap_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV9797; GFX12-NEXT: s_wait_loadcnt_dscnt 0x09798; GFX12-NEXT: global_inv scope:SCOPE_DEV9799; GFX12-NEXT: s_cbranch_execz .LBB67_39800; GFX12-NEXT: s_branch .LBB67_49801; GFX12-NEXT: .LBB67_2:9802; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr19803; GFX12-NEXT: .LBB67_3: ; %atomicrmw.private9804; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 09805; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s59806; GFX12-NEXT: s_cselect_b32 s0, s0, -19807; GFX12-NEXT: scratch_load_b64 v[0:1], off, s09808; GFX12-NEXT: scratch_store_b64 off, v[2:3], s09809; GFX12-NEXT: .LBB67_4: ; %atomicrmw.end9810; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s39811; GFX12-NEXT: s_wait_loadcnt 0x09812; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]9813; GFX12-NEXT: s_endpgm9814entry:9815 %gep = getelementptr inbounds i64, ptr %out, i64 49816 %tmp0 = atomicrmw volatile xchg ptr %gep, i64 %in syncscope("agent") seq_cst9817 store i64 %tmp0, ptr %out29818 ret void9819}9820 9821define amdgpu_kernel void @atomic_xchg_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {9822; GCN1-LABEL: atomic_xchg_i64_addr64_offset:9823; GCN1: ; %bb.0: ; %entry9824; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD09825; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD19826; GCN1-NEXT: s_mov_b32 s14, -19827; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd9828; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x99829; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f9830; GCN1-NEXT: s_mov_b32 s15, 0xe8f0009831; GCN1-NEXT: s_add_u32 s12, s12, s119832; GCN1-NEXT: s_addc_u32 s13, s13, 09833; GCN1-NEXT: s_waitcnt lgkmcnt(0)9834; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 39835; GCN1-NEXT: s_add_u32 s0, s0, s49836; GCN1-NEXT: s_addc_u32 s1, s1, s59837; GCN1-NEXT: s_add_u32 s0, s0, 329838; GCN1-NEXT: s_addc_u32 s1, s1, 09839; GCN1-NEXT: s_cmp_eq_u32 s1, s89840; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 09841; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]9842; GCN1-NEXT: s_mov_b64 s[4:5], -19843; GCN1-NEXT: s_cbranch_vccnz .LBB68_39844; GCN1-NEXT: ; %bb.1: ; %Flow9845; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]9846; GCN1-NEXT: s_cbranch_vccz .LBB68_49847; GCN1-NEXT: .LBB68_2: ; %atomicrmw.phi9848; GCN1-NEXT: s_endpgm9849; GCN1-NEXT: .LBB68_3: ; %atomicrmw.global9850; GCN1-NEXT: v_mov_b32_e32 v0, s09851; GCN1-NEXT: v_mov_b32_e32 v2, s29852; GCN1-NEXT: v_mov_b32_e32 v1, s19853; GCN1-NEXT: v_mov_b32_e32 v3, s39854; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]9855; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9856; GCN1-NEXT: buffer_wbinvl1_vol9857; GCN1-NEXT: s_cbranch_execnz .LBB68_29858; GCN1-NEXT: .LBB68_4: ; %atomicrmw.private9859; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 09860; GCN1-NEXT: v_mov_b32_e32 v0, s29861; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec9862; GCN1-NEXT: s_cselect_b32 s0, s0, -19863; GCN1-NEXT: v_mov_b32_e32 v1, s09864; GCN1-NEXT: s_add_i32 s0, s0, 49865; GCN1-NEXT: buffer_store_dword v0, v1, s[12:15], 0 offen9866; GCN1-NEXT: v_mov_b32_e32 v0, s39867; GCN1-NEXT: v_mov_b32_e32 v1, s09868; GCN1-NEXT: buffer_store_dword v0, v1, s[12:15], 0 offen9869; GCN1-NEXT: s_endpgm9870;9871; GCN2-LABEL: atomic_xchg_i64_addr64_offset:9872; GCN2: ; %bb.0: ; %entry9873; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD09874; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD19875; GCN2-NEXT: s_mov_b32 s90, -19876; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x349877; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249878; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc9879; GCN2-NEXT: s_mov_b32 s91, 0xe800009880; GCN2-NEXT: s_add_u32 s88, s88, s119881; GCN2-NEXT: s_addc_u32 s89, s89, 09882; GCN2-NEXT: s_waitcnt lgkmcnt(0)9883; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 39884; GCN2-NEXT: s_add_u32 s0, s0, s49885; GCN2-NEXT: s_addc_u32 s1, s1, s59886; GCN2-NEXT: s_add_u32 s0, s0, 329887; GCN2-NEXT: s_addc_u32 s1, s1, 09888; GCN2-NEXT: s_cmp_eq_u32 s1, s89889; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 09890; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]9891; GCN2-NEXT: s_mov_b64 s[4:5], -19892; GCN2-NEXT: s_cbranch_vccnz .LBB68_39893; GCN2-NEXT: ; %bb.1: ; %Flow9894; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]9895; GCN2-NEXT: s_cbranch_vccz .LBB68_49896; GCN2-NEXT: .LBB68_2: ; %atomicrmw.phi9897; GCN2-NEXT: s_endpgm9898; GCN2-NEXT: .LBB68_3: ; %atomicrmw.global9899; GCN2-NEXT: v_mov_b32_e32 v0, s09900; GCN2-NEXT: v_mov_b32_e32 v2, s29901; GCN2-NEXT: v_mov_b32_e32 v1, s19902; GCN2-NEXT: v_mov_b32_e32 v3, s39903; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]9904; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9905; GCN2-NEXT: buffer_wbinvl1_vol9906; GCN2-NEXT: s_cbranch_execnz .LBB68_29907; GCN2-NEXT: .LBB68_4: ; %atomicrmw.private9908; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 09909; GCN2-NEXT: s_cselect_b32 s0, s0, -19910; GCN2-NEXT: v_mov_b32_e32 v0, s29911; GCN2-NEXT: v_mov_b32_e32 v1, s09912; GCN2-NEXT: s_add_i32 s0, s0, 49913; GCN2-NEXT: buffer_store_dword v0, v1, s[88:91], 0 offen9914; GCN2-NEXT: v_mov_b32_e32 v0, s39915; GCN2-NEXT: v_mov_b32_e32 v1, s09916; GCN2-NEXT: buffer_store_dword v0, v1, s[88:91], 0 offen9917; GCN2-NEXT: s_endpgm9918;9919; GFX12-LABEL: atomic_xchg_i64_addr64_offset:9920; GFX12: ; %bb.0: ; %entry9921; GFX12-NEXT: s_clause 0x19922; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x349923; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x249924; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base9925; GFX12-NEXT: s_wait_kmcnt 0x09926; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 39927; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)9928; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]9929; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 329930; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)9931; GFX12-NEXT: s_cmp_eq_u32 s1, s79932; GFX12-NEXT: s_cselect_b32 s4, -1, 09933; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s49934; GFX12-NEXT: s_mov_b32 s4, -19935; GFX12-NEXT: s_cbranch_vccnz .LBB68_39936; GFX12-NEXT: ; %bb.1: ; %Flow9937; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s49938; GFX12-NEXT: s_cbranch_vccz .LBB68_49939; GFX12-NEXT: .LBB68_2: ; %atomicrmw.phi9940; GFX12-NEXT: s_endpgm9941; GFX12-NEXT: .LBB68_3: ; %atomicrmw.global9942; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s19943; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s39944; GFX12-NEXT: flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV9945; GFX12-NEXT: s_wait_storecnt_dscnt 0x09946; GFX12-NEXT: global_inv scope:SCOPE_DEV9947; GFX12-NEXT: s_cbranch_execnz .LBB68_29948; GFX12-NEXT: .LBB68_4: ; %atomicrmw.private9949; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s39950; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 09951; GFX12-NEXT: s_cselect_b32 s0, s0, -19952; GFX12-NEXT: scratch_store_b64 off, v[0:1], s09953; GFX12-NEXT: s_endpgm9954entry:9955 %ptr = getelementptr inbounds i64, ptr %out, i64 %index9956 %gep = getelementptr inbounds i64, ptr %ptr, i64 49957 %tmp0 = atomicrmw volatile xchg ptr %gep, i64 %in syncscope("agent") seq_cst9958 ret void9959}9960 9961define amdgpu_kernel void @atomic_xchg_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {9962; GCN1-LABEL: atomic_xchg_i64_ret_addr64_offset:9963; GCN1: ; %bb.0: ; %entry9964; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD09965; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD19966; GCN1-NEXT: s_mov_b32 s18, -19967; GCN1-NEXT: s_mov_b32 s19, 0xe8f0009968; GCN1-NEXT: s_add_u32 s16, s16, s119969; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x99970; GCN1-NEXT: s_load_dword s2, s[4:5], 0x419971; GCN1-NEXT: s_addc_u32 s17, s17, 09972; GCN1-NEXT: s_waitcnt lgkmcnt(0)9973; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 39974; GCN1-NEXT: s_add_u32 s0, s8, s09975; GCN1-NEXT: s_addc_u32 s1, s9, s19976; GCN1-NEXT: s_add_u32 s0, s0, 329977; GCN1-NEXT: s_addc_u32 s1, s1, 09978; GCN1-NEXT: s_cmp_eq_u32 s1, s29979; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 09980; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]9981; GCN1-NEXT: s_cbranch_vccz .LBB69_29982; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global9983; GCN1-NEXT: v_mov_b32_e32 v0, s09984; GCN1-NEXT: v_mov_b32_e32 v2, s129985; GCN1-NEXT: v_mov_b32_e32 v1, s19986; GCN1-NEXT: v_mov_b32_e32 v3, s139987; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc9988; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9989; GCN1-NEXT: buffer_wbinvl1_vol9990; GCN1-NEXT: s_cbranch_execz .LBB69_39991; GCN1-NEXT: s_branch .LBB69_49992; GCN1-NEXT: .LBB69_2:9993; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr19994; GCN1-NEXT: .LBB69_3: ; %atomicrmw.private9995; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 09996; GCN1-NEXT: v_mov_b32_e32 v4, s129997; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec9998; GCN1-NEXT: s_cselect_b32 s0, s0, -19999; GCN1-NEXT: v_mov_b32_e32 v2, s010000; GCN1-NEXT: s_add_i32 s0, s0, 410001; GCN1-NEXT: v_mov_b32_e32 v3, s010002; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen10003; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen10004; GCN1-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen10005; GCN1-NEXT: v_mov_b32_e32 v2, s1310006; GCN1-NEXT: buffer_store_dword v2, v3, s[16:19], 0 offen10007; GCN1-NEXT: .LBB69_4: ; %atomicrmw.end10008; GCN1-NEXT: v_mov_b32_e32 v2, s1010009; GCN1-NEXT: v_mov_b32_e32 v3, s1110010; GCN1-NEXT: s_waitcnt vmcnt(2)10011; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]10012; GCN1-NEXT: s_endpgm10013;10014; GCN2-LABEL: atomic_xchg_i64_ret_addr64_offset:10015; GCN2: ; %bb.0: ; %entry10016; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD010017; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD110018; GCN2-NEXT: s_mov_b32 s90, -110019; GCN2-NEXT: s_mov_b32 s91, 0xe8000010020; GCN2-NEXT: s_add_u32 s88, s88, s1110021; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2410022; GCN2-NEXT: s_load_dword s2, s[4:5], 0x10410023; GCN2-NEXT: s_addc_u32 s89, s89, 010024; GCN2-NEXT: s_waitcnt lgkmcnt(0)10025; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 310026; GCN2-NEXT: s_add_u32 s0, s8, s010027; GCN2-NEXT: s_addc_u32 s1, s9, s110028; GCN2-NEXT: s_add_u32 s0, s0, 3210029; GCN2-NEXT: s_addc_u32 s1, s1, 010030; GCN2-NEXT: s_cmp_eq_u32 s1, s210031; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 010032; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]10033; GCN2-NEXT: s_cbranch_vccz .LBB69_210034; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global10035; GCN2-NEXT: v_mov_b32_e32 v0, s010036; GCN2-NEXT: v_mov_b32_e32 v2, s1210037; GCN2-NEXT: v_mov_b32_e32 v1, s110038; GCN2-NEXT: v_mov_b32_e32 v3, s1310039; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc10040; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10041; GCN2-NEXT: buffer_wbinvl1_vol10042; GCN2-NEXT: s_cbranch_execz .LBB69_310043; GCN2-NEXT: s_branch .LBB69_410044; GCN2-NEXT: .LBB69_2:10045; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr110046; GCN2-NEXT: .LBB69_3: ; %atomicrmw.private10047; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 010048; GCN2-NEXT: s_cselect_b32 s0, s0, -110049; GCN2-NEXT: v_mov_b32_e32 v2, s010050; GCN2-NEXT: s_add_i32 s0, s0, 410051; GCN2-NEXT: v_mov_b32_e32 v3, s010052; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen10053; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen10054; GCN2-NEXT: v_mov_b32_e32 v4, s1210055; GCN2-NEXT: buffer_store_dword v4, v2, s[88:91], 0 offen10056; GCN2-NEXT: v_mov_b32_e32 v2, s1310057; GCN2-NEXT: buffer_store_dword v2, v3, s[88:91], 0 offen10058; GCN2-NEXT: .LBB69_4: ; %atomicrmw.end10059; GCN2-NEXT: v_mov_b32_e32 v2, s1010060; GCN2-NEXT: v_mov_b32_e32 v3, s1110061; GCN2-NEXT: s_waitcnt vmcnt(2)10062; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]10063; GCN2-NEXT: s_endpgm10064;10065; GFX12-LABEL: atomic_xchg_i64_ret_addr64_offset:10066; GFX12: ; %bb.0: ; %entry10067; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x2410068; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base10069; GFX12-NEXT: s_wait_kmcnt 0x010070; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 310071; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)10072; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]10073; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 3210074; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)10075; GFX12-NEXT: s_cmp_eq_u32 s1, s910076; GFX12-NEXT: s_cselect_b32 s6, -1, 010077; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s610078; GFX12-NEXT: s_cbranch_vccz .LBB69_210079; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global10080; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s110081; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s510082; GFX12-NEXT: flat_atomic_swap_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV10083; GFX12-NEXT: s_wait_loadcnt_dscnt 0x010084; GFX12-NEXT: global_inv scope:SCOPE_DEV10085; GFX12-NEXT: s_cbranch_execz .LBB69_310086; GFX12-NEXT: s_branch .LBB69_410087; GFX12-NEXT: .LBB69_2:10088; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr110089; GFX12-NEXT: .LBB69_3: ; %atomicrmw.private10090; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 010091; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s510092; GFX12-NEXT: s_cselect_b32 s0, s0, -110093; GFX12-NEXT: scratch_load_b64 v[0:1], off, s010094; GFX12-NEXT: scratch_store_b64 off, v[2:3], s010095; GFX12-NEXT: .LBB69_4: ; %atomicrmw.end10096; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s310097; GFX12-NEXT: s_wait_loadcnt 0x010098; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]10099; GFX12-NEXT: s_endpgm10100entry:10101 %ptr = getelementptr inbounds i64, ptr %out, i64 %index10102 %gep = getelementptr inbounds i64, ptr %ptr, i64 410103 %tmp0 = atomicrmw volatile xchg ptr %gep, i64 %in syncscope("agent") seq_cst10104 store i64 %tmp0, ptr %out210105 ret void10106}10107 10108define amdgpu_kernel void @atomic_xchg_i64(ptr %out, i64 %in) {10109; GCN1-LABEL: atomic_xchg_i64:10110; GCN1: ; %bb.0: ; %entry10111; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD010112; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD110113; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x910114; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d10115; GCN1-NEXT: s_mov_b32 s14, -110116; GCN1-NEXT: s_mov_b32 s15, 0xe8f00010117; GCN1-NEXT: s_add_u32 s12, s12, s1110118; GCN1-NEXT: s_addc_u32 s13, s13, 010119; GCN1-NEXT: s_waitcnt lgkmcnt(0)10120; GCN1-NEXT: s_cmp_eq_u32 s1, s410121; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 010122; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]10123; GCN1-NEXT: s_mov_b64 s[4:5], -110124; GCN1-NEXT: s_cbranch_vccnz .LBB70_310125; GCN1-NEXT: ; %bb.1: ; %Flow10126; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]10127; GCN1-NEXT: s_cbranch_vccz .LBB70_410128; GCN1-NEXT: .LBB70_2: ; %atomicrmw.phi10129; GCN1-NEXT: s_endpgm10130; GCN1-NEXT: .LBB70_3: ; %atomicrmw.global10131; GCN1-NEXT: v_mov_b32_e32 v0, s010132; GCN1-NEXT: v_mov_b32_e32 v2, s210133; GCN1-NEXT: v_mov_b32_e32 v1, s110134; GCN1-NEXT: v_mov_b32_e32 v3, s310135; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]10136; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10137; GCN1-NEXT: buffer_wbinvl1_vol10138; GCN1-NEXT: s_cbranch_execnz .LBB70_210139; GCN1-NEXT: .LBB70_4: ; %atomicrmw.private10140; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 010141; GCN1-NEXT: v_mov_b32_e32 v0, s210142; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec10143; GCN1-NEXT: s_cselect_b32 s0, s0, -110144; GCN1-NEXT: v_mov_b32_e32 v1, s010145; GCN1-NEXT: s_add_i32 s0, s0, 410146; GCN1-NEXT: buffer_store_dword v0, v1, s[12:15], 0 offen10147; GCN1-NEXT: v_mov_b32_e32 v0, s310148; GCN1-NEXT: v_mov_b32_e32 v1, s010149; GCN1-NEXT: buffer_store_dword v0, v1, s[12:15], 0 offen10150; GCN1-NEXT: s_endpgm10151;10152; GCN2-LABEL: atomic_xchg_i64:10153; GCN2: ; %bb.0: ; %entry10154; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD010155; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD110156; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410157; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf410158; GCN2-NEXT: s_mov_b32 s90, -110159; GCN2-NEXT: s_mov_b32 s91, 0xe8000010160; GCN2-NEXT: s_add_u32 s88, s88, s1110161; GCN2-NEXT: s_addc_u32 s89, s89, 010162; GCN2-NEXT: s_waitcnt lgkmcnt(0)10163; GCN2-NEXT: s_cmp_eq_u32 s1, s410164; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 010165; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]10166; GCN2-NEXT: s_mov_b64 s[4:5], -110167; GCN2-NEXT: s_cbranch_vccnz .LBB70_310168; GCN2-NEXT: ; %bb.1: ; %Flow10169; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]10170; GCN2-NEXT: s_cbranch_vccz .LBB70_410171; GCN2-NEXT: .LBB70_2: ; %atomicrmw.phi10172; GCN2-NEXT: s_endpgm10173; GCN2-NEXT: .LBB70_3: ; %atomicrmw.global10174; GCN2-NEXT: v_mov_b32_e32 v0, s010175; GCN2-NEXT: v_mov_b32_e32 v2, s210176; GCN2-NEXT: v_mov_b32_e32 v1, s110177; GCN2-NEXT: v_mov_b32_e32 v3, s310178; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]10179; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10180; GCN2-NEXT: buffer_wbinvl1_vol10181; GCN2-NEXT: s_cbranch_execnz .LBB70_210182; GCN2-NEXT: .LBB70_4: ; %atomicrmw.private10183; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 010184; GCN2-NEXT: s_cselect_b32 s0, s0, -110185; GCN2-NEXT: v_mov_b32_e32 v0, s210186; GCN2-NEXT: v_mov_b32_e32 v1, s010187; GCN2-NEXT: s_add_i32 s0, s0, 410188; GCN2-NEXT: buffer_store_dword v0, v1, s[88:91], 0 offen10189; GCN2-NEXT: v_mov_b32_e32 v0, s310190; GCN2-NEXT: v_mov_b32_e32 v1, s010191; GCN2-NEXT: buffer_store_dword v0, v1, s[88:91], 0 offen10192; GCN2-NEXT: s_endpgm10193;10194; GFX12-LABEL: atomic_xchg_i64:10195; GFX12: ; %bb.0: ; %entry10196; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2410197; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base10198; GFX12-NEXT: s_wait_kmcnt 0x010199; GFX12-NEXT: s_cmp_eq_u32 s1, s710200; GFX12-NEXT: s_cselect_b32 s4, -1, 010201; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)10202; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s410203; GFX12-NEXT: s_mov_b32 s4, -110204; GFX12-NEXT: s_cbranch_vccnz .LBB70_310205; GFX12-NEXT: ; %bb.1: ; %Flow10206; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s410207; GFX12-NEXT: s_cbranch_vccz .LBB70_410208; GFX12-NEXT: .LBB70_2: ; %atomicrmw.phi10209; GFX12-NEXT: s_endpgm10210; GFX12-NEXT: .LBB70_3: ; %atomicrmw.global10211; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s110212; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s310213; GFX12-NEXT: flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV10214; GFX12-NEXT: s_wait_storecnt_dscnt 0x010215; GFX12-NEXT: global_inv scope:SCOPE_DEV10216; GFX12-NEXT: s_cbranch_execnz .LBB70_210217; GFX12-NEXT: .LBB70_4: ; %atomicrmw.private10218; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s310219; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 010220; GFX12-NEXT: s_cselect_b32 s0, s0, -110221; GFX12-NEXT: scratch_store_b64 off, v[0:1], s010222; GFX12-NEXT: s_endpgm10223entry:10224 %tmp0 = atomicrmw volatile xchg ptr %out, i64 %in syncscope("agent") seq_cst10225 ret void10226}10227 10228define amdgpu_kernel void @atomic_xchg_i64_ret(ptr %out, ptr %out2, i64 %in) {10229; GCN1-LABEL: atomic_xchg_i64_ret:10230; GCN1: ; %bb.0: ; %entry10231; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD010232; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD110233; GCN1-NEXT: s_mov_b32 s14, -110234; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f10235; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x910236; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd10237; GCN1-NEXT: s_mov_b32 s15, 0xe8f00010238; GCN1-NEXT: s_add_u32 s12, s12, s1110239; GCN1-NEXT: s_addc_u32 s13, s13, 010240; GCN1-NEXT: s_waitcnt lgkmcnt(0)10241; GCN1-NEXT: s_cmp_eq_u32 s1, s610242; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 010243; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]10244; GCN1-NEXT: s_cbranch_vccz .LBB71_210245; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global10246; GCN1-NEXT: v_mov_b32_e32 v0, s010247; GCN1-NEXT: v_mov_b32_e32 v2, s410248; GCN1-NEXT: v_mov_b32_e32 v1, s110249; GCN1-NEXT: v_mov_b32_e32 v3, s510250; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc10251; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10252; GCN1-NEXT: buffer_wbinvl1_vol10253; GCN1-NEXT: s_cbranch_execz .LBB71_310254; GCN1-NEXT: s_branch .LBB71_410255; GCN1-NEXT: .LBB71_2:10256; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr110257; GCN1-NEXT: .LBB71_3: ; %atomicrmw.private10258; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 010259; GCN1-NEXT: v_mov_b32_e32 v4, s410260; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec10261; GCN1-NEXT: s_cselect_b32 s0, s0, -110262; GCN1-NEXT: v_mov_b32_e32 v2, s010263; GCN1-NEXT: s_add_i32 s0, s0, 410264; GCN1-NEXT: v_mov_b32_e32 v3, s010265; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen10266; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen10267; GCN1-NEXT: buffer_store_dword v4, v2, s[12:15], 0 offen10268; GCN1-NEXT: v_mov_b32_e32 v2, s510269; GCN1-NEXT: buffer_store_dword v2, v3, s[12:15], 0 offen10270; GCN1-NEXT: .LBB71_4: ; %atomicrmw.end10271; GCN1-NEXT: v_mov_b32_e32 v2, s210272; GCN1-NEXT: v_mov_b32_e32 v3, s310273; GCN1-NEXT: s_waitcnt vmcnt(2)10274; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]10275; GCN1-NEXT: s_endpgm10276;10277; GCN2-LABEL: atomic_xchg_i64_ret:10278; GCN2: ; %bb.0: ; %entry10279; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD010280; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD110281; GCN2-NEXT: s_mov_b32 s90, -110282; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc10283; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410284; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x3410285; GCN2-NEXT: s_mov_b32 s91, 0xe8000010286; GCN2-NEXT: s_add_u32 s88, s88, s1110287; GCN2-NEXT: s_addc_u32 s89, s89, 010288; GCN2-NEXT: s_waitcnt lgkmcnt(0)10289; GCN2-NEXT: s_cmp_eq_u32 s1, s610290; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 010291; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]10292; GCN2-NEXT: s_cbranch_vccz .LBB71_210293; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global10294; GCN2-NEXT: v_mov_b32_e32 v0, s010295; GCN2-NEXT: v_mov_b32_e32 v2, s410296; GCN2-NEXT: v_mov_b32_e32 v1, s110297; GCN2-NEXT: v_mov_b32_e32 v3, s510298; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc10299; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10300; GCN2-NEXT: buffer_wbinvl1_vol10301; GCN2-NEXT: s_cbranch_execz .LBB71_310302; GCN2-NEXT: s_branch .LBB71_410303; GCN2-NEXT: .LBB71_2:10304; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr110305; GCN2-NEXT: .LBB71_3: ; %atomicrmw.private10306; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 010307; GCN2-NEXT: s_cselect_b32 s0, s0, -110308; GCN2-NEXT: v_mov_b32_e32 v2, s010309; GCN2-NEXT: s_add_i32 s0, s0, 410310; GCN2-NEXT: v_mov_b32_e32 v3, s010311; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen10312; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen10313; GCN2-NEXT: v_mov_b32_e32 v4, s410314; GCN2-NEXT: buffer_store_dword v4, v2, s[88:91], 0 offen10315; GCN2-NEXT: v_mov_b32_e32 v2, s510316; GCN2-NEXT: buffer_store_dword v2, v3, s[88:91], 0 offen10317; GCN2-NEXT: .LBB71_4: ; %atomicrmw.end10318; GCN2-NEXT: v_mov_b32_e32 v2, s210319; GCN2-NEXT: v_mov_b32_e32 v3, s310320; GCN2-NEXT: s_waitcnt vmcnt(2)10321; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]10322; GCN2-NEXT: s_endpgm10323;10324; GFX12-LABEL: atomic_xchg_i64_ret:10325; GFX12: ; %bb.0: ; %entry10326; GFX12-NEXT: s_clause 0x110327; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2410328; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x3410329; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base10330; GFX12-NEXT: s_wait_kmcnt 0x010331; GFX12-NEXT: s_cmp_eq_u32 s1, s710332; GFX12-NEXT: s_cselect_b32 s6, -1, 010333; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)10334; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s610335; GFX12-NEXT: s_cbranch_vccz .LBB71_210336; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global10337; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s110338; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s510339; GFX12-NEXT: flat_atomic_swap_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV10340; GFX12-NEXT: s_wait_loadcnt_dscnt 0x010341; GFX12-NEXT: global_inv scope:SCOPE_DEV10342; GFX12-NEXT: s_cbranch_execz .LBB71_310343; GFX12-NEXT: s_branch .LBB71_410344; GFX12-NEXT: .LBB71_2:10345; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr110346; GFX12-NEXT: .LBB71_3: ; %atomicrmw.private10347; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 010348; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s510349; GFX12-NEXT: s_cselect_b32 s0, s0, -110350; GFX12-NEXT: scratch_load_b64 v[0:1], off, s010351; GFX12-NEXT: scratch_store_b64 off, v[2:3], s010352; GFX12-NEXT: .LBB71_4: ; %atomicrmw.end10353; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s310354; GFX12-NEXT: s_wait_loadcnt 0x010355; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]10356; GFX12-NEXT: s_endpgm10357entry:10358 %tmp0 = atomicrmw volatile xchg ptr %out, i64 %in syncscope("agent") seq_cst10359 store i64 %tmp0, ptr %out210360 ret void10361}10362 10363define amdgpu_kernel void @atomic_xchg_i64_addr64(ptr %out, i64 %in, i64 %index) {10364; GCN1-LABEL: atomic_xchg_i64_addr64:10365; GCN1: ; %bb.0: ; %entry10366; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD010367; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD110368; GCN1-NEXT: s_mov_b32 s14, -110369; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd10370; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x910371; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f10372; GCN1-NEXT: s_mov_b32 s15, 0xe8f00010373; GCN1-NEXT: s_add_u32 s12, s12, s1110374; GCN1-NEXT: s_addc_u32 s13, s13, 010375; GCN1-NEXT: s_waitcnt lgkmcnt(0)10376; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 310377; GCN1-NEXT: s_add_u32 s0, s0, s410378; GCN1-NEXT: s_addc_u32 s1, s1, s510379; GCN1-NEXT: s_cmp_eq_u32 s1, s810380; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 010381; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]10382; GCN1-NEXT: s_mov_b64 s[4:5], -110383; GCN1-NEXT: s_cbranch_vccnz .LBB72_310384; GCN1-NEXT: ; %bb.1: ; %Flow10385; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]10386; GCN1-NEXT: s_cbranch_vccz .LBB72_410387; GCN1-NEXT: .LBB72_2: ; %atomicrmw.phi10388; GCN1-NEXT: s_endpgm10389; GCN1-NEXT: .LBB72_3: ; %atomicrmw.global10390; GCN1-NEXT: v_mov_b32_e32 v0, s010391; GCN1-NEXT: v_mov_b32_e32 v2, s210392; GCN1-NEXT: v_mov_b32_e32 v1, s110393; GCN1-NEXT: v_mov_b32_e32 v3, s310394; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]10395; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10396; GCN1-NEXT: buffer_wbinvl1_vol10397; GCN1-NEXT: s_cbranch_execnz .LBB72_210398; GCN1-NEXT: .LBB72_4: ; %atomicrmw.private10399; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 010400; GCN1-NEXT: v_mov_b32_e32 v0, s210401; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec10402; GCN1-NEXT: s_cselect_b32 s0, s0, -110403; GCN1-NEXT: v_mov_b32_e32 v1, s010404; GCN1-NEXT: s_add_i32 s0, s0, 410405; GCN1-NEXT: buffer_store_dword v0, v1, s[12:15], 0 offen10406; GCN1-NEXT: v_mov_b32_e32 v0, s310407; GCN1-NEXT: v_mov_b32_e32 v1, s010408; GCN1-NEXT: buffer_store_dword v0, v1, s[12:15], 0 offen10409; GCN1-NEXT: s_endpgm10410;10411; GCN2-LABEL: atomic_xchg_i64_addr64:10412; GCN2: ; %bb.0: ; %entry10413; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD010414; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD110415; GCN2-NEXT: s_mov_b32 s90, -110416; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3410417; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410418; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc10419; GCN2-NEXT: s_mov_b32 s91, 0xe8000010420; GCN2-NEXT: s_add_u32 s88, s88, s1110421; GCN2-NEXT: s_addc_u32 s89, s89, 010422; GCN2-NEXT: s_waitcnt lgkmcnt(0)10423; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 310424; GCN2-NEXT: s_add_u32 s0, s0, s410425; GCN2-NEXT: s_addc_u32 s1, s1, s510426; GCN2-NEXT: s_cmp_eq_u32 s1, s810427; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 010428; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]10429; GCN2-NEXT: s_mov_b64 s[4:5], -110430; GCN2-NEXT: s_cbranch_vccnz .LBB72_310431; GCN2-NEXT: ; %bb.1: ; %Flow10432; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]10433; GCN2-NEXT: s_cbranch_vccz .LBB72_410434; GCN2-NEXT: .LBB72_2: ; %atomicrmw.phi10435; GCN2-NEXT: s_endpgm10436; GCN2-NEXT: .LBB72_3: ; %atomicrmw.global10437; GCN2-NEXT: v_mov_b32_e32 v0, s010438; GCN2-NEXT: v_mov_b32_e32 v2, s210439; GCN2-NEXT: v_mov_b32_e32 v1, s110440; GCN2-NEXT: v_mov_b32_e32 v3, s310441; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]10442; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10443; GCN2-NEXT: buffer_wbinvl1_vol10444; GCN2-NEXT: s_cbranch_execnz .LBB72_210445; GCN2-NEXT: .LBB72_4: ; %atomicrmw.private10446; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 010447; GCN2-NEXT: s_cselect_b32 s0, s0, -110448; GCN2-NEXT: v_mov_b32_e32 v0, s210449; GCN2-NEXT: v_mov_b32_e32 v1, s010450; GCN2-NEXT: s_add_i32 s0, s0, 410451; GCN2-NEXT: buffer_store_dword v0, v1, s[88:91], 0 offen10452; GCN2-NEXT: v_mov_b32_e32 v0, s310453; GCN2-NEXT: v_mov_b32_e32 v1, s010454; GCN2-NEXT: buffer_store_dword v0, v1, s[88:91], 0 offen10455; GCN2-NEXT: s_endpgm10456;10457; GFX12-LABEL: atomic_xchg_i64_addr64:10458; GFX12: ; %bb.0: ; %entry10459; GFX12-NEXT: s_clause 0x110460; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x3410461; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2410462; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base10463; GFX12-NEXT: s_wait_kmcnt 0x010464; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 310465; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)10466; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]10467; GFX12-NEXT: s_cmp_eq_u32 s1, s710468; GFX12-NEXT: s_cselect_b32 s4, -1, 010469; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)10470; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s410471; GFX12-NEXT: s_mov_b32 s4, -110472; GFX12-NEXT: s_cbranch_vccnz .LBB72_310473; GFX12-NEXT: ; %bb.1: ; %Flow10474; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s410475; GFX12-NEXT: s_cbranch_vccz .LBB72_410476; GFX12-NEXT: .LBB72_2: ; %atomicrmw.phi10477; GFX12-NEXT: s_endpgm10478; GFX12-NEXT: .LBB72_3: ; %atomicrmw.global10479; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s110480; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s310481; GFX12-NEXT: flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV10482; GFX12-NEXT: s_wait_storecnt_dscnt 0x010483; GFX12-NEXT: global_inv scope:SCOPE_DEV10484; GFX12-NEXT: s_cbranch_execnz .LBB72_210485; GFX12-NEXT: .LBB72_4: ; %atomicrmw.private10486; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s310487; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 010488; GFX12-NEXT: s_cselect_b32 s0, s0, -110489; GFX12-NEXT: scratch_store_b64 off, v[0:1], s010490; GFX12-NEXT: s_endpgm10491entry:10492 %ptr = getelementptr inbounds i64, ptr %out, i64 %index10493 %tmp0 = atomicrmw volatile xchg ptr %ptr, i64 %in syncscope("agent") seq_cst10494 ret void10495}10496 10497define amdgpu_kernel void @atomic_xchg_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {10498; GCN1-LABEL: atomic_xchg_i64_ret_addr64:10499; GCN1: ; %bb.0: ; %entry10500; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD010501; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD110502; GCN1-NEXT: s_mov_b32 s18, -110503; GCN1-NEXT: s_mov_b32 s19, 0xe8f00010504; GCN1-NEXT: s_add_u32 s16, s16, s1110505; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x910506; GCN1-NEXT: s_load_dword s2, s[4:5], 0x4110507; GCN1-NEXT: s_addc_u32 s17, s17, 010508; GCN1-NEXT: s_waitcnt lgkmcnt(0)10509; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 310510; GCN1-NEXT: s_add_u32 s0, s8, s010511; GCN1-NEXT: s_addc_u32 s1, s9, s110512; GCN1-NEXT: s_cmp_eq_u32 s1, s210513; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 010514; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]10515; GCN1-NEXT: s_cbranch_vccz .LBB73_210516; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global10517; GCN1-NEXT: v_mov_b32_e32 v0, s010518; GCN1-NEXT: v_mov_b32_e32 v2, s1210519; GCN1-NEXT: v_mov_b32_e32 v1, s110520; GCN1-NEXT: v_mov_b32_e32 v3, s1310521; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc10522; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10523; GCN1-NEXT: buffer_wbinvl1_vol10524; GCN1-NEXT: s_cbranch_execz .LBB73_310525; GCN1-NEXT: s_branch .LBB73_410526; GCN1-NEXT: .LBB73_2:10527; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr110528; GCN1-NEXT: .LBB73_3: ; %atomicrmw.private10529; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 010530; GCN1-NEXT: v_mov_b32_e32 v4, s1210531; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec10532; GCN1-NEXT: s_cselect_b32 s0, s0, -110533; GCN1-NEXT: v_mov_b32_e32 v2, s010534; GCN1-NEXT: s_add_i32 s0, s0, 410535; GCN1-NEXT: v_mov_b32_e32 v3, s010536; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen10537; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen10538; GCN1-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen10539; GCN1-NEXT: v_mov_b32_e32 v2, s1310540; GCN1-NEXT: buffer_store_dword v2, v3, s[16:19], 0 offen10541; GCN1-NEXT: .LBB73_4: ; %atomicrmw.end10542; GCN1-NEXT: v_mov_b32_e32 v2, s1010543; GCN1-NEXT: v_mov_b32_e32 v3, s1110544; GCN1-NEXT: s_waitcnt vmcnt(2)10545; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]10546; GCN1-NEXT: s_endpgm10547;10548; GCN2-LABEL: atomic_xchg_i64_ret_addr64:10549; GCN2: ; %bb.0: ; %entry10550; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD010551; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD110552; GCN2-NEXT: s_mov_b32 s90, -110553; GCN2-NEXT: s_mov_b32 s91, 0xe8000010554; GCN2-NEXT: s_add_u32 s88, s88, s1110555; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2410556; GCN2-NEXT: s_load_dword s2, s[4:5], 0x10410557; GCN2-NEXT: s_addc_u32 s89, s89, 010558; GCN2-NEXT: s_waitcnt lgkmcnt(0)10559; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 310560; GCN2-NEXT: s_add_u32 s0, s8, s010561; GCN2-NEXT: s_addc_u32 s1, s9, s110562; GCN2-NEXT: s_cmp_eq_u32 s1, s210563; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 010564; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]10565; GCN2-NEXT: s_cbranch_vccz .LBB73_210566; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global10567; GCN2-NEXT: v_mov_b32_e32 v0, s010568; GCN2-NEXT: v_mov_b32_e32 v2, s1210569; GCN2-NEXT: v_mov_b32_e32 v1, s110570; GCN2-NEXT: v_mov_b32_e32 v3, s1310571; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc10572; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10573; GCN2-NEXT: buffer_wbinvl1_vol10574; GCN2-NEXT: s_cbranch_execz .LBB73_310575; GCN2-NEXT: s_branch .LBB73_410576; GCN2-NEXT: .LBB73_2:10577; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr110578; GCN2-NEXT: .LBB73_3: ; %atomicrmw.private10579; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 010580; GCN2-NEXT: s_cselect_b32 s0, s0, -110581; GCN2-NEXT: v_mov_b32_e32 v2, s010582; GCN2-NEXT: s_add_i32 s0, s0, 410583; GCN2-NEXT: v_mov_b32_e32 v3, s010584; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen10585; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen10586; GCN2-NEXT: v_mov_b32_e32 v4, s1210587; GCN2-NEXT: buffer_store_dword v4, v2, s[88:91], 0 offen10588; GCN2-NEXT: v_mov_b32_e32 v2, s1310589; GCN2-NEXT: buffer_store_dword v2, v3, s[88:91], 0 offen10590; GCN2-NEXT: .LBB73_4: ; %atomicrmw.end10591; GCN2-NEXT: v_mov_b32_e32 v2, s1010592; GCN2-NEXT: v_mov_b32_e32 v3, s1110593; GCN2-NEXT: s_waitcnt vmcnt(2)10594; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]10595; GCN2-NEXT: s_endpgm10596;10597; GFX12-LABEL: atomic_xchg_i64_ret_addr64:10598; GFX12: ; %bb.0: ; %entry10599; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x2410600; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base10601; GFX12-NEXT: s_wait_kmcnt 0x010602; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 310603; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)10604; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]10605; GFX12-NEXT: s_cmp_eq_u32 s1, s910606; GFX12-NEXT: s_cselect_b32 s6, -1, 010607; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)10608; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s610609; GFX12-NEXT: s_cbranch_vccz .LBB73_210610; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global10611; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s110612; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s510613; GFX12-NEXT: flat_atomic_swap_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV10614; GFX12-NEXT: s_wait_loadcnt_dscnt 0x010615; GFX12-NEXT: global_inv scope:SCOPE_DEV10616; GFX12-NEXT: s_cbranch_execz .LBB73_310617; GFX12-NEXT: s_branch .LBB73_410618; GFX12-NEXT: .LBB73_2:10619; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr110620; GFX12-NEXT: .LBB73_3: ; %atomicrmw.private10621; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 010622; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s510623; GFX12-NEXT: s_cselect_b32 s0, s0, -110624; GFX12-NEXT: scratch_load_b64 v[0:1], off, s010625; GFX12-NEXT: scratch_store_b64 off, v[2:3], s010626; GFX12-NEXT: .LBB73_4: ; %atomicrmw.end10627; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s310628; GFX12-NEXT: s_wait_loadcnt 0x010629; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]10630; GFX12-NEXT: s_endpgm10631entry:10632 %ptr = getelementptr inbounds i64, ptr %out, i64 %index10633 %tmp0 = atomicrmw volatile xchg ptr %ptr, i64 %in syncscope("agent") seq_cst10634 store i64 %tmp0, ptr %out210635 ret void10636}10637 10638define amdgpu_kernel void @atomic_xor_i64_offset(ptr %out, i64 %in) {10639; GCN1-LABEL: atomic_xor_i64_offset:10640; GCN1: ; %bb.0: ; %entry10641; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD010642; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD110643; GCN1-NEXT: s_mov_b32 s14, -110644; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x910645; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d10646; GCN1-NEXT: s_mov_b32 s15, 0xe8f00010647; GCN1-NEXT: s_add_u32 s12, s12, s1110648; GCN1-NEXT: s_addc_u32 s13, s13, 010649; GCN1-NEXT: s_waitcnt lgkmcnt(0)10650; GCN1-NEXT: s_add_u32 s0, s0, 3210651; GCN1-NEXT: s_addc_u32 s1, s1, 010652; GCN1-NEXT: s_cmp_eq_u32 s1, s410653; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 010654; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]10655; GCN1-NEXT: s_mov_b64 s[4:5], -110656; GCN1-NEXT: s_cbranch_vccnz .LBB74_310657; GCN1-NEXT: ; %bb.1: ; %Flow10658; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]10659; GCN1-NEXT: s_cbranch_vccz .LBB74_410660; GCN1-NEXT: .LBB74_2: ; %atomicrmw.phi10661; GCN1-NEXT: s_endpgm10662; GCN1-NEXT: .LBB74_3: ; %atomicrmw.global10663; GCN1-NEXT: v_mov_b32_e32 v0, s010664; GCN1-NEXT: v_mov_b32_e32 v2, s210665; GCN1-NEXT: v_mov_b32_e32 v1, s110666; GCN1-NEXT: v_mov_b32_e32 v3, s310667; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]10668; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10669; GCN1-NEXT: buffer_wbinvl1_vol10670; GCN1-NEXT: s_cbranch_execnz .LBB74_210671; GCN1-NEXT: .LBB74_4: ; %atomicrmw.private10672; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 010673; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec10674; GCN1-NEXT: s_cselect_b32 s0, s0, -110675; GCN1-NEXT: v_mov_b32_e32 v0, s010676; GCN1-NEXT: s_add_i32 s0, s0, 410677; GCN1-NEXT: v_mov_b32_e32 v1, s010678; GCN1-NEXT: buffer_load_dword v2, v0, s[12:15], 0 offen10679; GCN1-NEXT: buffer_load_dword v3, v1, s[12:15], 0 offen10680; GCN1-NEXT: s_waitcnt vmcnt(1)10681; GCN1-NEXT: v_xor_b32_e32 v2, s2, v210682; GCN1-NEXT: s_waitcnt vmcnt(0)10683; GCN1-NEXT: v_xor_b32_e32 v3, s3, v310684; GCN1-NEXT: buffer_store_dword v2, v0, s[12:15], 0 offen10685; GCN1-NEXT: buffer_store_dword v3, v1, s[12:15], 0 offen10686; GCN1-NEXT: s_endpgm10687;10688; GCN2-LABEL: atomic_xor_i64_offset:10689; GCN2: ; %bb.0: ; %entry10690; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD010691; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD110692; GCN2-NEXT: s_mov_b32 s90, -110693; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410694; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf410695; GCN2-NEXT: s_mov_b32 s91, 0xe8000010696; GCN2-NEXT: s_add_u32 s88, s88, s1110697; GCN2-NEXT: s_addc_u32 s89, s89, 010698; GCN2-NEXT: s_waitcnt lgkmcnt(0)10699; GCN2-NEXT: s_add_u32 s0, s0, 3210700; GCN2-NEXT: s_addc_u32 s1, s1, 010701; GCN2-NEXT: s_cmp_eq_u32 s1, s410702; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 010703; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]10704; GCN2-NEXT: s_mov_b64 s[4:5], -110705; GCN2-NEXT: s_cbranch_vccnz .LBB74_310706; GCN2-NEXT: ; %bb.1: ; %Flow10707; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]10708; GCN2-NEXT: s_cbranch_vccz .LBB74_410709; GCN2-NEXT: .LBB74_2: ; %atomicrmw.phi10710; GCN2-NEXT: s_endpgm10711; GCN2-NEXT: .LBB74_3: ; %atomicrmw.global10712; GCN2-NEXT: v_mov_b32_e32 v0, s010713; GCN2-NEXT: v_mov_b32_e32 v2, s210714; GCN2-NEXT: v_mov_b32_e32 v1, s110715; GCN2-NEXT: v_mov_b32_e32 v3, s310716; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]10717; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10718; GCN2-NEXT: buffer_wbinvl1_vol10719; GCN2-NEXT: s_cbranch_execnz .LBB74_210720; GCN2-NEXT: .LBB74_4: ; %atomicrmw.private10721; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 010722; GCN2-NEXT: s_cselect_b32 s0, s0, -110723; GCN2-NEXT: v_mov_b32_e32 v0, s010724; GCN2-NEXT: s_add_i32 s0, s0, 410725; GCN2-NEXT: v_mov_b32_e32 v1, s010726; GCN2-NEXT: buffer_load_dword v2, v0, s[88:91], 0 offen10727; GCN2-NEXT: buffer_load_dword v3, v1, s[88:91], 0 offen10728; GCN2-NEXT: s_waitcnt vmcnt(1)10729; GCN2-NEXT: v_xor_b32_e32 v2, s2, v210730; GCN2-NEXT: s_waitcnt vmcnt(0)10731; GCN2-NEXT: v_xor_b32_e32 v3, s3, v310732; GCN2-NEXT: buffer_store_dword v2, v0, s[88:91], 0 offen10733; GCN2-NEXT: buffer_store_dword v3, v1, s[88:91], 0 offen10734; GCN2-NEXT: s_endpgm10735;10736; GFX12-LABEL: atomic_xor_i64_offset:10737; GFX12: ; %bb.0: ; %entry10738; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2410739; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base10740; GFX12-NEXT: s_wait_kmcnt 0x010741; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 3210742; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)10743; GFX12-NEXT: s_cmp_eq_u32 s1, s710744; GFX12-NEXT: s_cselect_b32 s4, -1, 010745; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s410746; GFX12-NEXT: s_mov_b32 s4, -110747; GFX12-NEXT: s_cbranch_vccnz .LBB74_310748; GFX12-NEXT: ; %bb.1: ; %Flow10749; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s410750; GFX12-NEXT: s_cbranch_vccz .LBB74_410751; GFX12-NEXT: .LBB74_2: ; %atomicrmw.phi10752; GFX12-NEXT: s_endpgm10753; GFX12-NEXT: .LBB74_3: ; %atomicrmw.global10754; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s110755; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s310756; GFX12-NEXT: flat_atomic_xor_b64 v[0:1], v[2:3] scope:SCOPE_DEV10757; GFX12-NEXT: s_wait_storecnt_dscnt 0x010758; GFX12-NEXT: global_inv scope:SCOPE_DEV10759; GFX12-NEXT: s_cbranch_execnz .LBB74_210760; GFX12-NEXT: .LBB74_4: ; %atomicrmw.private10761; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 010762; GFX12-NEXT: s_cselect_b32 s0, s0, -110763; GFX12-NEXT: scratch_load_b64 v[0:1], off, s010764; GFX12-NEXT: s_wait_loadcnt 0x010765; GFX12-NEXT: v_xor_b32_e32 v1, s3, v110766; GFX12-NEXT: v_xor_b32_e32 v0, s2, v010767; GFX12-NEXT: scratch_store_b64 off, v[0:1], s010768; GFX12-NEXT: s_endpgm10769entry:10770 %gep = getelementptr inbounds i64, ptr %out, i64 410771 %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !010772 ret void10773}10774 10775define amdgpu_kernel void @atomic_xor_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {10776; GCN1-LABEL: atomic_xor_i64_ret_offset:10777; GCN1: ; %bb.0: ; %entry10778; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD010779; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD110780; GCN1-NEXT: s_mov_b32 s14, -110781; GCN1-NEXT: s_mov_b32 s15, 0xe8f00010782; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x910783; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f10784; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd10785; GCN1-NEXT: s_add_u32 s12, s12, s1110786; GCN1-NEXT: s_addc_u32 s13, s13, 010787; GCN1-NEXT: s_waitcnt lgkmcnt(0)10788; GCN1-NEXT: s_add_u32 s0, s0, 3210789; GCN1-NEXT: s_addc_u32 s1, s1, 010790; GCN1-NEXT: s_cmp_eq_u32 s1, s610791; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 010792; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]10793; GCN1-NEXT: s_cbranch_vccz .LBB75_210794; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global10795; GCN1-NEXT: v_mov_b32_e32 v0, s010796; GCN1-NEXT: v_mov_b32_e32 v2, s410797; GCN1-NEXT: v_mov_b32_e32 v1, s110798; GCN1-NEXT: v_mov_b32_e32 v3, s510799; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc10800; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10801; GCN1-NEXT: buffer_wbinvl1_vol10802; GCN1-NEXT: s_cbranch_execz .LBB75_310803; GCN1-NEXT: s_branch .LBB75_410804; GCN1-NEXT: .LBB75_2:10805; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr110806; GCN1-NEXT: .LBB75_3: ; %atomicrmw.private10807; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 010808; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec10809; GCN1-NEXT: s_cselect_b32 s0, s0, -110810; GCN1-NEXT: v_mov_b32_e32 v2, s010811; GCN1-NEXT: s_add_i32 s0, s0, 410812; GCN1-NEXT: v_mov_b32_e32 v3, s010813; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen10814; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen10815; GCN1-NEXT: s_waitcnt vmcnt(1)10816; GCN1-NEXT: v_xor_b32_e32 v4, s4, v010817; GCN1-NEXT: s_waitcnt vmcnt(0)10818; GCN1-NEXT: v_xor_b32_e32 v5, s5, v110819; GCN1-NEXT: buffer_store_dword v4, v2, s[12:15], 0 offen10820; GCN1-NEXT: buffer_store_dword v5, v3, s[12:15], 0 offen10821; GCN1-NEXT: .LBB75_4: ; %atomicrmw.end10822; GCN1-NEXT: v_mov_b32_e32 v2, s210823; GCN1-NEXT: v_mov_b32_e32 v3, s310824; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]10825; GCN1-NEXT: s_endpgm10826;10827; GCN2-LABEL: atomic_xor_i64_ret_offset:10828; GCN2: ; %bb.0: ; %entry10829; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD010830; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD110831; GCN2-NEXT: s_mov_b32 s90, -110832; GCN2-NEXT: s_mov_b32 s91, 0xe8000010833; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410834; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc10835; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x3410836; GCN2-NEXT: s_add_u32 s88, s88, s1110837; GCN2-NEXT: s_addc_u32 s89, s89, 010838; GCN2-NEXT: s_waitcnt lgkmcnt(0)10839; GCN2-NEXT: s_add_u32 s0, s0, 3210840; GCN2-NEXT: s_addc_u32 s1, s1, 010841; GCN2-NEXT: s_cmp_eq_u32 s1, s610842; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 010843; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]10844; GCN2-NEXT: s_cbranch_vccz .LBB75_210845; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global10846; GCN2-NEXT: v_mov_b32_e32 v0, s010847; GCN2-NEXT: v_mov_b32_e32 v2, s410848; GCN2-NEXT: v_mov_b32_e32 v1, s110849; GCN2-NEXT: v_mov_b32_e32 v3, s510850; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc10851; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10852; GCN2-NEXT: buffer_wbinvl1_vol10853; GCN2-NEXT: s_cbranch_execz .LBB75_310854; GCN2-NEXT: s_branch .LBB75_410855; GCN2-NEXT: .LBB75_2:10856; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr110857; GCN2-NEXT: .LBB75_3: ; %atomicrmw.private10858; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 010859; GCN2-NEXT: s_cselect_b32 s0, s0, -110860; GCN2-NEXT: v_mov_b32_e32 v2, s010861; GCN2-NEXT: s_add_i32 s0, s0, 410862; GCN2-NEXT: v_mov_b32_e32 v3, s010863; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen10864; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen10865; GCN2-NEXT: s_waitcnt vmcnt(1)10866; GCN2-NEXT: v_xor_b32_e32 v4, s4, v010867; GCN2-NEXT: s_waitcnt vmcnt(0)10868; GCN2-NEXT: v_xor_b32_e32 v5, s5, v110869; GCN2-NEXT: buffer_store_dword v4, v2, s[88:91], 0 offen10870; GCN2-NEXT: buffer_store_dword v5, v3, s[88:91], 0 offen10871; GCN2-NEXT: .LBB75_4: ; %atomicrmw.end10872; GCN2-NEXT: v_mov_b32_e32 v2, s210873; GCN2-NEXT: v_mov_b32_e32 v3, s310874; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]10875; GCN2-NEXT: s_endpgm10876;10877; GFX12-LABEL: atomic_xor_i64_ret_offset:10878; GFX12: ; %bb.0: ; %entry10879; GFX12-NEXT: s_clause 0x110880; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2410881; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x3410882; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base10883; GFX12-NEXT: s_wait_kmcnt 0x010884; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 3210885; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)10886; GFX12-NEXT: s_cmp_eq_u32 s1, s710887; GFX12-NEXT: s_cselect_b32 s6, -1, 010888; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s610889; GFX12-NEXT: s_cbranch_vccz .LBB75_210890; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global10891; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s110892; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s510893; GFX12-NEXT: flat_atomic_xor_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV10894; GFX12-NEXT: s_wait_loadcnt_dscnt 0x010895; GFX12-NEXT: global_inv scope:SCOPE_DEV10896; GFX12-NEXT: s_cbranch_execz .LBB75_310897; GFX12-NEXT: s_branch .LBB75_410898; GFX12-NEXT: .LBB75_2:10899; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr110900; GFX12-NEXT: .LBB75_3: ; %atomicrmw.private10901; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 010902; GFX12-NEXT: s_cselect_b32 s0, s0, -110903; GFX12-NEXT: scratch_load_b64 v[0:1], off, s010904; GFX12-NEXT: s_wait_loadcnt 0x010905; GFX12-NEXT: v_xor_b32_e32 v3, s5, v110906; GFX12-NEXT: v_xor_b32_e32 v2, s4, v010907; GFX12-NEXT: scratch_store_b64 off, v[2:3], s010908; GFX12-NEXT: .LBB75_4: ; %atomicrmw.end10909; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s310910; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]10911; GFX12-NEXT: s_endpgm10912entry:10913 %gep = getelementptr inbounds i64, ptr %out, i64 410914 %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !010915 store i64 %tmp0, ptr %out210916 ret void10917}10918 10919define amdgpu_kernel void @atomic_xor_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {10920; GCN1-LABEL: atomic_xor_i64_addr64_offset:10921; GCN1: ; %bb.0: ; %entry10922; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD010923; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD110924; GCN1-NEXT: s_mov_b32 s14, -110925; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd10926; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x910927; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f10928; GCN1-NEXT: s_mov_b32 s15, 0xe8f00010929; GCN1-NEXT: s_add_u32 s12, s12, s1110930; GCN1-NEXT: s_addc_u32 s13, s13, 010931; GCN1-NEXT: s_waitcnt lgkmcnt(0)10932; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 310933; GCN1-NEXT: s_add_u32 s0, s0, s410934; GCN1-NEXT: s_addc_u32 s1, s1, s510935; GCN1-NEXT: s_add_u32 s0, s0, 3210936; GCN1-NEXT: s_addc_u32 s1, s1, 010937; GCN1-NEXT: s_cmp_eq_u32 s1, s810938; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 010939; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]10940; GCN1-NEXT: s_mov_b64 s[4:5], -110941; GCN1-NEXT: s_cbranch_vccnz .LBB76_310942; GCN1-NEXT: ; %bb.1: ; %Flow10943; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]10944; GCN1-NEXT: s_cbranch_vccz .LBB76_410945; GCN1-NEXT: .LBB76_2: ; %atomicrmw.phi10946; GCN1-NEXT: s_endpgm10947; GCN1-NEXT: .LBB76_3: ; %atomicrmw.global10948; GCN1-NEXT: v_mov_b32_e32 v0, s010949; GCN1-NEXT: v_mov_b32_e32 v2, s210950; GCN1-NEXT: v_mov_b32_e32 v1, s110951; GCN1-NEXT: v_mov_b32_e32 v3, s310952; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]10953; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10954; GCN1-NEXT: buffer_wbinvl1_vol10955; GCN1-NEXT: s_cbranch_execnz .LBB76_210956; GCN1-NEXT: .LBB76_4: ; %atomicrmw.private10957; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 010958; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec10959; GCN1-NEXT: s_cselect_b32 s0, s0, -110960; GCN1-NEXT: v_mov_b32_e32 v0, s010961; GCN1-NEXT: s_add_i32 s0, s0, 410962; GCN1-NEXT: v_mov_b32_e32 v1, s010963; GCN1-NEXT: buffer_load_dword v2, v0, s[12:15], 0 offen10964; GCN1-NEXT: buffer_load_dword v3, v1, s[12:15], 0 offen10965; GCN1-NEXT: s_waitcnt vmcnt(1)10966; GCN1-NEXT: v_xor_b32_e32 v2, s2, v210967; GCN1-NEXT: s_waitcnt vmcnt(0)10968; GCN1-NEXT: v_xor_b32_e32 v3, s3, v310969; GCN1-NEXT: buffer_store_dword v2, v0, s[12:15], 0 offen10970; GCN1-NEXT: buffer_store_dword v3, v1, s[12:15], 0 offen10971; GCN1-NEXT: s_endpgm10972;10973; GCN2-LABEL: atomic_xor_i64_addr64_offset:10974; GCN2: ; %bb.0: ; %entry10975; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD010976; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD110977; GCN2-NEXT: s_mov_b32 s90, -110978; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3410979; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410980; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc10981; GCN2-NEXT: s_mov_b32 s91, 0xe8000010982; GCN2-NEXT: s_add_u32 s88, s88, s1110983; GCN2-NEXT: s_addc_u32 s89, s89, 010984; GCN2-NEXT: s_waitcnt lgkmcnt(0)10985; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 310986; GCN2-NEXT: s_add_u32 s0, s0, s410987; GCN2-NEXT: s_addc_u32 s1, s1, s510988; GCN2-NEXT: s_add_u32 s0, s0, 3210989; GCN2-NEXT: s_addc_u32 s1, s1, 010990; GCN2-NEXT: s_cmp_eq_u32 s1, s810991; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 010992; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]10993; GCN2-NEXT: s_mov_b64 s[4:5], -110994; GCN2-NEXT: s_cbranch_vccnz .LBB76_310995; GCN2-NEXT: ; %bb.1: ; %Flow10996; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]10997; GCN2-NEXT: s_cbranch_vccz .LBB76_410998; GCN2-NEXT: .LBB76_2: ; %atomicrmw.phi10999; GCN2-NEXT: s_endpgm11000; GCN2-NEXT: .LBB76_3: ; %atomicrmw.global11001; GCN2-NEXT: v_mov_b32_e32 v0, s011002; GCN2-NEXT: v_mov_b32_e32 v2, s211003; GCN2-NEXT: v_mov_b32_e32 v1, s111004; GCN2-NEXT: v_mov_b32_e32 v3, s311005; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]11006; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11007; GCN2-NEXT: buffer_wbinvl1_vol11008; GCN2-NEXT: s_cbranch_execnz .LBB76_211009; GCN2-NEXT: .LBB76_4: ; %atomicrmw.private11010; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 011011; GCN2-NEXT: s_cselect_b32 s0, s0, -111012; GCN2-NEXT: v_mov_b32_e32 v0, s011013; GCN2-NEXT: s_add_i32 s0, s0, 411014; GCN2-NEXT: v_mov_b32_e32 v1, s011015; GCN2-NEXT: buffer_load_dword v2, v0, s[88:91], 0 offen11016; GCN2-NEXT: buffer_load_dword v3, v1, s[88:91], 0 offen11017; GCN2-NEXT: s_waitcnt vmcnt(1)11018; GCN2-NEXT: v_xor_b32_e32 v2, s2, v211019; GCN2-NEXT: s_waitcnt vmcnt(0)11020; GCN2-NEXT: v_xor_b32_e32 v3, s3, v311021; GCN2-NEXT: buffer_store_dword v2, v0, s[88:91], 0 offen11022; GCN2-NEXT: buffer_store_dword v3, v1, s[88:91], 0 offen11023; GCN2-NEXT: s_endpgm11024;11025; GFX12-LABEL: atomic_xor_i64_addr64_offset:11026; GFX12: ; %bb.0: ; %entry11027; GFX12-NEXT: s_clause 0x111028; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x3411029; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2411030; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base11031; GFX12-NEXT: s_wait_kmcnt 0x011032; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 311033; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)11034; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]11035; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 3211036; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)11037; GFX12-NEXT: s_cmp_eq_u32 s1, s711038; GFX12-NEXT: s_cselect_b32 s4, -1, 011039; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s411040; GFX12-NEXT: s_mov_b32 s4, -111041; GFX12-NEXT: s_cbranch_vccnz .LBB76_311042; GFX12-NEXT: ; %bb.1: ; %Flow11043; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s411044; GFX12-NEXT: s_cbranch_vccz .LBB76_411045; GFX12-NEXT: .LBB76_2: ; %atomicrmw.phi11046; GFX12-NEXT: s_endpgm11047; GFX12-NEXT: .LBB76_3: ; %atomicrmw.global11048; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s111049; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s311050; GFX12-NEXT: flat_atomic_xor_b64 v[0:1], v[2:3] scope:SCOPE_DEV11051; GFX12-NEXT: s_wait_storecnt_dscnt 0x011052; GFX12-NEXT: global_inv scope:SCOPE_DEV11053; GFX12-NEXT: s_cbranch_execnz .LBB76_211054; GFX12-NEXT: .LBB76_4: ; %atomicrmw.private11055; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 011056; GFX12-NEXT: s_cselect_b32 s0, s0, -111057; GFX12-NEXT: scratch_load_b64 v[0:1], off, s011058; GFX12-NEXT: s_wait_loadcnt 0x011059; GFX12-NEXT: v_xor_b32_e32 v1, s3, v111060; GFX12-NEXT: v_xor_b32_e32 v0, s2, v011061; GFX12-NEXT: scratch_store_b64 off, v[0:1], s011062; GFX12-NEXT: s_endpgm11063entry:11064 %ptr = getelementptr inbounds i64, ptr %out, i64 %index11065 %gep = getelementptr inbounds i64, ptr %ptr, i64 411066 %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !011067 ret void11068}11069 11070define amdgpu_kernel void @atomic_xor_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {11071; GCN1-LABEL: atomic_xor_i64_ret_addr64_offset:11072; GCN1: ; %bb.0: ; %entry11073; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD011074; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD111075; GCN1-NEXT: s_mov_b32 s18, -111076; GCN1-NEXT: s_mov_b32 s19, 0xe8f00011077; GCN1-NEXT: s_add_u32 s16, s16, s1111078; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x911079; GCN1-NEXT: s_load_dword s2, s[4:5], 0x4111080; GCN1-NEXT: s_addc_u32 s17, s17, 011081; GCN1-NEXT: s_waitcnt lgkmcnt(0)11082; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 311083; GCN1-NEXT: s_add_u32 s0, s8, s011084; GCN1-NEXT: s_addc_u32 s1, s9, s111085; GCN1-NEXT: s_add_u32 s0, s0, 3211086; GCN1-NEXT: s_addc_u32 s1, s1, 011087; GCN1-NEXT: s_cmp_eq_u32 s1, s211088; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 011089; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]11090; GCN1-NEXT: s_cbranch_vccz .LBB77_211091; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global11092; GCN1-NEXT: v_mov_b32_e32 v0, s011093; GCN1-NEXT: v_mov_b32_e32 v2, s1211094; GCN1-NEXT: v_mov_b32_e32 v1, s111095; GCN1-NEXT: v_mov_b32_e32 v3, s1311096; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc11097; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11098; GCN1-NEXT: buffer_wbinvl1_vol11099; GCN1-NEXT: s_cbranch_execz .LBB77_311100; GCN1-NEXT: s_branch .LBB77_411101; GCN1-NEXT: .LBB77_2:11102; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr111103; GCN1-NEXT: .LBB77_3: ; %atomicrmw.private11104; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 011105; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec11106; GCN1-NEXT: s_cselect_b32 s0, s0, -111107; GCN1-NEXT: v_mov_b32_e32 v2, s011108; GCN1-NEXT: s_add_i32 s0, s0, 411109; GCN1-NEXT: v_mov_b32_e32 v3, s011110; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen11111; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen11112; GCN1-NEXT: s_waitcnt vmcnt(1)11113; GCN1-NEXT: v_xor_b32_e32 v4, s12, v011114; GCN1-NEXT: s_waitcnt vmcnt(0)11115; GCN1-NEXT: v_xor_b32_e32 v5, s13, v111116; GCN1-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen11117; GCN1-NEXT: buffer_store_dword v5, v3, s[16:19], 0 offen11118; GCN1-NEXT: .LBB77_4: ; %atomicrmw.end11119; GCN1-NEXT: v_mov_b32_e32 v2, s1011120; GCN1-NEXT: v_mov_b32_e32 v3, s1111121; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]11122; GCN1-NEXT: s_endpgm11123;11124; GCN2-LABEL: atomic_xor_i64_ret_addr64_offset:11125; GCN2: ; %bb.0: ; %entry11126; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD011127; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD111128; GCN2-NEXT: s_mov_b32 s90, -111129; GCN2-NEXT: s_mov_b32 s91, 0xe8000011130; GCN2-NEXT: s_add_u32 s88, s88, s1111131; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2411132; GCN2-NEXT: s_load_dword s2, s[4:5], 0x10411133; GCN2-NEXT: s_addc_u32 s89, s89, 011134; GCN2-NEXT: s_waitcnt lgkmcnt(0)11135; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 311136; GCN2-NEXT: s_add_u32 s0, s8, s011137; GCN2-NEXT: s_addc_u32 s1, s9, s111138; GCN2-NEXT: s_add_u32 s0, s0, 3211139; GCN2-NEXT: s_addc_u32 s1, s1, 011140; GCN2-NEXT: s_cmp_eq_u32 s1, s211141; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 011142; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]11143; GCN2-NEXT: s_cbranch_vccz .LBB77_211144; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global11145; GCN2-NEXT: v_mov_b32_e32 v0, s011146; GCN2-NEXT: v_mov_b32_e32 v2, s1211147; GCN2-NEXT: v_mov_b32_e32 v1, s111148; GCN2-NEXT: v_mov_b32_e32 v3, s1311149; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc11150; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11151; GCN2-NEXT: buffer_wbinvl1_vol11152; GCN2-NEXT: s_cbranch_execz .LBB77_311153; GCN2-NEXT: s_branch .LBB77_411154; GCN2-NEXT: .LBB77_2:11155; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr111156; GCN2-NEXT: .LBB77_3: ; %atomicrmw.private11157; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 011158; GCN2-NEXT: s_cselect_b32 s0, s0, -111159; GCN2-NEXT: v_mov_b32_e32 v2, s011160; GCN2-NEXT: s_add_i32 s0, s0, 411161; GCN2-NEXT: v_mov_b32_e32 v3, s011162; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen11163; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen11164; GCN2-NEXT: s_waitcnt vmcnt(1)11165; GCN2-NEXT: v_xor_b32_e32 v4, s12, v011166; GCN2-NEXT: s_waitcnt vmcnt(0)11167; GCN2-NEXT: v_xor_b32_e32 v5, s13, v111168; GCN2-NEXT: buffer_store_dword v4, v2, s[88:91], 0 offen11169; GCN2-NEXT: buffer_store_dword v5, v3, s[88:91], 0 offen11170; GCN2-NEXT: .LBB77_4: ; %atomicrmw.end11171; GCN2-NEXT: v_mov_b32_e32 v2, s1011172; GCN2-NEXT: v_mov_b32_e32 v3, s1111173; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]11174; GCN2-NEXT: s_endpgm11175;11176; GFX12-LABEL: atomic_xor_i64_ret_addr64_offset:11177; GFX12: ; %bb.0: ; %entry11178; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x2411179; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base11180; GFX12-NEXT: s_wait_kmcnt 0x011181; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 311182; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)11183; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]11184; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 3211185; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)11186; GFX12-NEXT: s_cmp_eq_u32 s1, s911187; GFX12-NEXT: s_cselect_b32 s6, -1, 011188; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s611189; GFX12-NEXT: s_cbranch_vccz .LBB77_211190; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global11191; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s111192; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s511193; GFX12-NEXT: flat_atomic_xor_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV11194; GFX12-NEXT: s_wait_loadcnt_dscnt 0x011195; GFX12-NEXT: global_inv scope:SCOPE_DEV11196; GFX12-NEXT: s_cbranch_execz .LBB77_311197; GFX12-NEXT: s_branch .LBB77_411198; GFX12-NEXT: .LBB77_2:11199; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr111200; GFX12-NEXT: .LBB77_3: ; %atomicrmw.private11201; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 011202; GFX12-NEXT: s_cselect_b32 s0, s0, -111203; GFX12-NEXT: scratch_load_b64 v[0:1], off, s011204; GFX12-NEXT: s_wait_loadcnt 0x011205; GFX12-NEXT: v_xor_b32_e32 v3, s5, v111206; GFX12-NEXT: v_xor_b32_e32 v2, s4, v011207; GFX12-NEXT: scratch_store_b64 off, v[2:3], s011208; GFX12-NEXT: .LBB77_4: ; %atomicrmw.end11209; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s311210; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]11211; GFX12-NEXT: s_endpgm11212entry:11213 %ptr = getelementptr inbounds i64, ptr %out, i64 %index11214 %gep = getelementptr inbounds i64, ptr %ptr, i64 411215 %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !011216 store i64 %tmp0, ptr %out211217 ret void11218}11219 11220define amdgpu_kernel void @atomic_xor_i64(ptr %out, i64 %in) {11221; GCN1-LABEL: atomic_xor_i64:11222; GCN1: ; %bb.0: ; %entry11223; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD011224; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD111225; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x911226; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d11227; GCN1-NEXT: s_mov_b32 s14, -111228; GCN1-NEXT: s_mov_b32 s15, 0xe8f00011229; GCN1-NEXT: s_add_u32 s12, s12, s1111230; GCN1-NEXT: s_addc_u32 s13, s13, 011231; GCN1-NEXT: s_waitcnt lgkmcnt(0)11232; GCN1-NEXT: s_cmp_eq_u32 s1, s411233; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 011234; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]11235; GCN1-NEXT: s_mov_b64 s[4:5], -111236; GCN1-NEXT: s_cbranch_vccnz .LBB78_311237; GCN1-NEXT: ; %bb.1: ; %Flow11238; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]11239; GCN1-NEXT: s_cbranch_vccz .LBB78_411240; GCN1-NEXT: .LBB78_2: ; %atomicrmw.phi11241; GCN1-NEXT: s_endpgm11242; GCN1-NEXT: .LBB78_3: ; %atomicrmw.global11243; GCN1-NEXT: v_mov_b32_e32 v0, s011244; GCN1-NEXT: v_mov_b32_e32 v2, s211245; GCN1-NEXT: v_mov_b32_e32 v1, s111246; GCN1-NEXT: v_mov_b32_e32 v3, s311247; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]11248; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11249; GCN1-NEXT: buffer_wbinvl1_vol11250; GCN1-NEXT: s_cbranch_execnz .LBB78_211251; GCN1-NEXT: .LBB78_4: ; %atomicrmw.private11252; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 011253; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec11254; GCN1-NEXT: s_cselect_b32 s0, s0, -111255; GCN1-NEXT: v_mov_b32_e32 v0, s011256; GCN1-NEXT: s_add_i32 s0, s0, 411257; GCN1-NEXT: v_mov_b32_e32 v1, s011258; GCN1-NEXT: buffer_load_dword v2, v0, s[12:15], 0 offen11259; GCN1-NEXT: buffer_load_dword v3, v1, s[12:15], 0 offen11260; GCN1-NEXT: s_waitcnt vmcnt(1)11261; GCN1-NEXT: v_xor_b32_e32 v2, s2, v211262; GCN1-NEXT: s_waitcnt vmcnt(0)11263; GCN1-NEXT: v_xor_b32_e32 v3, s3, v311264; GCN1-NEXT: buffer_store_dword v2, v0, s[12:15], 0 offen11265; GCN1-NEXT: buffer_store_dword v3, v1, s[12:15], 0 offen11266; GCN1-NEXT: s_endpgm11267;11268; GCN2-LABEL: atomic_xor_i64:11269; GCN2: ; %bb.0: ; %entry11270; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD011271; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD111272; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2411273; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf411274; GCN2-NEXT: s_mov_b32 s90, -111275; GCN2-NEXT: s_mov_b32 s91, 0xe8000011276; GCN2-NEXT: s_add_u32 s88, s88, s1111277; GCN2-NEXT: s_addc_u32 s89, s89, 011278; GCN2-NEXT: s_waitcnt lgkmcnt(0)11279; GCN2-NEXT: s_cmp_eq_u32 s1, s411280; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 011281; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]11282; GCN2-NEXT: s_mov_b64 s[4:5], -111283; GCN2-NEXT: s_cbranch_vccnz .LBB78_311284; GCN2-NEXT: ; %bb.1: ; %Flow11285; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]11286; GCN2-NEXT: s_cbranch_vccz .LBB78_411287; GCN2-NEXT: .LBB78_2: ; %atomicrmw.phi11288; GCN2-NEXT: s_endpgm11289; GCN2-NEXT: .LBB78_3: ; %atomicrmw.global11290; GCN2-NEXT: v_mov_b32_e32 v0, s011291; GCN2-NEXT: v_mov_b32_e32 v2, s211292; GCN2-NEXT: v_mov_b32_e32 v1, s111293; GCN2-NEXT: v_mov_b32_e32 v3, s311294; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]11295; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11296; GCN2-NEXT: buffer_wbinvl1_vol11297; GCN2-NEXT: s_cbranch_execnz .LBB78_211298; GCN2-NEXT: .LBB78_4: ; %atomicrmw.private11299; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 011300; GCN2-NEXT: s_cselect_b32 s0, s0, -111301; GCN2-NEXT: v_mov_b32_e32 v0, s011302; GCN2-NEXT: s_add_i32 s0, s0, 411303; GCN2-NEXT: v_mov_b32_e32 v1, s011304; GCN2-NEXT: buffer_load_dword v2, v0, s[88:91], 0 offen11305; GCN2-NEXT: buffer_load_dword v3, v1, s[88:91], 0 offen11306; GCN2-NEXT: s_waitcnt vmcnt(1)11307; GCN2-NEXT: v_xor_b32_e32 v2, s2, v211308; GCN2-NEXT: s_waitcnt vmcnt(0)11309; GCN2-NEXT: v_xor_b32_e32 v3, s3, v311310; GCN2-NEXT: buffer_store_dword v2, v0, s[88:91], 0 offen11311; GCN2-NEXT: buffer_store_dword v3, v1, s[88:91], 0 offen11312; GCN2-NEXT: s_endpgm11313;11314; GFX12-LABEL: atomic_xor_i64:11315; GFX12: ; %bb.0: ; %entry11316; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2411317; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base11318; GFX12-NEXT: s_wait_kmcnt 0x011319; GFX12-NEXT: s_cmp_eq_u32 s1, s711320; GFX12-NEXT: s_cselect_b32 s4, -1, 011321; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11322; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s411323; GFX12-NEXT: s_mov_b32 s4, -111324; GFX12-NEXT: s_cbranch_vccnz .LBB78_311325; GFX12-NEXT: ; %bb.1: ; %Flow11326; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s411327; GFX12-NEXT: s_cbranch_vccz .LBB78_411328; GFX12-NEXT: .LBB78_2: ; %atomicrmw.phi11329; GFX12-NEXT: s_endpgm11330; GFX12-NEXT: .LBB78_3: ; %atomicrmw.global11331; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s111332; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s311333; GFX12-NEXT: flat_atomic_xor_b64 v[0:1], v[2:3] scope:SCOPE_DEV11334; GFX12-NEXT: s_wait_storecnt_dscnt 0x011335; GFX12-NEXT: global_inv scope:SCOPE_DEV11336; GFX12-NEXT: s_cbranch_execnz .LBB78_211337; GFX12-NEXT: .LBB78_4: ; %atomicrmw.private11338; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 011339; GFX12-NEXT: s_cselect_b32 s0, s0, -111340; GFX12-NEXT: scratch_load_b64 v[0:1], off, s011341; GFX12-NEXT: s_wait_loadcnt 0x011342; GFX12-NEXT: v_xor_b32_e32 v1, s3, v111343; GFX12-NEXT: v_xor_b32_e32 v0, s2, v011344; GFX12-NEXT: scratch_store_b64 off, v[0:1], s011345; GFX12-NEXT: s_endpgm11346entry:11347 %tmp0 = atomicrmw volatile xor ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !011348 ret void11349}11350 11351define amdgpu_kernel void @atomic_xor_i64_ret(ptr %out, ptr %out2, i64 %in) {11352; GCN1-LABEL: atomic_xor_i64_ret:11353; GCN1: ; %bb.0: ; %entry11354; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD011355; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD111356; GCN1-NEXT: s_mov_b32 s14, -111357; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f11358; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x911359; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd11360; GCN1-NEXT: s_mov_b32 s15, 0xe8f00011361; GCN1-NEXT: s_add_u32 s12, s12, s1111362; GCN1-NEXT: s_addc_u32 s13, s13, 011363; GCN1-NEXT: s_waitcnt lgkmcnt(0)11364; GCN1-NEXT: s_cmp_eq_u32 s1, s611365; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 011366; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]11367; GCN1-NEXT: s_cbranch_vccz .LBB79_211368; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global11369; GCN1-NEXT: v_mov_b32_e32 v0, s011370; GCN1-NEXT: v_mov_b32_e32 v2, s411371; GCN1-NEXT: v_mov_b32_e32 v1, s111372; GCN1-NEXT: v_mov_b32_e32 v3, s511373; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc11374; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11375; GCN1-NEXT: buffer_wbinvl1_vol11376; GCN1-NEXT: s_cbranch_execz .LBB79_311377; GCN1-NEXT: s_branch .LBB79_411378; GCN1-NEXT: .LBB79_2:11379; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr111380; GCN1-NEXT: .LBB79_3: ; %atomicrmw.private11381; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 011382; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec11383; GCN1-NEXT: s_cselect_b32 s0, s0, -111384; GCN1-NEXT: v_mov_b32_e32 v2, s011385; GCN1-NEXT: s_add_i32 s0, s0, 411386; GCN1-NEXT: v_mov_b32_e32 v3, s011387; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen11388; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen11389; GCN1-NEXT: s_waitcnt vmcnt(1)11390; GCN1-NEXT: v_xor_b32_e32 v4, s4, v011391; GCN1-NEXT: s_waitcnt vmcnt(0)11392; GCN1-NEXT: v_xor_b32_e32 v5, s5, v111393; GCN1-NEXT: buffer_store_dword v4, v2, s[12:15], 0 offen11394; GCN1-NEXT: buffer_store_dword v5, v3, s[12:15], 0 offen11395; GCN1-NEXT: .LBB79_4: ; %atomicrmw.end11396; GCN1-NEXT: v_mov_b32_e32 v2, s211397; GCN1-NEXT: v_mov_b32_e32 v3, s311398; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]11399; GCN1-NEXT: s_endpgm11400;11401; GCN2-LABEL: atomic_xor_i64_ret:11402; GCN2: ; %bb.0: ; %entry11403; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD011404; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD111405; GCN2-NEXT: s_mov_b32 s90, -111406; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc11407; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2411408; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x3411409; GCN2-NEXT: s_mov_b32 s91, 0xe8000011410; GCN2-NEXT: s_add_u32 s88, s88, s1111411; GCN2-NEXT: s_addc_u32 s89, s89, 011412; GCN2-NEXT: s_waitcnt lgkmcnt(0)11413; GCN2-NEXT: s_cmp_eq_u32 s1, s611414; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 011415; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]11416; GCN2-NEXT: s_cbranch_vccz .LBB79_211417; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global11418; GCN2-NEXT: v_mov_b32_e32 v0, s011419; GCN2-NEXT: v_mov_b32_e32 v2, s411420; GCN2-NEXT: v_mov_b32_e32 v1, s111421; GCN2-NEXT: v_mov_b32_e32 v3, s511422; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc11423; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11424; GCN2-NEXT: buffer_wbinvl1_vol11425; GCN2-NEXT: s_cbranch_execz .LBB79_311426; GCN2-NEXT: s_branch .LBB79_411427; GCN2-NEXT: .LBB79_2:11428; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr111429; GCN2-NEXT: .LBB79_3: ; %atomicrmw.private11430; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 011431; GCN2-NEXT: s_cselect_b32 s0, s0, -111432; GCN2-NEXT: v_mov_b32_e32 v2, s011433; GCN2-NEXT: s_add_i32 s0, s0, 411434; GCN2-NEXT: v_mov_b32_e32 v3, s011435; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen11436; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen11437; GCN2-NEXT: s_waitcnt vmcnt(1)11438; GCN2-NEXT: v_xor_b32_e32 v4, s4, v011439; GCN2-NEXT: s_waitcnt vmcnt(0)11440; GCN2-NEXT: v_xor_b32_e32 v5, s5, v111441; GCN2-NEXT: buffer_store_dword v4, v2, s[88:91], 0 offen11442; GCN2-NEXT: buffer_store_dword v5, v3, s[88:91], 0 offen11443; GCN2-NEXT: .LBB79_4: ; %atomicrmw.end11444; GCN2-NEXT: v_mov_b32_e32 v2, s211445; GCN2-NEXT: v_mov_b32_e32 v3, s311446; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]11447; GCN2-NEXT: s_endpgm11448;11449; GFX12-LABEL: atomic_xor_i64_ret:11450; GFX12: ; %bb.0: ; %entry11451; GFX12-NEXT: s_clause 0x111452; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2411453; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x3411454; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base11455; GFX12-NEXT: s_wait_kmcnt 0x011456; GFX12-NEXT: s_cmp_eq_u32 s1, s711457; GFX12-NEXT: s_cselect_b32 s6, -1, 011458; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11459; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s611460; GFX12-NEXT: s_cbranch_vccz .LBB79_211461; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global11462; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s111463; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s511464; GFX12-NEXT: flat_atomic_xor_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV11465; GFX12-NEXT: s_wait_loadcnt_dscnt 0x011466; GFX12-NEXT: global_inv scope:SCOPE_DEV11467; GFX12-NEXT: s_cbranch_execz .LBB79_311468; GFX12-NEXT: s_branch .LBB79_411469; GFX12-NEXT: .LBB79_2:11470; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr111471; GFX12-NEXT: .LBB79_3: ; %atomicrmw.private11472; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 011473; GFX12-NEXT: s_cselect_b32 s0, s0, -111474; GFX12-NEXT: scratch_load_b64 v[0:1], off, s011475; GFX12-NEXT: s_wait_loadcnt 0x011476; GFX12-NEXT: v_xor_b32_e32 v3, s5, v111477; GFX12-NEXT: v_xor_b32_e32 v2, s4, v011478; GFX12-NEXT: scratch_store_b64 off, v[2:3], s011479; GFX12-NEXT: .LBB79_4: ; %atomicrmw.end11480; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s311481; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]11482; GFX12-NEXT: s_endpgm11483entry:11484 %tmp0 = atomicrmw volatile xor ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !011485 store i64 %tmp0, ptr %out211486 ret void11487}11488 11489define amdgpu_kernel void @atomic_xor_i64_addr64(ptr %out, i64 %in, i64 %index) {11490; GCN1-LABEL: atomic_xor_i64_addr64:11491; GCN1: ; %bb.0: ; %entry11492; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD011493; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD111494; GCN1-NEXT: s_mov_b32 s14, -111495; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd11496; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x911497; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f11498; GCN1-NEXT: s_mov_b32 s15, 0xe8f00011499; GCN1-NEXT: s_add_u32 s12, s12, s1111500; GCN1-NEXT: s_addc_u32 s13, s13, 011501; GCN1-NEXT: s_waitcnt lgkmcnt(0)11502; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 311503; GCN1-NEXT: s_add_u32 s0, s0, s411504; GCN1-NEXT: s_addc_u32 s1, s1, s511505; GCN1-NEXT: s_cmp_eq_u32 s1, s811506; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 011507; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]11508; GCN1-NEXT: s_mov_b64 s[4:5], -111509; GCN1-NEXT: s_cbranch_vccnz .LBB80_311510; GCN1-NEXT: ; %bb.1: ; %Flow11511; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]11512; GCN1-NEXT: s_cbranch_vccz .LBB80_411513; GCN1-NEXT: .LBB80_2: ; %atomicrmw.phi11514; GCN1-NEXT: s_endpgm11515; GCN1-NEXT: .LBB80_3: ; %atomicrmw.global11516; GCN1-NEXT: v_mov_b32_e32 v0, s011517; GCN1-NEXT: v_mov_b32_e32 v2, s211518; GCN1-NEXT: v_mov_b32_e32 v1, s111519; GCN1-NEXT: v_mov_b32_e32 v3, s311520; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]11521; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11522; GCN1-NEXT: buffer_wbinvl1_vol11523; GCN1-NEXT: s_cbranch_execnz .LBB80_211524; GCN1-NEXT: .LBB80_4: ; %atomicrmw.private11525; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 011526; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec11527; GCN1-NEXT: s_cselect_b32 s0, s0, -111528; GCN1-NEXT: v_mov_b32_e32 v0, s011529; GCN1-NEXT: s_add_i32 s0, s0, 411530; GCN1-NEXT: v_mov_b32_e32 v1, s011531; GCN1-NEXT: buffer_load_dword v2, v0, s[12:15], 0 offen11532; GCN1-NEXT: buffer_load_dword v3, v1, s[12:15], 0 offen11533; GCN1-NEXT: s_waitcnt vmcnt(1)11534; GCN1-NEXT: v_xor_b32_e32 v2, s2, v211535; GCN1-NEXT: s_waitcnt vmcnt(0)11536; GCN1-NEXT: v_xor_b32_e32 v3, s3, v311537; GCN1-NEXT: buffer_store_dword v2, v0, s[12:15], 0 offen11538; GCN1-NEXT: buffer_store_dword v3, v1, s[12:15], 0 offen11539; GCN1-NEXT: s_endpgm11540;11541; GCN2-LABEL: atomic_xor_i64_addr64:11542; GCN2: ; %bb.0: ; %entry11543; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD011544; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD111545; GCN2-NEXT: s_mov_b32 s90, -111546; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3411547; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2411548; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc11549; GCN2-NEXT: s_mov_b32 s91, 0xe8000011550; GCN2-NEXT: s_add_u32 s88, s88, s1111551; GCN2-NEXT: s_addc_u32 s89, s89, 011552; GCN2-NEXT: s_waitcnt lgkmcnt(0)11553; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 311554; GCN2-NEXT: s_add_u32 s0, s0, s411555; GCN2-NEXT: s_addc_u32 s1, s1, s511556; GCN2-NEXT: s_cmp_eq_u32 s1, s811557; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 011558; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]11559; GCN2-NEXT: s_mov_b64 s[4:5], -111560; GCN2-NEXT: s_cbranch_vccnz .LBB80_311561; GCN2-NEXT: ; %bb.1: ; %Flow11562; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]11563; GCN2-NEXT: s_cbranch_vccz .LBB80_411564; GCN2-NEXT: .LBB80_2: ; %atomicrmw.phi11565; GCN2-NEXT: s_endpgm11566; GCN2-NEXT: .LBB80_3: ; %atomicrmw.global11567; GCN2-NEXT: v_mov_b32_e32 v0, s011568; GCN2-NEXT: v_mov_b32_e32 v2, s211569; GCN2-NEXT: v_mov_b32_e32 v1, s111570; GCN2-NEXT: v_mov_b32_e32 v3, s311571; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]11572; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11573; GCN2-NEXT: buffer_wbinvl1_vol11574; GCN2-NEXT: s_cbranch_execnz .LBB80_211575; GCN2-NEXT: .LBB80_4: ; %atomicrmw.private11576; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 011577; GCN2-NEXT: s_cselect_b32 s0, s0, -111578; GCN2-NEXT: v_mov_b32_e32 v0, s011579; GCN2-NEXT: s_add_i32 s0, s0, 411580; GCN2-NEXT: v_mov_b32_e32 v1, s011581; GCN2-NEXT: buffer_load_dword v2, v0, s[88:91], 0 offen11582; GCN2-NEXT: buffer_load_dword v3, v1, s[88:91], 0 offen11583; GCN2-NEXT: s_waitcnt vmcnt(1)11584; GCN2-NEXT: v_xor_b32_e32 v2, s2, v211585; GCN2-NEXT: s_waitcnt vmcnt(0)11586; GCN2-NEXT: v_xor_b32_e32 v3, s3, v311587; GCN2-NEXT: buffer_store_dword v2, v0, s[88:91], 0 offen11588; GCN2-NEXT: buffer_store_dword v3, v1, s[88:91], 0 offen11589; GCN2-NEXT: s_endpgm11590;11591; GFX12-LABEL: atomic_xor_i64_addr64:11592; GFX12: ; %bb.0: ; %entry11593; GFX12-NEXT: s_clause 0x111594; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x3411595; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2411596; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base11597; GFX12-NEXT: s_wait_kmcnt 0x011598; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 311599; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)11600; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]11601; GFX12-NEXT: s_cmp_eq_u32 s1, s711602; GFX12-NEXT: s_cselect_b32 s4, -1, 011603; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11604; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s411605; GFX12-NEXT: s_mov_b32 s4, -111606; GFX12-NEXT: s_cbranch_vccnz .LBB80_311607; GFX12-NEXT: ; %bb.1: ; %Flow11608; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s411609; GFX12-NEXT: s_cbranch_vccz .LBB80_411610; GFX12-NEXT: .LBB80_2: ; %atomicrmw.phi11611; GFX12-NEXT: s_endpgm11612; GFX12-NEXT: .LBB80_3: ; %atomicrmw.global11613; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s111614; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s311615; GFX12-NEXT: flat_atomic_xor_b64 v[0:1], v[2:3] scope:SCOPE_DEV11616; GFX12-NEXT: s_wait_storecnt_dscnt 0x011617; GFX12-NEXT: global_inv scope:SCOPE_DEV11618; GFX12-NEXT: s_cbranch_execnz .LBB80_211619; GFX12-NEXT: .LBB80_4: ; %atomicrmw.private11620; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 011621; GFX12-NEXT: s_cselect_b32 s0, s0, -111622; GFX12-NEXT: scratch_load_b64 v[0:1], off, s011623; GFX12-NEXT: s_wait_loadcnt 0x011624; GFX12-NEXT: v_xor_b32_e32 v1, s3, v111625; GFX12-NEXT: v_xor_b32_e32 v0, s2, v011626; GFX12-NEXT: scratch_store_b64 off, v[0:1], s011627; GFX12-NEXT: s_endpgm11628entry:11629 %ptr = getelementptr inbounds i64, ptr %out, i64 %index11630 %tmp0 = atomicrmw volatile xor ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !011631 ret void11632}11633 11634define amdgpu_kernel void @atomic_xor_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {11635; GCN1-LABEL: atomic_xor_i64_ret_addr64:11636; GCN1: ; %bb.0: ; %entry11637; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD011638; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD111639; GCN1-NEXT: s_mov_b32 s18, -111640; GCN1-NEXT: s_mov_b32 s19, 0xe8f00011641; GCN1-NEXT: s_add_u32 s16, s16, s1111642; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x911643; GCN1-NEXT: s_load_dword s2, s[4:5], 0x4111644; GCN1-NEXT: s_addc_u32 s17, s17, 011645; GCN1-NEXT: s_waitcnt lgkmcnt(0)11646; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 311647; GCN1-NEXT: s_add_u32 s0, s8, s011648; GCN1-NEXT: s_addc_u32 s1, s9, s111649; GCN1-NEXT: s_cmp_eq_u32 s1, s211650; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 011651; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]11652; GCN1-NEXT: s_cbranch_vccz .LBB81_211653; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global11654; GCN1-NEXT: v_mov_b32_e32 v0, s011655; GCN1-NEXT: v_mov_b32_e32 v2, s1211656; GCN1-NEXT: v_mov_b32_e32 v1, s111657; GCN1-NEXT: v_mov_b32_e32 v3, s1311658; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc11659; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11660; GCN1-NEXT: buffer_wbinvl1_vol11661; GCN1-NEXT: s_cbranch_execz .LBB81_311662; GCN1-NEXT: s_branch .LBB81_411663; GCN1-NEXT: .LBB81_2:11664; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr111665; GCN1-NEXT: .LBB81_3: ; %atomicrmw.private11666; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 011667; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec11668; GCN1-NEXT: s_cselect_b32 s0, s0, -111669; GCN1-NEXT: v_mov_b32_e32 v2, s011670; GCN1-NEXT: s_add_i32 s0, s0, 411671; GCN1-NEXT: v_mov_b32_e32 v3, s011672; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen11673; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen11674; GCN1-NEXT: s_waitcnt vmcnt(1)11675; GCN1-NEXT: v_xor_b32_e32 v4, s12, v011676; GCN1-NEXT: s_waitcnt vmcnt(0)11677; GCN1-NEXT: v_xor_b32_e32 v5, s13, v111678; GCN1-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen11679; GCN1-NEXT: buffer_store_dword v5, v3, s[16:19], 0 offen11680; GCN1-NEXT: .LBB81_4: ; %atomicrmw.end11681; GCN1-NEXT: v_mov_b32_e32 v2, s1011682; GCN1-NEXT: v_mov_b32_e32 v3, s1111683; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]11684; GCN1-NEXT: s_endpgm11685;11686; GCN2-LABEL: atomic_xor_i64_ret_addr64:11687; GCN2: ; %bb.0: ; %entry11688; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD011689; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD111690; GCN2-NEXT: s_mov_b32 s90, -111691; GCN2-NEXT: s_mov_b32 s91, 0xe8000011692; GCN2-NEXT: s_add_u32 s88, s88, s1111693; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2411694; GCN2-NEXT: s_load_dword s2, s[4:5], 0x10411695; GCN2-NEXT: s_addc_u32 s89, s89, 011696; GCN2-NEXT: s_waitcnt lgkmcnt(0)11697; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 311698; GCN2-NEXT: s_add_u32 s0, s8, s011699; GCN2-NEXT: s_addc_u32 s1, s9, s111700; GCN2-NEXT: s_cmp_eq_u32 s1, s211701; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 011702; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]11703; GCN2-NEXT: s_cbranch_vccz .LBB81_211704; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global11705; GCN2-NEXT: v_mov_b32_e32 v0, s011706; GCN2-NEXT: v_mov_b32_e32 v2, s1211707; GCN2-NEXT: v_mov_b32_e32 v1, s111708; GCN2-NEXT: v_mov_b32_e32 v3, s1311709; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc11710; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11711; GCN2-NEXT: buffer_wbinvl1_vol11712; GCN2-NEXT: s_cbranch_execz .LBB81_311713; GCN2-NEXT: s_branch .LBB81_411714; GCN2-NEXT: .LBB81_2:11715; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr111716; GCN2-NEXT: .LBB81_3: ; %atomicrmw.private11717; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 011718; GCN2-NEXT: s_cselect_b32 s0, s0, -111719; GCN2-NEXT: v_mov_b32_e32 v2, s011720; GCN2-NEXT: s_add_i32 s0, s0, 411721; GCN2-NEXT: v_mov_b32_e32 v3, s011722; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen11723; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen11724; GCN2-NEXT: s_waitcnt vmcnt(1)11725; GCN2-NEXT: v_xor_b32_e32 v4, s12, v011726; GCN2-NEXT: s_waitcnt vmcnt(0)11727; GCN2-NEXT: v_xor_b32_e32 v5, s13, v111728; GCN2-NEXT: buffer_store_dword v4, v2, s[88:91], 0 offen11729; GCN2-NEXT: buffer_store_dword v5, v3, s[88:91], 0 offen11730; GCN2-NEXT: .LBB81_4: ; %atomicrmw.end11731; GCN2-NEXT: v_mov_b32_e32 v2, s1011732; GCN2-NEXT: v_mov_b32_e32 v3, s1111733; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]11734; GCN2-NEXT: s_endpgm11735;11736; GFX12-LABEL: atomic_xor_i64_ret_addr64:11737; GFX12: ; %bb.0: ; %entry11738; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x2411739; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base11740; GFX12-NEXT: s_wait_kmcnt 0x011741; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 311742; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)11743; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]11744; GFX12-NEXT: s_cmp_eq_u32 s1, s911745; GFX12-NEXT: s_cselect_b32 s6, -1, 011746; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11747; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s611748; GFX12-NEXT: s_cbranch_vccz .LBB81_211749; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global11750; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s111751; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s511752; GFX12-NEXT: flat_atomic_xor_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV11753; GFX12-NEXT: s_wait_loadcnt_dscnt 0x011754; GFX12-NEXT: global_inv scope:SCOPE_DEV11755; GFX12-NEXT: s_cbranch_execz .LBB81_311756; GFX12-NEXT: s_branch .LBB81_411757; GFX12-NEXT: .LBB81_2:11758; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr111759; GFX12-NEXT: .LBB81_3: ; %atomicrmw.private11760; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 011761; GFX12-NEXT: s_cselect_b32 s0, s0, -111762; GFX12-NEXT: scratch_load_b64 v[0:1], off, s011763; GFX12-NEXT: s_wait_loadcnt 0x011764; GFX12-NEXT: v_xor_b32_e32 v3, s5, v111765; GFX12-NEXT: v_xor_b32_e32 v2, s4, v011766; GFX12-NEXT: scratch_store_b64 off, v[2:3], s011767; GFX12-NEXT: .LBB81_4: ; %atomicrmw.end11768; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s311769; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]11770; GFX12-NEXT: s_endpgm11771entry:11772 %ptr = getelementptr inbounds i64, ptr %out, i64 %index11773 %tmp0 = atomicrmw volatile xor ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !011774 store i64 %tmp0, ptr %out211775 ret void11776}11777 11778define amdgpu_kernel void @atomic_load_i64_offset(ptr %in, ptr %out) {11779; GCN1-LABEL: atomic_load_i64_offset:11780; GCN1: ; %bb.0: ; %entry11781; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x911782; GCN1-NEXT: s_waitcnt lgkmcnt(0)11783; GCN1-NEXT: s_add_u32 s0, s0, 3211784; GCN1-NEXT: s_addc_u32 s1, s1, 011785; GCN1-NEXT: v_mov_b32_e32 v0, s011786; GCN1-NEXT: v_mov_b32_e32 v1, s111787; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc11788; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11789; GCN1-NEXT: buffer_wbinvl1_vol11790; GCN1-NEXT: v_mov_b32_e32 v2, s211791; GCN1-NEXT: v_mov_b32_e32 v3, s311792; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]11793; GCN1-NEXT: s_endpgm11794;11795; GCN2-LABEL: atomic_load_i64_offset:11796; GCN2: ; %bb.0: ; %entry11797; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2411798; GCN2-NEXT: s_waitcnt lgkmcnt(0)11799; GCN2-NEXT: s_add_u32 s0, s0, 3211800; GCN2-NEXT: s_addc_u32 s1, s1, 011801; GCN2-NEXT: v_mov_b32_e32 v0, s011802; GCN2-NEXT: v_mov_b32_e32 v1, s111803; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc11804; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11805; GCN2-NEXT: buffer_wbinvl1_vol11806; GCN2-NEXT: v_mov_b32_e32 v2, s211807; GCN2-NEXT: v_mov_b32_e32 v3, s311808; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]11809; GCN2-NEXT: s_endpgm11810;11811; GFX12-LABEL: atomic_load_i64_offset:11812; GFX12: ; %bb.0: ; %entry11813; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2411814; GFX12-NEXT: s_wait_kmcnt 0x011815; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s111816; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s311817; GFX12-NEXT: flat_load_b64 v[0:1], v[0:1] offset:32 scope:SCOPE_SYS11818; GFX12-NEXT: s_wait_loadcnt_dscnt 0x011819; GFX12-NEXT: global_inv scope:SCOPE_SYS11820; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]11821; GFX12-NEXT: s_endpgm11822entry:11823 %gep = getelementptr inbounds i64, ptr %in, i64 411824 %val = load atomic i64, ptr %gep seq_cst, align 811825 store i64 %val, ptr %out11826 ret void11827}11828 11829define amdgpu_kernel void @atomic_load_i64(ptr %in, ptr %out) {11830; GCN1-LABEL: atomic_load_i64:11831; GCN1: ; %bb.0: ; %entry11832; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x911833; GCN1-NEXT: s_waitcnt lgkmcnt(0)11834; GCN1-NEXT: v_mov_b32_e32 v0, s011835; GCN1-NEXT: v_mov_b32_e32 v1, s111836; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc11837; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11838; GCN1-NEXT: buffer_wbinvl1_vol11839; GCN1-NEXT: v_mov_b32_e32 v2, s211840; GCN1-NEXT: v_mov_b32_e32 v3, s311841; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]11842; GCN1-NEXT: s_endpgm11843;11844; GCN2-LABEL: atomic_load_i64:11845; GCN2: ; %bb.0: ; %entry11846; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2411847; GCN2-NEXT: s_waitcnt lgkmcnt(0)11848; GCN2-NEXT: v_mov_b32_e32 v0, s011849; GCN2-NEXT: v_mov_b32_e32 v1, s111850; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc11851; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11852; GCN2-NEXT: buffer_wbinvl1_vol11853; GCN2-NEXT: v_mov_b32_e32 v2, s211854; GCN2-NEXT: v_mov_b32_e32 v3, s311855; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]11856; GCN2-NEXT: s_endpgm11857;11858; GFX12-LABEL: atomic_load_i64:11859; GFX12: ; %bb.0: ; %entry11860; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2411861; GFX12-NEXT: s_wait_kmcnt 0x011862; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s111863; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s311864; GFX12-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV11865; GFX12-NEXT: s_wait_loadcnt_dscnt 0x011866; GFX12-NEXT: global_inv scope:SCOPE_DEV11867; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]11868; GFX12-NEXT: s_endpgm11869entry:11870 %val = load atomic i64, ptr %in syncscope("agent") seq_cst, align 811871 store i64 %val, ptr %out11872 ret void11873}11874 11875define amdgpu_kernel void @atomic_load_i64_addr64_offset(ptr %in, ptr %out, i64 %index) {11876; GCN1-LABEL: atomic_load_i64_addr64_offset:11877; GCN1: ; %bb.0: ; %entry11878; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd11879; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x911880; GCN1-NEXT: s_waitcnt lgkmcnt(0)11881; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 311882; GCN1-NEXT: s_add_u32 s0, s0, s411883; GCN1-NEXT: s_addc_u32 s1, s1, s511884; GCN1-NEXT: s_add_u32 s0, s0, 3211885; GCN1-NEXT: s_addc_u32 s1, s1, 011886; GCN1-NEXT: v_mov_b32_e32 v0, s011887; GCN1-NEXT: v_mov_b32_e32 v1, s111888; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc11889; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11890; GCN1-NEXT: buffer_wbinvl1_vol11891; GCN1-NEXT: v_mov_b32_e32 v2, s211892; GCN1-NEXT: v_mov_b32_e32 v3, s311893; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]11894; GCN1-NEXT: s_endpgm11895;11896; GCN2-LABEL: atomic_load_i64_addr64_offset:11897; GCN2: ; %bb.0: ; %entry11898; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3411899; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2411900; GCN2-NEXT: s_waitcnt lgkmcnt(0)11901; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 311902; GCN2-NEXT: s_add_u32 s0, s0, s411903; GCN2-NEXT: s_addc_u32 s1, s1, s511904; GCN2-NEXT: s_add_u32 s0, s0, 3211905; GCN2-NEXT: s_addc_u32 s1, s1, 011906; GCN2-NEXT: v_mov_b32_e32 v0, s011907; GCN2-NEXT: v_mov_b32_e32 v1, s111908; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc11909; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11910; GCN2-NEXT: buffer_wbinvl1_vol11911; GCN2-NEXT: v_mov_b32_e32 v2, s211912; GCN2-NEXT: v_mov_b32_e32 v3, s311913; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]11914; GCN2-NEXT: s_endpgm11915;11916; GFX12-LABEL: atomic_load_i64_addr64_offset:11917; GFX12: ; %bb.0: ; %entry11918; GFX12-NEXT: s_clause 0x111919; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x3411920; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2411921; GFX12-NEXT: s_wait_kmcnt 0x011922; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 311923; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s311924; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]11925; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11926; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s111927; GFX12-NEXT: flat_load_b64 v[0:1], v[0:1] offset:32 scope:SCOPE_SYS11928; GFX12-NEXT: s_wait_loadcnt_dscnt 0x011929; GFX12-NEXT: global_inv scope:SCOPE_SYS11930; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]11931; GFX12-NEXT: s_endpgm11932entry:11933 %ptr = getelementptr inbounds i64, ptr %in, i64 %index11934 %gep = getelementptr inbounds i64, ptr %ptr, i64 411935 %val = load atomic i64, ptr %gep seq_cst, align 811936 store i64 %val, ptr %out11937 ret void11938}11939 11940define amdgpu_kernel void @atomic_load_i64_addr64(ptr %in, ptr %out, i64 %index) {11941; GCN1-LABEL: atomic_load_i64_addr64:11942; GCN1: ; %bb.0: ; %entry11943; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd11944; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x911945; GCN1-NEXT: s_waitcnt lgkmcnt(0)11946; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 311947; GCN1-NEXT: s_add_u32 s0, s0, s411948; GCN1-NEXT: s_addc_u32 s1, s1, s511949; GCN1-NEXT: v_mov_b32_e32 v0, s011950; GCN1-NEXT: v_mov_b32_e32 v1, s111951; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc11952; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11953; GCN1-NEXT: buffer_wbinvl1_vol11954; GCN1-NEXT: v_mov_b32_e32 v2, s211955; GCN1-NEXT: v_mov_b32_e32 v3, s311956; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]11957; GCN1-NEXT: s_endpgm11958;11959; GCN2-LABEL: atomic_load_i64_addr64:11960; GCN2: ; %bb.0: ; %entry11961; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3411962; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2411963; GCN2-NEXT: s_waitcnt lgkmcnt(0)11964; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 311965; GCN2-NEXT: s_add_u32 s0, s0, s411966; GCN2-NEXT: s_addc_u32 s1, s1, s511967; GCN2-NEXT: v_mov_b32_e32 v0, s011968; GCN2-NEXT: v_mov_b32_e32 v1, s111969; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc11970; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11971; GCN2-NEXT: buffer_wbinvl1_vol11972; GCN2-NEXT: v_mov_b32_e32 v2, s211973; GCN2-NEXT: v_mov_b32_e32 v3, s311974; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]11975; GCN2-NEXT: s_endpgm11976;11977; GFX12-LABEL: atomic_load_i64_addr64:11978; GFX12: ; %bb.0: ; %entry11979; GFX12-NEXT: s_clause 0x111980; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x3411981; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2411982; GFX12-NEXT: s_wait_kmcnt 0x011983; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 311984; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s311985; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]11986; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11987; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s111988; GFX12-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_SYS11989; GFX12-NEXT: s_wait_loadcnt_dscnt 0x011990; GFX12-NEXT: global_inv scope:SCOPE_SYS11991; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]11992; GFX12-NEXT: s_endpgm11993entry:11994 %ptr = getelementptr inbounds i64, ptr %in, i64 %index11995 %val = load atomic i64, ptr %ptr seq_cst, align 811996 store i64 %val, ptr %out11997 ret void11998}11999 12000define amdgpu_kernel void @atomic_store_i64_offset(i64 %in, ptr %out) {12001; GCN1-LABEL: atomic_store_i64_offset:12002; GCN1: ; %bb.0: ; %entry12003; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x912004; GCN1-NEXT: s_waitcnt lgkmcnt(0)12005; GCN1-NEXT: v_mov_b32_e32 v0, s012006; GCN1-NEXT: s_add_u32 s0, s2, 3212007; GCN1-NEXT: v_mov_b32_e32 v1, s112008; GCN1-NEXT: s_addc_u32 s1, s3, 012009; GCN1-NEXT: v_mov_b32_e32 v3, s112010; GCN1-NEXT: v_mov_b32_e32 v2, s012011; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]12012; GCN1-NEXT: s_endpgm12013;12014; GCN2-LABEL: atomic_store_i64_offset:12015; GCN2: ; %bb.0: ; %entry12016; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2412017; GCN2-NEXT: s_waitcnt lgkmcnt(0)12018; GCN2-NEXT: v_mov_b32_e32 v0, s012019; GCN2-NEXT: s_add_u32 s0, s2, 3212020; GCN2-NEXT: v_mov_b32_e32 v1, s112021; GCN2-NEXT: s_addc_u32 s1, s3, 012022; GCN2-NEXT: v_mov_b32_e32 v3, s112023; GCN2-NEXT: v_mov_b32_e32 v2, s012024; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]12025; GCN2-NEXT: s_endpgm12026;12027; GFX12-LABEL: atomic_store_i64_offset:12028; GFX12: ; %bb.0: ; %entry12029; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2412030; GFX12-NEXT: s_wait_kmcnt 0x012031; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s112032; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s312033; GFX12-NEXT: global_wb scope:SCOPE_SYS12034; GFX12-NEXT: s_wait_storecnt 0x012035; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1] offset:32 scope:SCOPE_SYS12036; GFX12-NEXT: s_endpgm12037entry:12038 %gep = getelementptr inbounds i64, ptr %out, i64 412039 store atomic i64 %in, ptr %gep seq_cst, align 812040 ret void12041}12042 12043define amdgpu_kernel void @atomic_store_i64(i64 %in, ptr %out) {12044; GCN1-LABEL: atomic_store_i64:12045; GCN1: ; %bb.0: ; %entry12046; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x912047; GCN1-NEXT: s_waitcnt lgkmcnt(0)12048; GCN1-NEXT: v_mov_b32_e32 v0, s012049; GCN1-NEXT: v_mov_b32_e32 v1, s112050; GCN1-NEXT: v_mov_b32_e32 v2, s212051; GCN1-NEXT: v_mov_b32_e32 v3, s312052; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]12053; GCN1-NEXT: s_endpgm12054;12055; GCN2-LABEL: atomic_store_i64:12056; GCN2: ; %bb.0: ; %entry12057; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2412058; GCN2-NEXT: s_waitcnt lgkmcnt(0)12059; GCN2-NEXT: v_mov_b32_e32 v0, s012060; GCN2-NEXT: v_mov_b32_e32 v1, s112061; GCN2-NEXT: v_mov_b32_e32 v2, s212062; GCN2-NEXT: v_mov_b32_e32 v3, s312063; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]12064; GCN2-NEXT: s_endpgm12065;12066; GFX12-LABEL: atomic_store_i64:12067; GFX12: ; %bb.0: ; %entry12068; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2412069; GFX12-NEXT: s_wait_kmcnt 0x012070; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s112071; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s312072; GFX12-NEXT: global_wb scope:SCOPE_SYS12073; GFX12-NEXT: s_wait_storecnt 0x012074; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_SYS12075; GFX12-NEXT: s_endpgm12076entry:12077 store atomic i64 %in, ptr %out seq_cst, align 812078 ret void12079}12080 12081define amdgpu_kernel void @atomic_store_i64_addr64_offset(i64 %in, ptr %out, i64 %index) {12082; GCN1-LABEL: atomic_store_i64_addr64_offset:12083; GCN1: ; %bb.0: ; %entry12084; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x912085; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd12086; GCN1-NEXT: s_waitcnt lgkmcnt(0)12087; GCN1-NEXT: v_mov_b32_e32 v0, s012088; GCN1-NEXT: v_mov_b32_e32 v1, s112089; GCN1-NEXT: s_lshl_b64 s[0:1], s[4:5], 312090; GCN1-NEXT: s_add_u32 s0, s2, s012091; GCN1-NEXT: s_addc_u32 s1, s3, s112092; GCN1-NEXT: s_add_u32 s0, s0, 3212093; GCN1-NEXT: s_addc_u32 s1, s1, 012094; GCN1-NEXT: v_mov_b32_e32 v3, s112095; GCN1-NEXT: v_mov_b32_e32 v2, s012096; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]12097; GCN1-NEXT: s_endpgm12098;12099; GCN2-LABEL: atomic_store_i64_addr64_offset:12100; GCN2: ; %bb.0: ; %entry12101; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2412102; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x3412103; GCN2-NEXT: s_waitcnt lgkmcnt(0)12104; GCN2-NEXT: v_mov_b32_e32 v0, s012105; GCN2-NEXT: v_mov_b32_e32 v1, s112106; GCN2-NEXT: s_lshl_b64 s[0:1], s[4:5], 312107; GCN2-NEXT: s_add_u32 s0, s2, s012108; GCN2-NEXT: s_addc_u32 s1, s3, s112109; GCN2-NEXT: s_add_u32 s0, s0, 3212110; GCN2-NEXT: s_addc_u32 s1, s1, 012111; GCN2-NEXT: v_mov_b32_e32 v3, s112112; GCN2-NEXT: v_mov_b32_e32 v2, s012113; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]12114; GCN2-NEXT: s_endpgm12115;12116; GFX12-LABEL: atomic_store_i64_addr64_offset:12117; GFX12: ; %bb.0: ; %entry12118; GFX12-NEXT: s_clause 0x112119; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x3412120; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2412121; GFX12-NEXT: s_wait_kmcnt 0x012122; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 312123; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s112124; GFX12-NEXT: s_add_nc_u64 s[2:3], s[2:3], s[4:5]12125; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12126; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s312127; GFX12-NEXT: global_wb scope:SCOPE_SYS12128; GFX12-NEXT: s_wait_storecnt 0x012129; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1] offset:32 scope:SCOPE_SYS12130; GFX12-NEXT: s_endpgm12131entry:12132 %ptr = getelementptr inbounds i64, ptr %out, i64 %index12133 %gep = getelementptr inbounds i64, ptr %ptr, i64 412134 store atomic i64 %in, ptr %gep seq_cst, align 812135 ret void12136}12137 12138define amdgpu_kernel void @atomic_store_i64_addr64(i64 %in, ptr %out, i64 %index) {12139; GCN1-LABEL: atomic_store_i64_addr64:12140; GCN1: ; %bb.0: ; %entry12141; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x912142; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd12143; GCN1-NEXT: s_waitcnt lgkmcnt(0)12144; GCN1-NEXT: v_mov_b32_e32 v0, s012145; GCN1-NEXT: v_mov_b32_e32 v1, s112146; GCN1-NEXT: s_lshl_b64 s[0:1], s[4:5], 312147; GCN1-NEXT: s_add_u32 s0, s2, s012148; GCN1-NEXT: s_addc_u32 s1, s3, s112149; GCN1-NEXT: v_mov_b32_e32 v3, s112150; GCN1-NEXT: v_mov_b32_e32 v2, s012151; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]12152; GCN1-NEXT: s_endpgm12153;12154; GCN2-LABEL: atomic_store_i64_addr64:12155; GCN2: ; %bb.0: ; %entry12156; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2412157; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x3412158; GCN2-NEXT: s_waitcnt lgkmcnt(0)12159; GCN2-NEXT: v_mov_b32_e32 v0, s012160; GCN2-NEXT: v_mov_b32_e32 v1, s112161; GCN2-NEXT: s_lshl_b64 s[0:1], s[4:5], 312162; GCN2-NEXT: s_add_u32 s0, s2, s012163; GCN2-NEXT: s_addc_u32 s1, s3, s112164; GCN2-NEXT: v_mov_b32_e32 v3, s112165; GCN2-NEXT: v_mov_b32_e32 v2, s012166; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]12167; GCN2-NEXT: s_endpgm12168;12169; GFX12-LABEL: atomic_store_i64_addr64:12170; GFX12: ; %bb.0: ; %entry12171; GFX12-NEXT: s_clause 0x112172; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x3412173; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2412174; GFX12-NEXT: s_wait_kmcnt 0x012175; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 312176; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s112177; GFX12-NEXT: s_add_nc_u64 s[2:3], s[2:3], s[4:5]12178; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12179; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s312180; GFX12-NEXT: global_wb scope:SCOPE_SYS12181; GFX12-NEXT: s_wait_storecnt 0x012182; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_SYS12183; GFX12-NEXT: s_endpgm12184entry:12185 %ptr = getelementptr inbounds i64, ptr %out, i64 %index12186 store atomic i64 %in, ptr %ptr seq_cst, align 812187 ret void12188}12189 12190define amdgpu_kernel void @atomic_cmpxchg_i64_offset(ptr %out, i64 %in, i64 %old) {12191; GCN1-LABEL: atomic_cmpxchg_i64_offset:12192; GCN1: ; %bb.0: ; %entry12193; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD012194; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD112195; GCN1-NEXT: s_mov_b32 s14, -112196; GCN1-NEXT: s_mov_b32 s15, 0xe8f00012197; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x912198; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f12199; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd12200; GCN1-NEXT: s_add_u32 s12, s12, s1112201; GCN1-NEXT: s_addc_u32 s13, s13, 012202; GCN1-NEXT: s_waitcnt lgkmcnt(0)12203; GCN1-NEXT: s_add_u32 s0, s0, 3212204; GCN1-NEXT: s_addc_u32 s1, s1, 012205; GCN1-NEXT: s_cmp_eq_u32 s1, s612206; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 012207; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]12208; GCN1-NEXT: s_mov_b64 s[6:7], -112209; GCN1-NEXT: s_cbranch_vccnz .LBB90_312210; GCN1-NEXT: ; %bb.1: ; %Flow12211; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]12212; GCN1-NEXT: s_cbranch_vccz .LBB90_412213; GCN1-NEXT: .LBB90_2: ; %atomicrmw.phi12214; GCN1-NEXT: s_endpgm12215; GCN1-NEXT: .LBB90_3: ; %atomicrmw.global12216; GCN1-NEXT: v_mov_b32_e32 v5, s112217; GCN1-NEXT: v_mov_b32_e32 v0, s212218; GCN1-NEXT: v_mov_b32_e32 v1, s312219; GCN1-NEXT: v_mov_b32_e32 v2, s412220; GCN1-NEXT: v_mov_b32_e32 v3, s512221; GCN1-NEXT: v_mov_b32_e32 v4, s012222; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:3]12223; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12224; GCN1-NEXT: buffer_wbinvl1_vol12225; GCN1-NEXT: s_cbranch_execnz .LBB90_212226; GCN1-NEXT: .LBB90_4: ; %atomicrmw.private12227; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 012228; GCN1-NEXT: v_mov_b32_e32 v5, s212229; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec12230; GCN1-NEXT: s_cselect_b32 s0, s0, -112231; GCN1-NEXT: v_mov_b32_e32 v2, s012232; GCN1-NEXT: s_add_i32 s0, s0, 412233; GCN1-NEXT: v_mov_b32_e32 v3, s012234; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen12235; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen12236; GCN1-NEXT: v_mov_b32_e32 v4, s312237; GCN1-NEXT: s_waitcnt vmcnt(0)12238; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]12239; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc12240; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc12241; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen12242; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen12243; GCN1-NEXT: s_endpgm12244;12245; GCN2-LABEL: atomic_cmpxchg_i64_offset:12246; GCN2: ; %bb.0: ; %entry12247; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD012248; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD112249; GCN2-NEXT: s_mov_b32 s90, -112250; GCN2-NEXT: s_mov_b32 s91, 0xe8000012251; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2412252; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc12253; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x3412254; GCN2-NEXT: s_add_u32 s88, s88, s1112255; GCN2-NEXT: s_addc_u32 s89, s89, 012256; GCN2-NEXT: s_waitcnt lgkmcnt(0)12257; GCN2-NEXT: s_add_u32 s0, s0, 3212258; GCN2-NEXT: s_addc_u32 s1, s1, 012259; GCN2-NEXT: s_cmp_eq_u32 s1, s612260; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 012261; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]12262; GCN2-NEXT: s_mov_b64 s[6:7], -112263; GCN2-NEXT: s_cbranch_vccnz .LBB90_312264; GCN2-NEXT: ; %bb.1: ; %Flow12265; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]12266; GCN2-NEXT: s_cbranch_vccz .LBB90_412267; GCN2-NEXT: .LBB90_2: ; %atomicrmw.phi12268; GCN2-NEXT: s_endpgm12269; GCN2-NEXT: .LBB90_3: ; %atomicrmw.global12270; GCN2-NEXT: v_mov_b32_e32 v5, s112271; GCN2-NEXT: v_mov_b32_e32 v0, s212272; GCN2-NEXT: v_mov_b32_e32 v1, s312273; GCN2-NEXT: v_mov_b32_e32 v2, s412274; GCN2-NEXT: v_mov_b32_e32 v3, s512275; GCN2-NEXT: v_mov_b32_e32 v4, s012276; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:3]12277; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12278; GCN2-NEXT: buffer_wbinvl1_vol12279; GCN2-NEXT: s_cbranch_execnz .LBB90_212280; GCN2-NEXT: .LBB90_4: ; %atomicrmw.private12281; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 012282; GCN2-NEXT: s_cselect_b32 s0, s0, -112283; GCN2-NEXT: v_mov_b32_e32 v2, s012284; GCN2-NEXT: s_add_i32 s0, s0, 412285; GCN2-NEXT: v_mov_b32_e32 v3, s012286; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen12287; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen12288; GCN2-NEXT: v_mov_b32_e32 v5, s212289; GCN2-NEXT: v_mov_b32_e32 v4, s312290; GCN2-NEXT: s_waitcnt vmcnt(0)12291; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]12292; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc12293; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc12294; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen12295; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen12296; GCN2-NEXT: s_endpgm12297;12298; GFX12-LABEL: atomic_cmpxchg_i64_offset:12299; GFX12: ; %bb.0: ; %entry12300; GFX12-NEXT: s_clause 0x112301; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2412302; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x3412303; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base12304; GFX12-NEXT: s_wait_kmcnt 0x012305; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 3212306; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)12307; GFX12-NEXT: s_cmp_eq_u32 s1, s712308; GFX12-NEXT: s_cselect_b32 s6, -1, 012309; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s612310; GFX12-NEXT: s_mov_b32 s6, -112311; GFX12-NEXT: s_cbranch_vccnz .LBB90_312312; GFX12-NEXT: ; %bb.1: ; %Flow12313; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s612314; GFX12-NEXT: s_cbranch_vccz .LBB90_412315; GFX12-NEXT: .LBB90_2: ; %atomicrmw.phi12316; GFX12-NEXT: s_endpgm12317; GFX12-NEXT: .LBB90_3: ; %atomicrmw.global12318; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s312319; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s512320; GFX12-NEXT: v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s012321; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:3] scope:SCOPE_DEV12322; GFX12-NEXT: s_wait_storecnt_dscnt 0x012323; GFX12-NEXT: global_inv scope:SCOPE_DEV12324; GFX12-NEXT: s_cbranch_execnz .LBB90_212325; GFX12-NEXT: .LBB90_4: ; %atomicrmw.private12326; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 012327; GFX12-NEXT: s_cselect_b32 s0, s0, -112328; GFX12-NEXT: scratch_load_b64 v[0:1], off, s012329; GFX12-NEXT: s_wait_loadcnt 0x012330; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]12331; GFX12-NEXT: v_cndmask_b32_e64 v1, v1, s3, vcc_lo12332; GFX12-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo12333; GFX12-NEXT: scratch_store_b64 off, v[0:1], s012334; GFX12-NEXT: s_endpgm12335entry:12336 %gep = getelementptr inbounds i64, ptr %out, i64 412337 %val = cmpxchg volatile ptr %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst12338 ret void12339}12340 12341define amdgpu_kernel void @atomic_cmpxchg_i64_soffset(ptr %out, i64 %in, i64 %old) {12342; GCN1-LABEL: atomic_cmpxchg_i64_soffset:12343; GCN1: ; %bb.0: ; %entry12344; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD012345; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD112346; GCN1-NEXT: s_mov_b32 s14, -112347; GCN1-NEXT: s_mov_b32 s15, 0xe8f00012348; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x912349; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f12350; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd12351; GCN1-NEXT: s_add_u32 s12, s12, s1112352; GCN1-NEXT: s_addc_u32 s13, s13, 012353; GCN1-NEXT: s_waitcnt lgkmcnt(0)12354; GCN1-NEXT: s_add_u32 s0, s0, 0x1194012355; GCN1-NEXT: s_addc_u32 s1, s1, 012356; GCN1-NEXT: s_cmp_eq_u32 s1, s612357; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 012358; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]12359; GCN1-NEXT: s_mov_b64 s[6:7], -112360; GCN1-NEXT: s_cbranch_vccnz .LBB91_312361; GCN1-NEXT: ; %bb.1: ; %Flow12362; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]12363; GCN1-NEXT: s_cbranch_vccz .LBB91_412364; GCN1-NEXT: .LBB91_2: ; %atomicrmw.phi12365; GCN1-NEXT: s_endpgm12366; GCN1-NEXT: .LBB91_3: ; %atomicrmw.global12367; GCN1-NEXT: v_mov_b32_e32 v5, s112368; GCN1-NEXT: v_mov_b32_e32 v0, s212369; GCN1-NEXT: v_mov_b32_e32 v1, s312370; GCN1-NEXT: v_mov_b32_e32 v2, s412371; GCN1-NEXT: v_mov_b32_e32 v3, s512372; GCN1-NEXT: v_mov_b32_e32 v4, s012373; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:3]12374; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12375; GCN1-NEXT: buffer_wbinvl1_vol12376; GCN1-NEXT: s_cbranch_execnz .LBB91_212377; GCN1-NEXT: .LBB91_4: ; %atomicrmw.private12378; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 012379; GCN1-NEXT: v_mov_b32_e32 v5, s212380; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec12381; GCN1-NEXT: s_cselect_b32 s0, s0, -112382; GCN1-NEXT: v_mov_b32_e32 v2, s012383; GCN1-NEXT: s_add_i32 s0, s0, 412384; GCN1-NEXT: v_mov_b32_e32 v3, s012385; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen12386; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen12387; GCN1-NEXT: v_mov_b32_e32 v4, s312388; GCN1-NEXT: s_waitcnt vmcnt(0)12389; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]12390; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc12391; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc12392; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen12393; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen12394; GCN1-NEXT: s_endpgm12395;12396; GCN2-LABEL: atomic_cmpxchg_i64_soffset:12397; GCN2: ; %bb.0: ; %entry12398; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD012399; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD112400; GCN2-NEXT: s_mov_b32 s90, -112401; GCN2-NEXT: s_mov_b32 s91, 0xe8000012402; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2412403; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc12404; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x3412405; GCN2-NEXT: s_add_u32 s88, s88, s1112406; GCN2-NEXT: s_addc_u32 s89, s89, 012407; GCN2-NEXT: s_waitcnt lgkmcnt(0)12408; GCN2-NEXT: s_add_u32 s0, s0, 0x1194012409; GCN2-NEXT: s_addc_u32 s1, s1, 012410; GCN2-NEXT: s_cmp_eq_u32 s1, s612411; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 012412; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]12413; GCN2-NEXT: s_mov_b64 s[6:7], -112414; GCN2-NEXT: s_cbranch_vccnz .LBB91_312415; GCN2-NEXT: ; %bb.1: ; %Flow12416; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]12417; GCN2-NEXT: s_cbranch_vccz .LBB91_412418; GCN2-NEXT: .LBB91_2: ; %atomicrmw.phi12419; GCN2-NEXT: s_endpgm12420; GCN2-NEXT: .LBB91_3: ; %atomicrmw.global12421; GCN2-NEXT: v_mov_b32_e32 v5, s112422; GCN2-NEXT: v_mov_b32_e32 v0, s212423; GCN2-NEXT: v_mov_b32_e32 v1, s312424; GCN2-NEXT: v_mov_b32_e32 v2, s412425; GCN2-NEXT: v_mov_b32_e32 v3, s512426; GCN2-NEXT: v_mov_b32_e32 v4, s012427; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:3]12428; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12429; GCN2-NEXT: buffer_wbinvl1_vol12430; GCN2-NEXT: s_cbranch_execnz .LBB91_212431; GCN2-NEXT: .LBB91_4: ; %atomicrmw.private12432; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 012433; GCN2-NEXT: s_cselect_b32 s0, s0, -112434; GCN2-NEXT: v_mov_b32_e32 v2, s012435; GCN2-NEXT: s_add_i32 s0, s0, 412436; GCN2-NEXT: v_mov_b32_e32 v3, s012437; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen12438; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen12439; GCN2-NEXT: v_mov_b32_e32 v5, s212440; GCN2-NEXT: v_mov_b32_e32 v4, s312441; GCN2-NEXT: s_waitcnt vmcnt(0)12442; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]12443; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc12444; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc12445; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen12446; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen12447; GCN2-NEXT: s_endpgm12448;12449; GFX12-LABEL: atomic_cmpxchg_i64_soffset:12450; GFX12: ; %bb.0: ; %entry12451; GFX12-NEXT: s_clause 0x112452; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2412453; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x3412454; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base12455; GFX12-NEXT: s_wait_kmcnt 0x012456; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 0x1194012457; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)12458; GFX12-NEXT: s_cmp_eq_u32 s1, s712459; GFX12-NEXT: s_cselect_b32 s6, -1, 012460; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s612461; GFX12-NEXT: s_mov_b32 s6, -112462; GFX12-NEXT: s_cbranch_vccnz .LBB91_312463; GFX12-NEXT: ; %bb.1: ; %Flow12464; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s612465; GFX12-NEXT: s_cbranch_vccz .LBB91_412466; GFX12-NEXT: .LBB91_2: ; %atomicrmw.phi12467; GFX12-NEXT: s_endpgm12468; GFX12-NEXT: .LBB91_3: ; %atomicrmw.global12469; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s312470; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s512471; GFX12-NEXT: v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s012472; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:3] scope:SCOPE_DEV12473; GFX12-NEXT: s_wait_storecnt_dscnt 0x012474; GFX12-NEXT: global_inv scope:SCOPE_DEV12475; GFX12-NEXT: s_cbranch_execnz .LBB91_212476; GFX12-NEXT: .LBB91_4: ; %atomicrmw.private12477; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 012478; GFX12-NEXT: s_cselect_b32 s0, s0, -112479; GFX12-NEXT: scratch_load_b64 v[0:1], off, s012480; GFX12-NEXT: s_wait_loadcnt 0x012481; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]12482; GFX12-NEXT: v_cndmask_b32_e64 v1, v1, s3, vcc_lo12483; GFX12-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo12484; GFX12-NEXT: scratch_store_b64 off, v[0:1], s012485; GFX12-NEXT: s_endpgm12486entry:12487 %gep = getelementptr inbounds i64, ptr %out, i64 900012488 %val = cmpxchg volatile ptr %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst12489 ret void12490}12491 12492define amdgpu_kernel void @atomic_cmpxchg_i64_ret_offset(ptr %out, ptr %out2, i64 %in, i64 %old) {12493; GCN1-LABEL: atomic_cmpxchg_i64_ret_offset:12494; GCN1: ; %bb.0: ; %entry12495; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD012496; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD112497; GCN1-NEXT: s_mov_b32 s18, -112498; GCN1-NEXT: s_mov_b32 s19, 0xe8f00012499; GCN1-NEXT: s_add_u32 s16, s16, s1112500; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x912501; GCN1-NEXT: s_load_dword s2, s[4:5], 0x4112502; GCN1-NEXT: s_addc_u32 s17, s17, 012503; GCN1-NEXT: s_waitcnt lgkmcnt(0)12504; GCN1-NEXT: s_add_u32 s0, s8, 3212505; GCN1-NEXT: s_addc_u32 s1, s9, 012506; GCN1-NEXT: s_cmp_eq_u32 s1, s212507; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 012508; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]12509; GCN1-NEXT: s_cbranch_vccz .LBB92_212510; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global12511; GCN1-NEXT: v_mov_b32_e32 v5, s112512; GCN1-NEXT: v_mov_b32_e32 v0, s1212513; GCN1-NEXT: v_mov_b32_e32 v1, s1312514; GCN1-NEXT: v_mov_b32_e32 v2, s1412515; GCN1-NEXT: v_mov_b32_e32 v3, s1512516; GCN1-NEXT: v_mov_b32_e32 v4, s012517; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc12518; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12519; GCN1-NEXT: buffer_wbinvl1_vol12520; GCN1-NEXT: s_cbranch_execz .LBB92_312521; GCN1-NEXT: s_branch .LBB92_412522; GCN1-NEXT: .LBB92_2:12523; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr112524; GCN1-NEXT: .LBB92_3: ; %atomicrmw.private12525; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 012526; GCN1-NEXT: v_mov_b32_e32 v5, s1212527; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec12528; GCN1-NEXT: s_cselect_b32 s0, s0, -112529; GCN1-NEXT: v_mov_b32_e32 v2, s012530; GCN1-NEXT: s_add_i32 s0, s0, 412531; GCN1-NEXT: v_mov_b32_e32 v3, s012532; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen12533; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen12534; GCN1-NEXT: v_mov_b32_e32 v4, s1312535; GCN1-NEXT: s_waitcnt vmcnt(0)12536; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]12537; GCN1-NEXT: v_cndmask_b32_e32 v5, v0, v5, vcc12538; GCN1-NEXT: v_cndmask_b32_e32 v4, v1, v4, vcc12539; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen12540; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen12541; GCN1-NEXT: .LBB92_4: ; %atomicrmw.end12542; GCN1-NEXT: v_mov_b32_e32 v2, s1012543; GCN1-NEXT: v_mov_b32_e32 v3, s1112544; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]12545; GCN1-NEXT: s_endpgm12546;12547; GCN2-LABEL: atomic_cmpxchg_i64_ret_offset:12548; GCN2: ; %bb.0: ; %entry12549; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD012550; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD112551; GCN2-NEXT: s_mov_b32 s90, -112552; GCN2-NEXT: s_mov_b32 s91, 0xe8000012553; GCN2-NEXT: s_add_u32 s88, s88, s1112554; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2412555; GCN2-NEXT: s_load_dword s2, s[4:5], 0x10412556; GCN2-NEXT: s_addc_u32 s89, s89, 012557; GCN2-NEXT: s_waitcnt lgkmcnt(0)12558; GCN2-NEXT: s_add_u32 s0, s8, 3212559; GCN2-NEXT: s_addc_u32 s1, s9, 012560; GCN2-NEXT: s_cmp_eq_u32 s1, s212561; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 012562; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]12563; GCN2-NEXT: s_cbranch_vccz .LBB92_212564; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global12565; GCN2-NEXT: v_mov_b32_e32 v5, s112566; GCN2-NEXT: v_mov_b32_e32 v0, s1212567; GCN2-NEXT: v_mov_b32_e32 v1, s1312568; GCN2-NEXT: v_mov_b32_e32 v2, s1412569; GCN2-NEXT: v_mov_b32_e32 v3, s1512570; GCN2-NEXT: v_mov_b32_e32 v4, s012571; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc12572; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12573; GCN2-NEXT: buffer_wbinvl1_vol12574; GCN2-NEXT: s_cbranch_execz .LBB92_312575; GCN2-NEXT: s_branch .LBB92_412576; GCN2-NEXT: .LBB92_2:12577; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr112578; GCN2-NEXT: .LBB92_3: ; %atomicrmw.private12579; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 012580; GCN2-NEXT: s_cselect_b32 s0, s0, -112581; GCN2-NEXT: v_mov_b32_e32 v2, s012582; GCN2-NEXT: s_add_i32 s0, s0, 412583; GCN2-NEXT: v_mov_b32_e32 v3, s012584; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen12585; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen12586; GCN2-NEXT: v_mov_b32_e32 v5, s1212587; GCN2-NEXT: v_mov_b32_e32 v4, s1312588; GCN2-NEXT: s_waitcnt vmcnt(0)12589; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]12590; GCN2-NEXT: v_cndmask_b32_e32 v5, v0, v5, vcc12591; GCN2-NEXT: v_cndmask_b32_e32 v4, v1, v4, vcc12592; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen12593; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen12594; GCN2-NEXT: .LBB92_4: ; %atomicrmw.end12595; GCN2-NEXT: v_mov_b32_e32 v2, s1012596; GCN2-NEXT: v_mov_b32_e32 v3, s1112597; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]12598; GCN2-NEXT: s_endpgm12599;12600; GFX12-LABEL: atomic_cmpxchg_i64_ret_offset:12601; GFX12: ; %bb.0: ; %entry12602; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x2412603; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base12604; GFX12-NEXT: s_wait_kmcnt 0x012605; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 3212606; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)12607; GFX12-NEXT: s_cmp_eq_u32 s1, s912608; GFX12-NEXT: s_cselect_b32 s8, -1, 012609; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s812610; GFX12-NEXT: s_cbranch_vccz .LBB92_212611; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global12612; GFX12-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s512613; GFX12-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s712614; GFX12-NEXT: v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s012615; GFX12-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[0:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV12616; GFX12-NEXT: s_wait_loadcnt_dscnt 0x012617; GFX12-NEXT: global_inv scope:SCOPE_DEV12618; GFX12-NEXT: s_cbranch_execz .LBB92_312619; GFX12-NEXT: s_branch .LBB92_412620; GFX12-NEXT: .LBB92_2:12621; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr112622; GFX12-NEXT: .LBB92_3: ; %atomicrmw.private12623; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 012624; GFX12-NEXT: s_cselect_b32 s0, s0, -112625; GFX12-NEXT: scratch_load_b64 v[0:1], off, s012626; GFX12-NEXT: s_wait_loadcnt 0x012627; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[6:7], v[0:1]12628; GFX12-NEXT: v_cndmask_b32_e64 v3, v1, s5, vcc_lo12629; GFX12-NEXT: v_cndmask_b32_e64 v2, v0, s4, vcc_lo12630; GFX12-NEXT: scratch_store_b64 off, v[2:3], s012631; GFX12-NEXT: .LBB92_4: ; %atomicrmw.end12632; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s312633; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]12634; GFX12-NEXT: s_endpgm12635entry:12636 %gep = getelementptr inbounds i64, ptr %out, i64 412637 %val = cmpxchg volatile ptr %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst12638 %extract0 = extractvalue { i64, i1 } %val, 012639 store i64 %extract0, ptr %out212640 ret void12641}12642 12643define amdgpu_kernel void @atomic_cmpxchg_i64_addr64_offset(ptr %out, i64 %in, i64 %index, i64 %old) {12644; GCN1-LABEL: atomic_cmpxchg_i64_addr64_offset:12645; GCN1: ; %bb.0: ; %entry12646; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD012647; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD112648; GCN1-NEXT: s_mov_b32 s18, -112649; GCN1-NEXT: s_mov_b32 s19, 0xe8f00012650; GCN1-NEXT: s_add_u32 s16, s16, s1112651; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x912652; GCN1-NEXT: s_load_dword s2, s[4:5], 0x4112653; GCN1-NEXT: s_addc_u32 s17, s17, 012654; GCN1-NEXT: s_waitcnt lgkmcnt(0)12655; GCN1-NEXT: s_lshl_b64 s[0:1], s[12:13], 312656; GCN1-NEXT: s_add_u32 s0, s8, s012657; GCN1-NEXT: s_addc_u32 s1, s9, s112658; GCN1-NEXT: s_add_u32 s0, s0, 3212659; GCN1-NEXT: s_addc_u32 s1, s1, 012660; GCN1-NEXT: s_cmp_eq_u32 s1, s212661; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 012662; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]12663; GCN1-NEXT: s_mov_b64 s[2:3], -112664; GCN1-NEXT: s_cbranch_vccnz .LBB93_312665; GCN1-NEXT: ; %bb.1: ; %Flow12666; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]12667; GCN1-NEXT: s_cbranch_vccz .LBB93_412668; GCN1-NEXT: .LBB93_2: ; %atomicrmw.phi12669; GCN1-NEXT: s_endpgm12670; GCN1-NEXT: .LBB93_3: ; %atomicrmw.global12671; GCN1-NEXT: v_mov_b32_e32 v5, s112672; GCN1-NEXT: v_mov_b32_e32 v0, s1012673; GCN1-NEXT: v_mov_b32_e32 v1, s1112674; GCN1-NEXT: v_mov_b32_e32 v2, s1412675; GCN1-NEXT: v_mov_b32_e32 v3, s1512676; GCN1-NEXT: v_mov_b32_e32 v4, s012677; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:3]12678; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12679; GCN1-NEXT: buffer_wbinvl1_vol12680; GCN1-NEXT: s_cbranch_execnz .LBB93_212681; GCN1-NEXT: .LBB93_4: ; %atomicrmw.private12682; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 012683; GCN1-NEXT: v_mov_b32_e32 v5, s1012684; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec12685; GCN1-NEXT: s_cselect_b32 s0, s0, -112686; GCN1-NEXT: v_mov_b32_e32 v2, s012687; GCN1-NEXT: s_add_i32 s0, s0, 412688; GCN1-NEXT: v_mov_b32_e32 v3, s012689; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen12690; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen12691; GCN1-NEXT: v_mov_b32_e32 v4, s1112692; GCN1-NEXT: s_waitcnt vmcnt(0)12693; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]12694; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc12695; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc12696; GCN1-NEXT: buffer_store_dword v0, v2, s[16:19], 0 offen12697; GCN1-NEXT: buffer_store_dword v1, v3, s[16:19], 0 offen12698; GCN1-NEXT: s_endpgm12699;12700; GCN2-LABEL: atomic_cmpxchg_i64_addr64_offset:12701; GCN2: ; %bb.0: ; %entry12702; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD012703; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD112704; GCN2-NEXT: s_mov_b32 s90, -112705; GCN2-NEXT: s_mov_b32 s91, 0xe8000012706; GCN2-NEXT: s_add_u32 s88, s88, s1112707; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2412708; GCN2-NEXT: s_load_dword s2, s[4:5], 0x10412709; GCN2-NEXT: s_addc_u32 s89, s89, 012710; GCN2-NEXT: s_waitcnt lgkmcnt(0)12711; GCN2-NEXT: s_lshl_b64 s[0:1], s[12:13], 312712; GCN2-NEXT: s_add_u32 s0, s8, s012713; GCN2-NEXT: s_addc_u32 s1, s9, s112714; GCN2-NEXT: s_add_u32 s0, s0, 3212715; GCN2-NEXT: s_addc_u32 s1, s1, 012716; GCN2-NEXT: s_cmp_eq_u32 s1, s212717; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 012718; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]12719; GCN2-NEXT: s_mov_b64 s[2:3], -112720; GCN2-NEXT: s_cbranch_vccnz .LBB93_312721; GCN2-NEXT: ; %bb.1: ; %Flow12722; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]12723; GCN2-NEXT: s_cbranch_vccz .LBB93_412724; GCN2-NEXT: .LBB93_2: ; %atomicrmw.phi12725; GCN2-NEXT: s_endpgm12726; GCN2-NEXT: .LBB93_3: ; %atomicrmw.global12727; GCN2-NEXT: v_mov_b32_e32 v5, s112728; GCN2-NEXT: v_mov_b32_e32 v0, s1012729; GCN2-NEXT: v_mov_b32_e32 v1, s1112730; GCN2-NEXT: v_mov_b32_e32 v2, s1412731; GCN2-NEXT: v_mov_b32_e32 v3, s1512732; GCN2-NEXT: v_mov_b32_e32 v4, s012733; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:3]12734; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12735; GCN2-NEXT: buffer_wbinvl1_vol12736; GCN2-NEXT: s_cbranch_execnz .LBB93_212737; GCN2-NEXT: .LBB93_4: ; %atomicrmw.private12738; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 012739; GCN2-NEXT: s_cselect_b32 s0, s0, -112740; GCN2-NEXT: v_mov_b32_e32 v2, s012741; GCN2-NEXT: s_add_i32 s0, s0, 412742; GCN2-NEXT: v_mov_b32_e32 v3, s012743; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen12744; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen12745; GCN2-NEXT: v_mov_b32_e32 v5, s1012746; GCN2-NEXT: v_mov_b32_e32 v4, s1112747; GCN2-NEXT: s_waitcnt vmcnt(0)12748; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]12749; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc12750; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc12751; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen12752; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen12753; GCN2-NEXT: s_endpgm12754;12755; GFX12-LABEL: atomic_cmpxchg_i64_addr64_offset:12756; GFX12: ; %bb.0: ; %entry12757; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x2412758; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base12759; GFX12-NEXT: s_wait_kmcnt 0x012760; GFX12-NEXT: s_lshl_b64 s[4:5], s[4:5], 312761; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)12762; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]12763; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 3212764; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)12765; GFX12-NEXT: s_cmp_eq_u32 s1, s912766; GFX12-NEXT: s_cselect_b32 s4, -1, 012767; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s412768; GFX12-NEXT: s_mov_b32 s4, -112769; GFX12-NEXT: s_cbranch_vccnz .LBB93_312770; GFX12-NEXT: ; %bb.1: ; %Flow12771; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s412772; GFX12-NEXT: s_cbranch_vccz .LBB93_412773; GFX12-NEXT: .LBB93_2: ; %atomicrmw.phi12774; GFX12-NEXT: s_endpgm12775; GFX12-NEXT: .LBB93_3: ; %atomicrmw.global12776; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s312777; GFX12-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s712778; GFX12-NEXT: v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s012779; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:3] scope:SCOPE_DEV12780; GFX12-NEXT: s_wait_storecnt_dscnt 0x012781; GFX12-NEXT: global_inv scope:SCOPE_DEV12782; GFX12-NEXT: s_cbranch_execnz .LBB93_212783; GFX12-NEXT: .LBB93_4: ; %atomicrmw.private12784; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 012785; GFX12-NEXT: s_cselect_b32 s0, s0, -112786; GFX12-NEXT: scratch_load_b64 v[0:1], off, s012787; GFX12-NEXT: s_wait_loadcnt 0x012788; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[6:7], v[0:1]12789; GFX12-NEXT: v_cndmask_b32_e64 v1, v1, s3, vcc_lo12790; GFX12-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo12791; GFX12-NEXT: scratch_store_b64 off, v[0:1], s012792; GFX12-NEXT: s_endpgm12793entry:12794 %ptr = getelementptr inbounds i64, ptr %out, i64 %index12795 %gep = getelementptr inbounds i64, ptr %ptr, i64 412796 %val = cmpxchg volatile ptr %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst12797 ret void12798}12799 12800define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index, i64 %old) {12801; GCN1-LABEL: atomic_cmpxchg_i64_ret_addr64_offset:12802; GCN1: ; %bb.0: ; %entry12803; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD012804; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD112805; GCN1-NEXT: s_mov_b32 s18, -112806; GCN1-NEXT: s_mov_b32 s19, 0xe8f00012807; GCN1-NEXT: s_add_u32 s16, s16, s1112808; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x912809; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x1112810; GCN1-NEXT: s_load_dword s4, s[4:5], 0x4312811; GCN1-NEXT: s_addc_u32 s17, s17, 012812; GCN1-NEXT: s_waitcnt lgkmcnt(0)12813; GCN1-NEXT: s_lshl_b64 s[2:3], s[14:15], 312814; GCN1-NEXT: s_add_u32 s2, s8, s212815; GCN1-NEXT: s_addc_u32 s3, s9, s312816; GCN1-NEXT: s_add_u32 s2, s2, 3212817; GCN1-NEXT: s_addc_u32 s3, s3, 012818; GCN1-NEXT: s_cmp_eq_u32 s3, s412819; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 012820; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]12821; GCN1-NEXT: s_cbranch_vccz .LBB94_212822; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global12823; GCN1-NEXT: v_mov_b32_e32 v5, s312824; GCN1-NEXT: v_mov_b32_e32 v0, s1212825; GCN1-NEXT: v_mov_b32_e32 v1, s1312826; GCN1-NEXT: v_mov_b32_e32 v2, s012827; GCN1-NEXT: v_mov_b32_e32 v3, s112828; GCN1-NEXT: v_mov_b32_e32 v4, s212829; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc12830; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12831; GCN1-NEXT: buffer_wbinvl1_vol12832; GCN1-NEXT: s_cbranch_execz .LBB94_312833; GCN1-NEXT: s_branch .LBB94_412834; GCN1-NEXT: .LBB94_2:12835; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr112836; GCN1-NEXT: .LBB94_3: ; %atomicrmw.private12837; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[2:3], 012838; GCN1-NEXT: v_mov_b32_e32 v5, s1212839; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec12840; GCN1-NEXT: s_cselect_b32 s2, s2, -112841; GCN1-NEXT: v_mov_b32_e32 v2, s212842; GCN1-NEXT: s_add_i32 s2, s2, 412843; GCN1-NEXT: v_mov_b32_e32 v3, s212844; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen12845; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen12846; GCN1-NEXT: v_mov_b32_e32 v4, s1312847; GCN1-NEXT: s_waitcnt vmcnt(0)12848; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, s[0:1], v[0:1]12849; GCN1-NEXT: v_cndmask_b32_e32 v5, v0, v5, vcc12850; GCN1-NEXT: v_cndmask_b32_e32 v4, v1, v4, vcc12851; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen12852; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen12853; GCN1-NEXT: .LBB94_4: ; %atomicrmw.end12854; GCN1-NEXT: v_mov_b32_e32 v2, s1012855; GCN1-NEXT: v_mov_b32_e32 v3, s1112856; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]12857; GCN1-NEXT: s_endpgm12858;12859; GCN2-LABEL: atomic_cmpxchg_i64_ret_addr64_offset:12860; GCN2: ; %bb.0: ; %entry12861; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD012862; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD112863; GCN2-NEXT: s_mov_b32 s90, -112864; GCN2-NEXT: s_mov_b32 s91, 0xe8000012865; GCN2-NEXT: s_add_u32 s88, s88, s1112866; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2412867; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x4412868; GCN2-NEXT: s_load_dword s4, s[4:5], 0x10c12869; GCN2-NEXT: s_addc_u32 s89, s89, 012870; GCN2-NEXT: s_waitcnt lgkmcnt(0)12871; GCN2-NEXT: s_lshl_b64 s[2:3], s[14:15], 312872; GCN2-NEXT: s_add_u32 s2, s8, s212873; GCN2-NEXT: s_addc_u32 s3, s9, s312874; GCN2-NEXT: s_add_u32 s2, s2, 3212875; GCN2-NEXT: s_addc_u32 s3, s3, 012876; GCN2-NEXT: s_cmp_eq_u32 s3, s412877; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 012878; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]12879; GCN2-NEXT: s_cbranch_vccz .LBB94_212880; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global12881; GCN2-NEXT: v_mov_b32_e32 v5, s312882; GCN2-NEXT: v_mov_b32_e32 v0, s1212883; GCN2-NEXT: v_mov_b32_e32 v1, s1312884; GCN2-NEXT: v_mov_b32_e32 v2, s012885; GCN2-NEXT: v_mov_b32_e32 v3, s112886; GCN2-NEXT: v_mov_b32_e32 v4, s212887; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc12888; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12889; GCN2-NEXT: buffer_wbinvl1_vol12890; GCN2-NEXT: s_cbranch_execz .LBB94_312891; GCN2-NEXT: s_branch .LBB94_412892; GCN2-NEXT: .LBB94_2:12893; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr112894; GCN2-NEXT: .LBB94_3: ; %atomicrmw.private12895; GCN2-NEXT: s_cmp_lg_u64 s[2:3], 012896; GCN2-NEXT: s_cselect_b32 s2, s2, -112897; GCN2-NEXT: v_mov_b32_e32 v2, s212898; GCN2-NEXT: s_add_i32 s2, s2, 412899; GCN2-NEXT: v_mov_b32_e32 v3, s212900; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen12901; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen12902; GCN2-NEXT: v_mov_b32_e32 v5, s1212903; GCN2-NEXT: v_mov_b32_e32 v4, s1312904; GCN2-NEXT: s_waitcnt vmcnt(0)12905; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, s[0:1], v[0:1]12906; GCN2-NEXT: v_cndmask_b32_e32 v5, v0, v5, vcc12907; GCN2-NEXT: v_cndmask_b32_e32 v4, v1, v4, vcc12908; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen12909; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen12910; GCN2-NEXT: .LBB94_4: ; %atomicrmw.end12911; GCN2-NEXT: v_mov_b32_e32 v2, s1012912; GCN2-NEXT: v_mov_b32_e32 v3, s1112913; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]12914; GCN2-NEXT: s_endpgm12915;12916; GFX12-LABEL: atomic_cmpxchg_i64_ret_addr64_offset:12917; GFX12: ; %bb.0: ; %entry12918; GFX12-NEXT: s_clause 0x112919; GFX12-NEXT: s_load_b256 s[8:15], s[4:5], 0x2412920; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x4412921; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base12922; GFX12-NEXT: s_wait_kmcnt 0x012923; GFX12-NEXT: s_lshl_b64 s[2:3], s[14:15], 312924; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)12925; GFX12-NEXT: s_add_nc_u64 s[2:3], s[8:9], s[2:3]12926; GFX12-NEXT: s_add_nc_u64 s[2:3], s[2:3], 3212927; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)12928; GFX12-NEXT: s_cmp_eq_u32 s3, s712929; GFX12-NEXT: s_cselect_b32 s4, -1, 012930; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s412931; GFX12-NEXT: s_cbranch_vccz .LBB94_212932; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global12933; GFX12-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s1312934; GFX12-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s112935; GFX12-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s212936; GFX12-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[0:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV12937; GFX12-NEXT: s_wait_loadcnt_dscnt 0x012938; GFX12-NEXT: global_inv scope:SCOPE_DEV12939; GFX12-NEXT: s_cbranch_execz .LBB94_312940; GFX12-NEXT: s_branch .LBB94_412941; GFX12-NEXT: .LBB94_2:12942; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr112943; GFX12-NEXT: .LBB94_3: ; %atomicrmw.private12944; GFX12-NEXT: s_cmp_lg_u64 s[2:3], 012945; GFX12-NEXT: s_cselect_b32 s2, s2, -112946; GFX12-NEXT: scratch_load_b64 v[0:1], off, s212947; GFX12-NEXT: s_wait_loadcnt 0x012948; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[0:1], v[0:1]12949; GFX12-NEXT: v_cndmask_b32_e64 v3, v1, s13, vcc_lo12950; GFX12-NEXT: v_cndmask_b32_e64 v2, v0, s12, vcc_lo12951; GFX12-NEXT: scratch_store_b64 off, v[2:3], s212952; GFX12-NEXT: .LBB94_4: ; %atomicrmw.end12953; GFX12-NEXT: v_dual_mov_b32 v2, s10 :: v_dual_mov_b32 v3, s1112954; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]12955; GFX12-NEXT: s_endpgm12956entry:12957 %ptr = getelementptr inbounds i64, ptr %out, i64 %index12958 %gep = getelementptr inbounds i64, ptr %ptr, i64 412959 %val = cmpxchg volatile ptr %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst12960 %extract0 = extractvalue { i64, i1 } %val, 012961 store i64 %extract0, ptr %out212962 ret void12963}12964 12965define amdgpu_kernel void @atomic_cmpxchg_i64(ptr %out, i64 %in, i64 %old) {12966; GCN1-LABEL: atomic_cmpxchg_i64:12967; GCN1: ; %bb.0: ; %entry12968; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD012969; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD112970; GCN1-NEXT: s_mov_b32 s14, -112971; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f12972; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x912973; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd12974; GCN1-NEXT: s_mov_b32 s15, 0xe8f00012975; GCN1-NEXT: s_add_u32 s12, s12, s1112976; GCN1-NEXT: s_addc_u32 s13, s13, 012977; GCN1-NEXT: s_waitcnt lgkmcnt(0)12978; GCN1-NEXT: s_cmp_eq_u32 s1, s612979; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 012980; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]12981; GCN1-NEXT: s_mov_b64 s[6:7], -112982; GCN1-NEXT: s_cbranch_vccnz .LBB95_312983; GCN1-NEXT: ; %bb.1: ; %Flow12984; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]12985; GCN1-NEXT: s_cbranch_vccz .LBB95_412986; GCN1-NEXT: .LBB95_2: ; %atomicrmw.phi12987; GCN1-NEXT: s_endpgm12988; GCN1-NEXT: .LBB95_3: ; %atomicrmw.global12989; GCN1-NEXT: v_mov_b32_e32 v5, s112990; GCN1-NEXT: v_mov_b32_e32 v0, s212991; GCN1-NEXT: v_mov_b32_e32 v1, s312992; GCN1-NEXT: v_mov_b32_e32 v2, s412993; GCN1-NEXT: v_mov_b32_e32 v3, s512994; GCN1-NEXT: v_mov_b32_e32 v4, s012995; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:3]12996; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12997; GCN1-NEXT: buffer_wbinvl1_vol12998; GCN1-NEXT: s_cbranch_execnz .LBB95_212999; GCN1-NEXT: .LBB95_4: ; %atomicrmw.private13000; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 013001; GCN1-NEXT: v_mov_b32_e32 v5, s213002; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec13003; GCN1-NEXT: s_cselect_b32 s0, s0, -113004; GCN1-NEXT: v_mov_b32_e32 v2, s013005; GCN1-NEXT: s_add_i32 s0, s0, 413006; GCN1-NEXT: v_mov_b32_e32 v3, s013007; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen13008; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen13009; GCN1-NEXT: v_mov_b32_e32 v4, s313010; GCN1-NEXT: s_waitcnt vmcnt(0)13011; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]13012; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc13013; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc13014; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen13015; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen13016; GCN1-NEXT: s_endpgm13017;13018; GCN2-LABEL: atomic_cmpxchg_i64:13019; GCN2: ; %bb.0: ; %entry13020; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD013021; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD113022; GCN2-NEXT: s_mov_b32 s90, -113023; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc13024; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2413025; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x3413026; GCN2-NEXT: s_mov_b32 s91, 0xe8000013027; GCN2-NEXT: s_add_u32 s88, s88, s1113028; GCN2-NEXT: s_addc_u32 s89, s89, 013029; GCN2-NEXT: s_waitcnt lgkmcnt(0)13030; GCN2-NEXT: s_cmp_eq_u32 s1, s613031; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 013032; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]13033; GCN2-NEXT: s_mov_b64 s[6:7], -113034; GCN2-NEXT: s_cbranch_vccnz .LBB95_313035; GCN2-NEXT: ; %bb.1: ; %Flow13036; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]13037; GCN2-NEXT: s_cbranch_vccz .LBB95_413038; GCN2-NEXT: .LBB95_2: ; %atomicrmw.phi13039; GCN2-NEXT: s_endpgm13040; GCN2-NEXT: .LBB95_3: ; %atomicrmw.global13041; GCN2-NEXT: v_mov_b32_e32 v5, s113042; GCN2-NEXT: v_mov_b32_e32 v0, s213043; GCN2-NEXT: v_mov_b32_e32 v1, s313044; GCN2-NEXT: v_mov_b32_e32 v2, s413045; GCN2-NEXT: v_mov_b32_e32 v3, s513046; GCN2-NEXT: v_mov_b32_e32 v4, s013047; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:3]13048; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13049; GCN2-NEXT: buffer_wbinvl1_vol13050; GCN2-NEXT: s_cbranch_execnz .LBB95_213051; GCN2-NEXT: .LBB95_4: ; %atomicrmw.private13052; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 013053; GCN2-NEXT: s_cselect_b32 s0, s0, -113054; GCN2-NEXT: v_mov_b32_e32 v2, s013055; GCN2-NEXT: s_add_i32 s0, s0, 413056; GCN2-NEXT: v_mov_b32_e32 v3, s013057; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen13058; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen13059; GCN2-NEXT: v_mov_b32_e32 v5, s213060; GCN2-NEXT: v_mov_b32_e32 v4, s313061; GCN2-NEXT: s_waitcnt vmcnt(0)13062; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]13063; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc13064; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc13065; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen13066; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen13067; GCN2-NEXT: s_endpgm13068;13069; GFX12-LABEL: atomic_cmpxchg_i64:13070; GFX12: ; %bb.0: ; %entry13071; GFX12-NEXT: s_clause 0x113072; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2413073; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x3413074; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base13075; GFX12-NEXT: s_wait_kmcnt 0x013076; GFX12-NEXT: s_cmp_eq_u32 s1, s713077; GFX12-NEXT: s_cselect_b32 s6, -1, 013078; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13079; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s613080; GFX12-NEXT: s_mov_b32 s6, -113081; GFX12-NEXT: s_cbranch_vccnz .LBB95_313082; GFX12-NEXT: ; %bb.1: ; %Flow13083; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s613084; GFX12-NEXT: s_cbranch_vccz .LBB95_413085; GFX12-NEXT: .LBB95_2: ; %atomicrmw.phi13086; GFX12-NEXT: s_endpgm13087; GFX12-NEXT: .LBB95_3: ; %atomicrmw.global13088; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s313089; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s513090; GFX12-NEXT: v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s013091; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:3] scope:SCOPE_DEV13092; GFX12-NEXT: s_wait_storecnt_dscnt 0x013093; GFX12-NEXT: global_inv scope:SCOPE_DEV13094; GFX12-NEXT: s_cbranch_execnz .LBB95_213095; GFX12-NEXT: .LBB95_4: ; %atomicrmw.private13096; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 013097; GFX12-NEXT: s_cselect_b32 s0, s0, -113098; GFX12-NEXT: scratch_load_b64 v[0:1], off, s013099; GFX12-NEXT: s_wait_loadcnt 0x013100; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]13101; GFX12-NEXT: v_cndmask_b32_e64 v1, v1, s3, vcc_lo13102; GFX12-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo13103; GFX12-NEXT: scratch_store_b64 off, v[0:1], s013104; GFX12-NEXT: s_endpgm13105entry:13106 %val = cmpxchg volatile ptr %out, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst13107 ret void13108}13109 13110define amdgpu_kernel void @atomic_cmpxchg_i64_ret(ptr %out, ptr %out2, i64 %in, i64 %old) {13111; GCN1-LABEL: atomic_cmpxchg_i64_ret:13112; GCN1: ; %bb.0: ; %entry13113; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD013114; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD113115; GCN1-NEXT: s_mov_b32 s18, -113116; GCN1-NEXT: s_mov_b32 s19, 0xe8f00013117; GCN1-NEXT: s_add_u32 s16, s16, s1113118; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x913119; GCN1-NEXT: s_load_dword s0, s[4:5], 0x4113120; GCN1-NEXT: s_addc_u32 s17, s17, 013121; GCN1-NEXT: s_waitcnt lgkmcnt(0)13122; GCN1-NEXT: s_cmp_eq_u32 s9, s013123; GCN1-NEXT: s_cselect_b64 s[0:1], -1, 013124; GCN1-NEXT: s_andn2_b64 vcc, exec, s[0:1]13125; GCN1-NEXT: s_cbranch_vccz .LBB96_213126; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global13127; GCN1-NEXT: v_mov_b32_e32 v4, s813128; GCN1-NEXT: v_mov_b32_e32 v0, s1213129; GCN1-NEXT: v_mov_b32_e32 v1, s1313130; GCN1-NEXT: v_mov_b32_e32 v2, s1413131; GCN1-NEXT: v_mov_b32_e32 v3, s1513132; GCN1-NEXT: v_mov_b32_e32 v5, s913133; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc13134; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13135; GCN1-NEXT: buffer_wbinvl1_vol13136; GCN1-NEXT: s_cbranch_execz .LBB96_313137; GCN1-NEXT: s_branch .LBB96_413138; GCN1-NEXT: .LBB96_2:13139; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr113140; GCN1-NEXT: .LBB96_3: ; %atomicrmw.private13141; GCN1-NEXT: v_cmp_ne_u64_e64 s[0:1], s[8:9], 013142; GCN1-NEXT: v_mov_b32_e32 v5, s1213143; GCN1-NEXT: s_and_b64 s[0:1], s[0:1], exec13144; GCN1-NEXT: s_cselect_b32 s0, s8, -113145; GCN1-NEXT: v_mov_b32_e32 v2, s013146; GCN1-NEXT: s_add_i32 s0, s0, 413147; GCN1-NEXT: v_mov_b32_e32 v3, s013148; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen13149; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen13150; GCN1-NEXT: v_mov_b32_e32 v4, s1313151; GCN1-NEXT: s_waitcnt vmcnt(0)13152; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]13153; GCN1-NEXT: v_cndmask_b32_e32 v5, v0, v5, vcc13154; GCN1-NEXT: v_cndmask_b32_e32 v4, v1, v4, vcc13155; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen13156; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen13157; GCN1-NEXT: .LBB96_4: ; %atomicrmw.end13158; GCN1-NEXT: v_mov_b32_e32 v2, s1013159; GCN1-NEXT: v_mov_b32_e32 v3, s1113160; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]13161; GCN1-NEXT: s_endpgm13162;13163; GCN2-LABEL: atomic_cmpxchg_i64_ret:13164; GCN2: ; %bb.0: ; %entry13165; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD013166; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD113167; GCN2-NEXT: s_mov_b32 s90, -113168; GCN2-NEXT: s_mov_b32 s91, 0xe8000013169; GCN2-NEXT: s_add_u32 s88, s88, s1113170; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2413171; GCN2-NEXT: s_load_dword s0, s[4:5], 0x10413172; GCN2-NEXT: s_addc_u32 s89, s89, 013173; GCN2-NEXT: s_waitcnt lgkmcnt(0)13174; GCN2-NEXT: s_cmp_eq_u32 s9, s013175; GCN2-NEXT: s_cselect_b64 s[0:1], -1, 013176; GCN2-NEXT: s_andn2_b64 vcc, exec, s[0:1]13177; GCN2-NEXT: s_cbranch_vccz .LBB96_213178; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global13179; GCN2-NEXT: v_mov_b32_e32 v4, s813180; GCN2-NEXT: v_mov_b32_e32 v0, s1213181; GCN2-NEXT: v_mov_b32_e32 v1, s1313182; GCN2-NEXT: v_mov_b32_e32 v2, s1413183; GCN2-NEXT: v_mov_b32_e32 v3, s1513184; GCN2-NEXT: v_mov_b32_e32 v5, s913185; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc13186; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13187; GCN2-NEXT: buffer_wbinvl1_vol13188; GCN2-NEXT: s_cbranch_execz .LBB96_313189; GCN2-NEXT: s_branch .LBB96_413190; GCN2-NEXT: .LBB96_2:13191; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr113192; GCN2-NEXT: .LBB96_3: ; %atomicrmw.private13193; GCN2-NEXT: s_cmp_lg_u64 s[8:9], 013194; GCN2-NEXT: s_cselect_b32 s0, s8, -113195; GCN2-NEXT: v_mov_b32_e32 v2, s013196; GCN2-NEXT: s_add_i32 s0, s0, 413197; GCN2-NEXT: v_mov_b32_e32 v3, s013198; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen13199; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen13200; GCN2-NEXT: v_mov_b32_e32 v5, s1213201; GCN2-NEXT: v_mov_b32_e32 v4, s1313202; GCN2-NEXT: s_waitcnt vmcnt(0)13203; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]13204; GCN2-NEXT: v_cndmask_b32_e32 v5, v0, v5, vcc13205; GCN2-NEXT: v_cndmask_b32_e32 v4, v1, v4, vcc13206; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen13207; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen13208; GCN2-NEXT: .LBB96_4: ; %atomicrmw.end13209; GCN2-NEXT: v_mov_b32_e32 v2, s1013210; GCN2-NEXT: v_mov_b32_e32 v3, s1113211; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]13212; GCN2-NEXT: s_endpgm13213;13214; GFX12-LABEL: atomic_cmpxchg_i64_ret:13215; GFX12: ; %bb.0: ; %entry13216; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x2413217; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base13218; GFX12-NEXT: s_wait_kmcnt 0x013219; GFX12-NEXT: s_cmp_eq_u32 s1, s913220; GFX12-NEXT: s_cselect_b32 s8, -1, 013221; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13222; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s813223; GFX12-NEXT: s_cbranch_vccz .LBB96_213224; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global13225; GFX12-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s513226; GFX12-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s713227; GFX12-NEXT: v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s013228; GFX12-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[0:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV13229; GFX12-NEXT: s_wait_loadcnt_dscnt 0x013230; GFX12-NEXT: global_inv scope:SCOPE_DEV13231; GFX12-NEXT: s_cbranch_execz .LBB96_313232; GFX12-NEXT: s_branch .LBB96_413233; GFX12-NEXT: .LBB96_2:13234; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr113235; GFX12-NEXT: .LBB96_3: ; %atomicrmw.private13236; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 013237; GFX12-NEXT: s_cselect_b32 s0, s0, -113238; GFX12-NEXT: scratch_load_b64 v[0:1], off, s013239; GFX12-NEXT: s_wait_loadcnt 0x013240; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[6:7], v[0:1]13241; GFX12-NEXT: v_cndmask_b32_e64 v3, v1, s5, vcc_lo13242; GFX12-NEXT: v_cndmask_b32_e64 v2, v0, s4, vcc_lo13243; GFX12-NEXT: scratch_store_b64 off, v[2:3], s013244; GFX12-NEXT: .LBB96_4: ; %atomicrmw.end13245; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s313246; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]13247; GFX12-NEXT: s_endpgm13248entry:13249 %val = cmpxchg volatile ptr %out, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst13250 %extract0 = extractvalue { i64, i1 } %val, 013251 store i64 %extract0, ptr %out213252 ret void13253}13254 13255define amdgpu_kernel void @atomic_cmpxchg_i64_addr64(ptr %out, i64 %in, i64 %index, i64 %old) {13256; GCN1-LABEL: atomic_cmpxchg_i64_addr64:13257; GCN1: ; %bb.0: ; %entry13258; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD013259; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD113260; GCN1-NEXT: s_mov_b32 s18, -113261; GCN1-NEXT: s_mov_b32 s19, 0xe8f00013262; GCN1-NEXT: s_add_u32 s16, s16, s1113263; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x913264; GCN1-NEXT: s_load_dword s2, s[4:5], 0x4113265; GCN1-NEXT: s_addc_u32 s17, s17, 013266; GCN1-NEXT: s_waitcnt lgkmcnt(0)13267; GCN1-NEXT: s_lshl_b64 s[0:1], s[12:13], 313268; GCN1-NEXT: s_add_u32 s0, s8, s013269; GCN1-NEXT: s_addc_u32 s1, s9, s113270; GCN1-NEXT: s_cmp_eq_u32 s1, s213271; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 013272; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]13273; GCN1-NEXT: s_mov_b64 s[2:3], -113274; GCN1-NEXT: s_cbranch_vccnz .LBB97_313275; GCN1-NEXT: ; %bb.1: ; %Flow13276; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]13277; GCN1-NEXT: s_cbranch_vccz .LBB97_413278; GCN1-NEXT: .LBB97_2: ; %atomicrmw.phi13279; GCN1-NEXT: s_endpgm13280; GCN1-NEXT: .LBB97_3: ; %atomicrmw.global13281; GCN1-NEXT: v_mov_b32_e32 v5, s113282; GCN1-NEXT: v_mov_b32_e32 v0, s1013283; GCN1-NEXT: v_mov_b32_e32 v1, s1113284; GCN1-NEXT: v_mov_b32_e32 v2, s1413285; GCN1-NEXT: v_mov_b32_e32 v3, s1513286; GCN1-NEXT: v_mov_b32_e32 v4, s013287; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:3]13288; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13289; GCN1-NEXT: buffer_wbinvl1_vol13290; GCN1-NEXT: s_cbranch_execnz .LBB97_213291; GCN1-NEXT: .LBB97_4: ; %atomicrmw.private13292; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 013293; GCN1-NEXT: v_mov_b32_e32 v5, s1013294; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec13295; GCN1-NEXT: s_cselect_b32 s0, s0, -113296; GCN1-NEXT: v_mov_b32_e32 v2, s013297; GCN1-NEXT: s_add_i32 s0, s0, 413298; GCN1-NEXT: v_mov_b32_e32 v3, s013299; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen13300; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen13301; GCN1-NEXT: v_mov_b32_e32 v4, s1113302; GCN1-NEXT: s_waitcnt vmcnt(0)13303; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]13304; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc13305; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc13306; GCN1-NEXT: buffer_store_dword v0, v2, s[16:19], 0 offen13307; GCN1-NEXT: buffer_store_dword v1, v3, s[16:19], 0 offen13308; GCN1-NEXT: s_endpgm13309;13310; GCN2-LABEL: atomic_cmpxchg_i64_addr64:13311; GCN2: ; %bb.0: ; %entry13312; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD013313; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD113314; GCN2-NEXT: s_mov_b32 s90, -113315; GCN2-NEXT: s_mov_b32 s91, 0xe8000013316; GCN2-NEXT: s_add_u32 s88, s88, s1113317; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2413318; GCN2-NEXT: s_load_dword s2, s[4:5], 0x10413319; GCN2-NEXT: s_addc_u32 s89, s89, 013320; GCN2-NEXT: s_waitcnt lgkmcnt(0)13321; GCN2-NEXT: s_lshl_b64 s[0:1], s[12:13], 313322; GCN2-NEXT: s_add_u32 s0, s8, s013323; GCN2-NEXT: s_addc_u32 s1, s9, s113324; GCN2-NEXT: s_cmp_eq_u32 s1, s213325; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 013326; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]13327; GCN2-NEXT: s_mov_b64 s[2:3], -113328; GCN2-NEXT: s_cbranch_vccnz .LBB97_313329; GCN2-NEXT: ; %bb.1: ; %Flow13330; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]13331; GCN2-NEXT: s_cbranch_vccz .LBB97_413332; GCN2-NEXT: .LBB97_2: ; %atomicrmw.phi13333; GCN2-NEXT: s_endpgm13334; GCN2-NEXT: .LBB97_3: ; %atomicrmw.global13335; GCN2-NEXT: v_mov_b32_e32 v5, s113336; GCN2-NEXT: v_mov_b32_e32 v0, s1013337; GCN2-NEXT: v_mov_b32_e32 v1, s1113338; GCN2-NEXT: v_mov_b32_e32 v2, s1413339; GCN2-NEXT: v_mov_b32_e32 v3, s1513340; GCN2-NEXT: v_mov_b32_e32 v4, s013341; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:3]13342; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13343; GCN2-NEXT: buffer_wbinvl1_vol13344; GCN2-NEXT: s_cbranch_execnz .LBB97_213345; GCN2-NEXT: .LBB97_4: ; %atomicrmw.private13346; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 013347; GCN2-NEXT: s_cselect_b32 s0, s0, -113348; GCN2-NEXT: v_mov_b32_e32 v2, s013349; GCN2-NEXT: s_add_i32 s0, s0, 413350; GCN2-NEXT: v_mov_b32_e32 v3, s013351; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen13352; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen13353; GCN2-NEXT: v_mov_b32_e32 v5, s1013354; GCN2-NEXT: v_mov_b32_e32 v4, s1113355; GCN2-NEXT: s_waitcnt vmcnt(0)13356; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]13357; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc13358; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc13359; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen13360; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen13361; GCN2-NEXT: s_endpgm13362;13363; GFX12-LABEL: atomic_cmpxchg_i64_addr64:13364; GFX12: ; %bb.0: ; %entry13365; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x2413366; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base13367; GFX12-NEXT: s_wait_kmcnt 0x013368; GFX12-NEXT: s_lshl_b64 s[4:5], s[4:5], 313369; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)13370; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]13371; GFX12-NEXT: s_cmp_eq_u32 s1, s913372; GFX12-NEXT: s_cselect_b32 s4, -1, 013373; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13374; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s413375; GFX12-NEXT: s_mov_b32 s4, -113376; GFX12-NEXT: s_cbranch_vccnz .LBB97_313377; GFX12-NEXT: ; %bb.1: ; %Flow13378; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s413379; GFX12-NEXT: s_cbranch_vccz .LBB97_413380; GFX12-NEXT: .LBB97_2: ; %atomicrmw.phi13381; GFX12-NEXT: s_endpgm13382; GFX12-NEXT: .LBB97_3: ; %atomicrmw.global13383; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s313384; GFX12-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s713385; GFX12-NEXT: v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s013386; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:3] scope:SCOPE_DEV13387; GFX12-NEXT: s_wait_storecnt_dscnt 0x013388; GFX12-NEXT: global_inv scope:SCOPE_DEV13389; GFX12-NEXT: s_cbranch_execnz .LBB97_213390; GFX12-NEXT: .LBB97_4: ; %atomicrmw.private13391; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 013392; GFX12-NEXT: s_cselect_b32 s0, s0, -113393; GFX12-NEXT: scratch_load_b64 v[0:1], off, s013394; GFX12-NEXT: s_wait_loadcnt 0x013395; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[6:7], v[0:1]13396; GFX12-NEXT: v_cndmask_b32_e64 v1, v1, s3, vcc_lo13397; GFX12-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo13398; GFX12-NEXT: scratch_store_b64 off, v[0:1], s013399; GFX12-NEXT: s_endpgm13400entry:13401 %ptr = getelementptr inbounds i64, ptr %out, i64 %index13402 %val = cmpxchg volatile ptr %ptr, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst13403 ret void13404}13405 13406define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index, i64 %old) {13407; GCN1-LABEL: atomic_cmpxchg_i64_ret_addr64:13408; GCN1: ; %bb.0: ; %entry13409; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD013410; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD113411; GCN1-NEXT: s_mov_b32 s18, -113412; GCN1-NEXT: s_mov_b32 s19, 0xe8f00013413; GCN1-NEXT: s_add_u32 s16, s16, s1113414; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x913415; GCN1-NEXT: s_load_dword s6, s[4:5], 0x4313416; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x1113417; GCN1-NEXT: s_addc_u32 s17, s17, 013418; GCN1-NEXT: s_waitcnt lgkmcnt(0)13419; GCN1-NEXT: s_lshl_b64 s[2:3], s[14:15], 313420; GCN1-NEXT: s_add_u32 s2, s8, s213421; GCN1-NEXT: s_addc_u32 s3, s9, s313422; GCN1-NEXT: s_cmp_eq_u32 s3, s613423; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 013424; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]13425; GCN1-NEXT: s_cbranch_vccz .LBB98_213426; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global13427; GCN1-NEXT: v_mov_b32_e32 v5, s313428; GCN1-NEXT: v_mov_b32_e32 v0, s1213429; GCN1-NEXT: v_mov_b32_e32 v1, s1313430; GCN1-NEXT: v_mov_b32_e32 v2, s013431; GCN1-NEXT: v_mov_b32_e32 v3, s113432; GCN1-NEXT: v_mov_b32_e32 v4, s213433; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc13434; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13435; GCN1-NEXT: buffer_wbinvl1_vol13436; GCN1-NEXT: s_cbranch_execz .LBB98_313437; GCN1-NEXT: s_branch .LBB98_413438; GCN1-NEXT: .LBB98_2:13439; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr113440; GCN1-NEXT: .LBB98_3: ; %atomicrmw.private13441; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[2:3], 013442; GCN1-NEXT: v_mov_b32_e32 v5, s1213443; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec13444; GCN1-NEXT: s_cselect_b32 s2, s2, -113445; GCN1-NEXT: v_mov_b32_e32 v2, s213446; GCN1-NEXT: s_add_i32 s2, s2, 413447; GCN1-NEXT: v_mov_b32_e32 v3, s213448; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen13449; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen13450; GCN1-NEXT: v_mov_b32_e32 v4, s1313451; GCN1-NEXT: s_waitcnt vmcnt(0)13452; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, s[0:1], v[0:1]13453; GCN1-NEXT: v_cndmask_b32_e32 v5, v0, v5, vcc13454; GCN1-NEXT: v_cndmask_b32_e32 v4, v1, v4, vcc13455; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen13456; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen13457; GCN1-NEXT: .LBB98_4: ; %atomicrmw.end13458; GCN1-NEXT: v_mov_b32_e32 v2, s1013459; GCN1-NEXT: v_mov_b32_e32 v3, s1113460; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]13461; GCN1-NEXT: s_endpgm13462;13463; GCN2-LABEL: atomic_cmpxchg_i64_ret_addr64:13464; GCN2: ; %bb.0: ; %entry13465; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD013466; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD113467; GCN2-NEXT: s_mov_b32 s90, -113468; GCN2-NEXT: s_mov_b32 s91, 0xe8000013469; GCN2-NEXT: s_add_u32 s88, s88, s1113470; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2413471; GCN2-NEXT: s_load_dword s6, s[4:5], 0x10c13472; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x4413473; GCN2-NEXT: s_addc_u32 s89, s89, 013474; GCN2-NEXT: s_waitcnt lgkmcnt(0)13475; GCN2-NEXT: s_lshl_b64 s[2:3], s[14:15], 313476; GCN2-NEXT: s_add_u32 s2, s8, s213477; GCN2-NEXT: s_addc_u32 s3, s9, s313478; GCN2-NEXT: s_cmp_eq_u32 s3, s613479; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 013480; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]13481; GCN2-NEXT: s_cbranch_vccz .LBB98_213482; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global13483; GCN2-NEXT: v_mov_b32_e32 v5, s313484; GCN2-NEXT: v_mov_b32_e32 v0, s1213485; GCN2-NEXT: v_mov_b32_e32 v1, s1313486; GCN2-NEXT: v_mov_b32_e32 v2, s013487; GCN2-NEXT: v_mov_b32_e32 v3, s113488; GCN2-NEXT: v_mov_b32_e32 v4, s213489; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc13490; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13491; GCN2-NEXT: buffer_wbinvl1_vol13492; GCN2-NEXT: s_cbranch_execz .LBB98_313493; GCN2-NEXT: s_branch .LBB98_413494; GCN2-NEXT: .LBB98_2:13495; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr113496; GCN2-NEXT: .LBB98_3: ; %atomicrmw.private13497; GCN2-NEXT: s_cmp_lg_u64 s[2:3], 013498; GCN2-NEXT: s_cselect_b32 s2, s2, -113499; GCN2-NEXT: v_mov_b32_e32 v2, s213500; GCN2-NEXT: s_add_i32 s2, s2, 413501; GCN2-NEXT: v_mov_b32_e32 v3, s213502; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen13503; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen13504; GCN2-NEXT: v_mov_b32_e32 v5, s1213505; GCN2-NEXT: v_mov_b32_e32 v4, s1313506; GCN2-NEXT: s_waitcnt vmcnt(0)13507; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, s[0:1], v[0:1]13508; GCN2-NEXT: v_cndmask_b32_e32 v5, v0, v5, vcc13509; GCN2-NEXT: v_cndmask_b32_e32 v4, v1, v4, vcc13510; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen13511; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen13512; GCN2-NEXT: .LBB98_4: ; %atomicrmw.end13513; GCN2-NEXT: v_mov_b32_e32 v2, s1013514; GCN2-NEXT: v_mov_b32_e32 v3, s1113515; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]13516; GCN2-NEXT: s_endpgm13517;13518; GFX12-LABEL: atomic_cmpxchg_i64_ret_addr64:13519; GFX12: ; %bb.0: ; %entry13520; GFX12-NEXT: s_clause 0x113521; GFX12-NEXT: s_load_b256 s[8:15], s[4:5], 0x2413522; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x4413523; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base13524; GFX12-NEXT: s_wait_kmcnt 0x013525; GFX12-NEXT: s_lshl_b64 s[2:3], s[14:15], 313526; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)13527; GFX12-NEXT: s_add_nc_u64 s[2:3], s[8:9], s[2:3]13528; GFX12-NEXT: s_cmp_eq_u32 s3, s713529; GFX12-NEXT: s_cselect_b32 s4, -1, 013530; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13531; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s413532; GFX12-NEXT: s_cbranch_vccz .LBB98_213533; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global13534; GFX12-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s1313535; GFX12-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s113536; GFX12-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s213537; GFX12-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[0:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV13538; GFX12-NEXT: s_wait_loadcnt_dscnt 0x013539; GFX12-NEXT: global_inv scope:SCOPE_DEV13540; GFX12-NEXT: s_cbranch_execz .LBB98_313541; GFX12-NEXT: s_branch .LBB98_413542; GFX12-NEXT: .LBB98_2:13543; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr113544; GFX12-NEXT: .LBB98_3: ; %atomicrmw.private13545; GFX12-NEXT: s_cmp_lg_u64 s[2:3], 013546; GFX12-NEXT: s_cselect_b32 s2, s2, -113547; GFX12-NEXT: scratch_load_b64 v[0:1], off, s213548; GFX12-NEXT: s_wait_loadcnt 0x013549; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[0:1], v[0:1]13550; GFX12-NEXT: v_cndmask_b32_e64 v3, v1, s13, vcc_lo13551; GFX12-NEXT: v_cndmask_b32_e64 v2, v0, s12, vcc_lo13552; GFX12-NEXT: scratch_store_b64 off, v[2:3], s213553; GFX12-NEXT: .LBB98_4: ; %atomicrmw.end13554; GFX12-NEXT: v_dual_mov_b32 v2, s10 :: v_dual_mov_b32 v3, s1113555; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]13556; GFX12-NEXT: s_endpgm13557entry:13558 %ptr = getelementptr inbounds i64, ptr %out, i64 %index13559 %val = cmpxchg volatile ptr %ptr, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst13560 %extract0 = extractvalue { i64, i1 } %val, 013561 store i64 %extract0, ptr %out213562 ret void13563}13564 13565define amdgpu_kernel void @atomic_load_f64_offset(ptr %in, ptr %out) {13566; GCN1-LABEL: atomic_load_f64_offset:13567; GCN1: ; %bb.0: ; %entry13568; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x913569; GCN1-NEXT: s_waitcnt lgkmcnt(0)13570; GCN1-NEXT: s_add_u32 s0, s0, 3213571; GCN1-NEXT: s_addc_u32 s1, s1, 013572; GCN1-NEXT: v_mov_b32_e32 v0, s013573; GCN1-NEXT: v_mov_b32_e32 v1, s113574; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc13575; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13576; GCN1-NEXT: buffer_wbinvl1_vol13577; GCN1-NEXT: v_mov_b32_e32 v2, s213578; GCN1-NEXT: v_mov_b32_e32 v3, s313579; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]13580; GCN1-NEXT: s_endpgm13581;13582; GCN2-LABEL: atomic_load_f64_offset:13583; GCN2: ; %bb.0: ; %entry13584; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2413585; GCN2-NEXT: s_waitcnt lgkmcnt(0)13586; GCN2-NEXT: s_add_u32 s0, s0, 3213587; GCN2-NEXT: s_addc_u32 s1, s1, 013588; GCN2-NEXT: v_mov_b32_e32 v0, s013589; GCN2-NEXT: v_mov_b32_e32 v1, s113590; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc13591; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13592; GCN2-NEXT: buffer_wbinvl1_vol13593; GCN2-NEXT: v_mov_b32_e32 v2, s213594; GCN2-NEXT: v_mov_b32_e32 v3, s313595; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]13596; GCN2-NEXT: s_endpgm13597;13598; GFX12-LABEL: atomic_load_f64_offset:13599; GFX12: ; %bb.0: ; %entry13600; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2413601; GFX12-NEXT: s_wait_kmcnt 0x013602; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s113603; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s313604; GFX12-NEXT: flat_load_b64 v[0:1], v[0:1] offset:32 scope:SCOPE_SYS13605; GFX12-NEXT: s_wait_loadcnt_dscnt 0x013606; GFX12-NEXT: global_inv scope:SCOPE_SYS13607; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]13608; GFX12-NEXT: s_endpgm13609entry:13610 %gep = getelementptr inbounds double, ptr %in, i64 413611 %val = load atomic double, ptr %gep seq_cst, align 813612 store double %val, ptr %out13613 ret void13614}13615 13616define amdgpu_kernel void @atomic_load_f64(ptr %in, ptr %out) {13617; GCN1-LABEL: atomic_load_f64:13618; GCN1: ; %bb.0: ; %entry13619; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x913620; GCN1-NEXT: s_waitcnt lgkmcnt(0)13621; GCN1-NEXT: v_mov_b32_e32 v0, s013622; GCN1-NEXT: v_mov_b32_e32 v1, s113623; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc13624; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13625; GCN1-NEXT: buffer_wbinvl1_vol13626; GCN1-NEXT: v_mov_b32_e32 v2, s213627; GCN1-NEXT: v_mov_b32_e32 v3, s313628; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]13629; GCN1-NEXT: s_endpgm13630;13631; GCN2-LABEL: atomic_load_f64:13632; GCN2: ; %bb.0: ; %entry13633; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2413634; GCN2-NEXT: s_waitcnt lgkmcnt(0)13635; GCN2-NEXT: v_mov_b32_e32 v0, s013636; GCN2-NEXT: v_mov_b32_e32 v1, s113637; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc13638; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13639; GCN2-NEXT: buffer_wbinvl1_vol13640; GCN2-NEXT: v_mov_b32_e32 v2, s213641; GCN2-NEXT: v_mov_b32_e32 v3, s313642; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]13643; GCN2-NEXT: s_endpgm13644;13645; GFX12-LABEL: atomic_load_f64:13646; GFX12: ; %bb.0: ; %entry13647; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2413648; GFX12-NEXT: s_wait_kmcnt 0x013649; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s113650; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s313651; GFX12-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV13652; GFX12-NEXT: s_wait_loadcnt_dscnt 0x013653; GFX12-NEXT: global_inv scope:SCOPE_DEV13654; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]13655; GFX12-NEXT: s_endpgm13656entry:13657 %val = load atomic double, ptr %in syncscope("agent") seq_cst, align 813658 store double %val, ptr %out13659 ret void13660}13661 13662define amdgpu_kernel void @atomic_load_f64_addr64_offset(ptr %in, ptr %out, i64 %index) {13663; GCN1-LABEL: atomic_load_f64_addr64_offset:13664; GCN1: ; %bb.0: ; %entry13665; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd13666; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x913667; GCN1-NEXT: s_waitcnt lgkmcnt(0)13668; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 313669; GCN1-NEXT: s_add_u32 s0, s0, s413670; GCN1-NEXT: s_addc_u32 s1, s1, s513671; GCN1-NEXT: s_add_u32 s0, s0, 3213672; GCN1-NEXT: s_addc_u32 s1, s1, 013673; GCN1-NEXT: v_mov_b32_e32 v0, s013674; GCN1-NEXT: v_mov_b32_e32 v1, s113675; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc13676; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13677; GCN1-NEXT: buffer_wbinvl1_vol13678; GCN1-NEXT: v_mov_b32_e32 v2, s213679; GCN1-NEXT: v_mov_b32_e32 v3, s313680; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]13681; GCN1-NEXT: s_endpgm13682;13683; GCN2-LABEL: atomic_load_f64_addr64_offset:13684; GCN2: ; %bb.0: ; %entry13685; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3413686; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2413687; GCN2-NEXT: s_waitcnt lgkmcnt(0)13688; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 313689; GCN2-NEXT: s_add_u32 s0, s0, s413690; GCN2-NEXT: s_addc_u32 s1, s1, s513691; GCN2-NEXT: s_add_u32 s0, s0, 3213692; GCN2-NEXT: s_addc_u32 s1, s1, 013693; GCN2-NEXT: v_mov_b32_e32 v0, s013694; GCN2-NEXT: v_mov_b32_e32 v1, s113695; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc13696; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13697; GCN2-NEXT: buffer_wbinvl1_vol13698; GCN2-NEXT: v_mov_b32_e32 v2, s213699; GCN2-NEXT: v_mov_b32_e32 v3, s313700; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]13701; GCN2-NEXT: s_endpgm13702;13703; GFX12-LABEL: atomic_load_f64_addr64_offset:13704; GFX12: ; %bb.0: ; %entry13705; GFX12-NEXT: s_clause 0x113706; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x3413707; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2413708; GFX12-NEXT: s_wait_kmcnt 0x013709; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 313710; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s313711; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]13712; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13713; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s113714; GFX12-NEXT: flat_load_b64 v[0:1], v[0:1] offset:32 scope:SCOPE_SYS13715; GFX12-NEXT: s_wait_loadcnt_dscnt 0x013716; GFX12-NEXT: global_inv scope:SCOPE_SYS13717; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]13718; GFX12-NEXT: s_endpgm13719entry:13720 %ptr = getelementptr inbounds double, ptr %in, i64 %index13721 %gep = getelementptr inbounds double, ptr %ptr, i64 413722 %val = load atomic double, ptr %gep seq_cst, align 813723 store double %val, ptr %out13724 ret void13725}13726 13727define amdgpu_kernel void @atomic_load_f64_addr64(ptr %in, ptr %out, i64 %index) {13728; GCN1-LABEL: atomic_load_f64_addr64:13729; GCN1: ; %bb.0: ; %entry13730; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd13731; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x913732; GCN1-NEXT: s_waitcnt lgkmcnt(0)13733; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 313734; GCN1-NEXT: s_add_u32 s0, s0, s413735; GCN1-NEXT: s_addc_u32 s1, s1, s513736; GCN1-NEXT: v_mov_b32_e32 v0, s013737; GCN1-NEXT: v_mov_b32_e32 v1, s113738; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc13739; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13740; GCN1-NEXT: buffer_wbinvl1_vol13741; GCN1-NEXT: v_mov_b32_e32 v2, s213742; GCN1-NEXT: v_mov_b32_e32 v3, s313743; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]13744; GCN1-NEXT: s_endpgm13745;13746; GCN2-LABEL: atomic_load_f64_addr64:13747; GCN2: ; %bb.0: ; %entry13748; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3413749; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2413750; GCN2-NEXT: s_waitcnt lgkmcnt(0)13751; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 313752; GCN2-NEXT: s_add_u32 s0, s0, s413753; GCN2-NEXT: s_addc_u32 s1, s1, s513754; GCN2-NEXT: v_mov_b32_e32 v0, s013755; GCN2-NEXT: v_mov_b32_e32 v1, s113756; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc13757; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13758; GCN2-NEXT: buffer_wbinvl1_vol13759; GCN2-NEXT: v_mov_b32_e32 v2, s213760; GCN2-NEXT: v_mov_b32_e32 v3, s313761; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]13762; GCN2-NEXT: s_endpgm13763;13764; GFX12-LABEL: atomic_load_f64_addr64:13765; GFX12: ; %bb.0: ; %entry13766; GFX12-NEXT: s_clause 0x113767; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x3413768; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2413769; GFX12-NEXT: s_wait_kmcnt 0x013770; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 313771; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s313772; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]13773; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13774; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s113775; GFX12-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_SYS13776; GFX12-NEXT: s_wait_loadcnt_dscnt 0x013777; GFX12-NEXT: global_inv scope:SCOPE_SYS13778; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]13779; GFX12-NEXT: s_endpgm13780entry:13781 %ptr = getelementptr inbounds double, ptr %in, i64 %index13782 %val = load atomic double, ptr %ptr seq_cst, align 813783 store double %val, ptr %out13784 ret void13785}13786 13787define amdgpu_kernel void @atomic_store_f64_offset(double %in, ptr %out) {13788; GCN1-LABEL: atomic_store_f64_offset:13789; GCN1: ; %bb.0: ; %entry13790; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x913791; GCN1-NEXT: s_waitcnt lgkmcnt(0)13792; GCN1-NEXT: v_mov_b32_e32 v0, s013793; GCN1-NEXT: s_add_u32 s0, s2, 3213794; GCN1-NEXT: v_mov_b32_e32 v1, s113795; GCN1-NEXT: s_addc_u32 s1, s3, 013796; GCN1-NEXT: v_mov_b32_e32 v3, s113797; GCN1-NEXT: v_mov_b32_e32 v2, s013798; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]13799; GCN1-NEXT: s_endpgm13800;13801; GCN2-LABEL: atomic_store_f64_offset:13802; GCN2: ; %bb.0: ; %entry13803; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2413804; GCN2-NEXT: s_waitcnt lgkmcnt(0)13805; GCN2-NEXT: v_mov_b32_e32 v0, s013806; GCN2-NEXT: s_add_u32 s0, s2, 3213807; GCN2-NEXT: v_mov_b32_e32 v1, s113808; GCN2-NEXT: s_addc_u32 s1, s3, 013809; GCN2-NEXT: v_mov_b32_e32 v3, s113810; GCN2-NEXT: v_mov_b32_e32 v2, s013811; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]13812; GCN2-NEXT: s_endpgm13813;13814; GFX12-LABEL: atomic_store_f64_offset:13815; GFX12: ; %bb.0: ; %entry13816; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2413817; GFX12-NEXT: s_wait_kmcnt 0x013818; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s113819; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s313820; GFX12-NEXT: global_wb scope:SCOPE_SYS13821; GFX12-NEXT: s_wait_storecnt 0x013822; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1] offset:32 scope:SCOPE_SYS13823; GFX12-NEXT: s_endpgm13824entry:13825 %gep = getelementptr inbounds double, ptr %out, i64 413826 store atomic double %in, ptr %gep seq_cst, align 813827 ret void13828}13829 13830define amdgpu_kernel void @atomic_store_f64(double %in, ptr %out) {13831; GCN1-LABEL: atomic_store_f64:13832; GCN1: ; %bb.0: ; %entry13833; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x913834; GCN1-NEXT: s_waitcnt lgkmcnt(0)13835; GCN1-NEXT: v_mov_b32_e32 v0, s013836; GCN1-NEXT: v_mov_b32_e32 v1, s113837; GCN1-NEXT: v_mov_b32_e32 v2, s213838; GCN1-NEXT: v_mov_b32_e32 v3, s313839; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]13840; GCN1-NEXT: s_endpgm13841;13842; GCN2-LABEL: atomic_store_f64:13843; GCN2: ; %bb.0: ; %entry13844; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2413845; GCN2-NEXT: s_waitcnt lgkmcnt(0)13846; GCN2-NEXT: v_mov_b32_e32 v0, s013847; GCN2-NEXT: v_mov_b32_e32 v1, s113848; GCN2-NEXT: v_mov_b32_e32 v2, s213849; GCN2-NEXT: v_mov_b32_e32 v3, s313850; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]13851; GCN2-NEXT: s_endpgm13852;13853; GFX12-LABEL: atomic_store_f64:13854; GFX12: ; %bb.0: ; %entry13855; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2413856; GFX12-NEXT: s_wait_kmcnt 0x013857; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s113858; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s313859; GFX12-NEXT: global_wb scope:SCOPE_SYS13860; GFX12-NEXT: s_wait_storecnt 0x013861; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_SYS13862; GFX12-NEXT: s_endpgm13863entry:13864 store atomic double %in, ptr %out seq_cst, align 813865 ret void13866}13867 13868define amdgpu_kernel void @atomic_store_f64_addr64_offset(double %in, ptr %out, i64 %index) {13869; GCN1-LABEL: atomic_store_f64_addr64_offset:13870; GCN1: ; %bb.0: ; %entry13871; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x913872; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd13873; GCN1-NEXT: s_waitcnt lgkmcnt(0)13874; GCN1-NEXT: v_mov_b32_e32 v0, s013875; GCN1-NEXT: v_mov_b32_e32 v1, s113876; GCN1-NEXT: s_lshl_b64 s[0:1], s[4:5], 313877; GCN1-NEXT: s_add_u32 s0, s2, s013878; GCN1-NEXT: s_addc_u32 s1, s3, s113879; GCN1-NEXT: s_add_u32 s0, s0, 3213880; GCN1-NEXT: s_addc_u32 s1, s1, 013881; GCN1-NEXT: v_mov_b32_e32 v3, s113882; GCN1-NEXT: v_mov_b32_e32 v2, s013883; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]13884; GCN1-NEXT: s_endpgm13885;13886; GCN2-LABEL: atomic_store_f64_addr64_offset:13887; GCN2: ; %bb.0: ; %entry13888; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2413889; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x3413890; GCN2-NEXT: s_waitcnt lgkmcnt(0)13891; GCN2-NEXT: v_mov_b32_e32 v0, s013892; GCN2-NEXT: v_mov_b32_e32 v1, s113893; GCN2-NEXT: s_lshl_b64 s[0:1], s[4:5], 313894; GCN2-NEXT: s_add_u32 s0, s2, s013895; GCN2-NEXT: s_addc_u32 s1, s3, s113896; GCN2-NEXT: s_add_u32 s0, s0, 3213897; GCN2-NEXT: s_addc_u32 s1, s1, 013898; GCN2-NEXT: v_mov_b32_e32 v3, s113899; GCN2-NEXT: v_mov_b32_e32 v2, s013900; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]13901; GCN2-NEXT: s_endpgm13902;13903; GFX12-LABEL: atomic_store_f64_addr64_offset:13904; GFX12: ; %bb.0: ; %entry13905; GFX12-NEXT: s_clause 0x113906; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x3413907; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2413908; GFX12-NEXT: s_wait_kmcnt 0x013909; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 313910; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s113911; GFX12-NEXT: s_add_nc_u64 s[2:3], s[2:3], s[4:5]13912; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13913; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s313914; GFX12-NEXT: global_wb scope:SCOPE_SYS13915; GFX12-NEXT: s_wait_storecnt 0x013916; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1] offset:32 scope:SCOPE_SYS13917; GFX12-NEXT: s_endpgm13918entry:13919 %ptr = getelementptr inbounds double, ptr %out, i64 %index13920 %gep = getelementptr inbounds double, ptr %ptr, i64 413921 store atomic double %in, ptr %gep seq_cst, align 813922 ret void13923}13924 13925define amdgpu_kernel void @atomic_store_f64_addr64(double %in, ptr %out, i64 %index) {13926; GCN1-LABEL: atomic_store_f64_addr64:13927; GCN1: ; %bb.0: ; %entry13928; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x913929; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd13930; GCN1-NEXT: s_waitcnt lgkmcnt(0)13931; GCN1-NEXT: v_mov_b32_e32 v0, s013932; GCN1-NEXT: v_mov_b32_e32 v1, s113933; GCN1-NEXT: s_lshl_b64 s[0:1], s[4:5], 313934; GCN1-NEXT: s_add_u32 s0, s2, s013935; GCN1-NEXT: s_addc_u32 s1, s3, s113936; GCN1-NEXT: v_mov_b32_e32 v3, s113937; GCN1-NEXT: v_mov_b32_e32 v2, s013938; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]13939; GCN1-NEXT: s_endpgm13940;13941; GCN2-LABEL: atomic_store_f64_addr64:13942; GCN2: ; %bb.0: ; %entry13943; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2413944; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x3413945; GCN2-NEXT: s_waitcnt lgkmcnt(0)13946; GCN2-NEXT: v_mov_b32_e32 v0, s013947; GCN2-NEXT: v_mov_b32_e32 v1, s113948; GCN2-NEXT: s_lshl_b64 s[0:1], s[4:5], 313949; GCN2-NEXT: s_add_u32 s0, s2, s013950; GCN2-NEXT: s_addc_u32 s1, s3, s113951; GCN2-NEXT: v_mov_b32_e32 v3, s113952; GCN2-NEXT: v_mov_b32_e32 v2, s013953; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]13954; GCN2-NEXT: s_endpgm13955;13956; GFX12-LABEL: atomic_store_f64_addr64:13957; GFX12: ; %bb.0: ; %entry13958; GFX12-NEXT: s_clause 0x113959; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x3413960; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2413961; GFX12-NEXT: s_wait_kmcnt 0x013962; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 313963; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s113964; GFX12-NEXT: s_add_nc_u64 s[2:3], s[2:3], s[4:5]13965; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13966; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s313967; GFX12-NEXT: global_wb scope:SCOPE_SYS13968; GFX12-NEXT: s_wait_storecnt 0x013969; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1] scope:SCOPE_SYS13970; GFX12-NEXT: s_endpgm13971entry:13972 %ptr = getelementptr inbounds double, ptr %out, i64 %index13973 store atomic double %in, ptr %ptr seq_cst, align 813974 ret void13975}13976 13977define amdgpu_kernel void @atomic_inc_i64_offset(ptr %out, i64 %in) {13978; GCN1-LABEL: atomic_inc_i64_offset:13979; GCN1: ; %bb.0: ; %entry13980; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD013981; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD113982; GCN1-NEXT: s_mov_b32 s14, -113983; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x913984; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d13985; GCN1-NEXT: s_mov_b32 s15, 0xe8f00013986; GCN1-NEXT: s_add_u32 s12, s12, s1113987; GCN1-NEXT: s_addc_u32 s13, s13, 013988; GCN1-NEXT: s_waitcnt lgkmcnt(0)13989; GCN1-NEXT: s_add_u32 s0, s0, 3213990; GCN1-NEXT: s_addc_u32 s1, s1, 013991; GCN1-NEXT: s_cmp_eq_u32 s1, s413992; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 013993; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]13994; GCN1-NEXT: s_mov_b64 s[4:5], -113995; GCN1-NEXT: s_cbranch_vccnz .LBB107_313996; GCN1-NEXT: ; %bb.1: ; %Flow13997; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]13998; GCN1-NEXT: s_cbranch_vccz .LBB107_413999; GCN1-NEXT: .LBB107_2: ; %atomicrmw.phi14000; GCN1-NEXT: s_endpgm14001; GCN1-NEXT: .LBB107_3: ; %atomicrmw.global14002; GCN1-NEXT: v_mov_b32_e32 v0, s014003; GCN1-NEXT: v_mov_b32_e32 v2, s214004; GCN1-NEXT: v_mov_b32_e32 v1, s114005; GCN1-NEXT: v_mov_b32_e32 v3, s314006; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]14007; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14008; GCN1-NEXT: buffer_wbinvl1_vol14009; GCN1-NEXT: s_cbranch_execnz .LBB107_214010; GCN1-NEXT: .LBB107_4: ; %atomicrmw.private14011; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 014012; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec14013; GCN1-NEXT: s_cselect_b32 s0, s0, -114014; GCN1-NEXT: v_mov_b32_e32 v2, s014015; GCN1-NEXT: s_add_i32 s0, s0, 414016; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen14017; GCN1-NEXT: v_mov_b32_e32 v3, s014018; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen14019; GCN1-NEXT: s_waitcnt vmcnt(1)14020; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v014021; GCN1-NEXT: s_waitcnt vmcnt(0)14022; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc14023; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]14024; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v4, vcc14025; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v5, vcc14026; GCN1-NEXT: buffer_store_dword v1, v2, s[12:15], 0 offen14027; GCN1-NEXT: buffer_store_dword v0, v3, s[12:15], 0 offen14028; GCN1-NEXT: s_endpgm14029;14030; GCN2-LABEL: atomic_inc_i64_offset:14031; GCN2: ; %bb.0: ; %entry14032; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD014033; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD114034; GCN2-NEXT: s_mov_b32 s90, -114035; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2414036; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf414037; GCN2-NEXT: s_mov_b32 s91, 0xe8000014038; GCN2-NEXT: s_add_u32 s88, s88, s1114039; GCN2-NEXT: s_addc_u32 s89, s89, 014040; GCN2-NEXT: s_waitcnt lgkmcnt(0)14041; GCN2-NEXT: s_add_u32 s0, s0, 3214042; GCN2-NEXT: s_addc_u32 s1, s1, 014043; GCN2-NEXT: s_cmp_eq_u32 s1, s414044; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 014045; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]14046; GCN2-NEXT: s_mov_b64 s[4:5], -114047; GCN2-NEXT: s_cbranch_vccnz .LBB107_314048; GCN2-NEXT: ; %bb.1: ; %Flow14049; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]14050; GCN2-NEXT: s_cbranch_vccz .LBB107_414051; GCN2-NEXT: .LBB107_2: ; %atomicrmw.phi14052; GCN2-NEXT: s_endpgm14053; GCN2-NEXT: .LBB107_3: ; %atomicrmw.global14054; GCN2-NEXT: v_mov_b32_e32 v0, s014055; GCN2-NEXT: v_mov_b32_e32 v2, s214056; GCN2-NEXT: v_mov_b32_e32 v1, s114057; GCN2-NEXT: v_mov_b32_e32 v3, s314058; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]14059; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14060; GCN2-NEXT: buffer_wbinvl1_vol14061; GCN2-NEXT: s_cbranch_execnz .LBB107_214062; GCN2-NEXT: .LBB107_4: ; %atomicrmw.private14063; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 014064; GCN2-NEXT: s_cselect_b32 s0, s0, -114065; GCN2-NEXT: v_mov_b32_e32 v2, s014066; GCN2-NEXT: s_add_i32 s0, s0, 414067; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen14068; GCN2-NEXT: v_mov_b32_e32 v3, s014069; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen14070; GCN2-NEXT: s_waitcnt vmcnt(1)14071; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v014072; GCN2-NEXT: s_waitcnt vmcnt(0)14073; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc14074; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]14075; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v4, vcc14076; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v5, vcc14077; GCN2-NEXT: buffer_store_dword v1, v2, s[88:91], 0 offen14078; GCN2-NEXT: buffer_store_dword v0, v3, s[88:91], 0 offen14079; GCN2-NEXT: s_endpgm14080;14081; GFX12-LABEL: atomic_inc_i64_offset:14082; GFX12: ; %bb.0: ; %entry14083; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2414084; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base14085; GFX12-NEXT: s_wait_kmcnt 0x014086; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 3214087; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)14088; GFX12-NEXT: s_cmp_eq_u32 s1, s714089; GFX12-NEXT: s_cselect_b32 s4, -1, 014090; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s414091; GFX12-NEXT: s_mov_b32 s4, -114092; GFX12-NEXT: s_cbranch_vccnz .LBB107_314093; GFX12-NEXT: ; %bb.1: ; %Flow14094; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s414095; GFX12-NEXT: s_cbranch_vccz .LBB107_414096; GFX12-NEXT: .LBB107_2: ; %atomicrmw.phi14097; GFX12-NEXT: s_endpgm14098; GFX12-NEXT: .LBB107_3: ; %atomicrmw.global14099; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s114100; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s314101; GFX12-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3] scope:SCOPE_DEV14102; GFX12-NEXT: s_wait_storecnt_dscnt 0x014103; GFX12-NEXT: global_inv scope:SCOPE_DEV14104; GFX12-NEXT: s_cbranch_execnz .LBB107_214105; GFX12-NEXT: .LBB107_4: ; %atomicrmw.private14106; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 014107; GFX12-NEXT: s_cselect_b32 s0, s0, -114108; GFX12-NEXT: scratch_load_b64 v[0:1], off, s014109; GFX12-NEXT: s_wait_loadcnt 0x014110; GFX12-NEXT: v_add_co_u32 v2, vcc_lo, v0, 114111; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)14112; GFX12-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo14113; GFX12-NEXT: v_cmp_gt_u64_e32 vcc_lo, s[2:3], v[0:1]14114; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)14115; GFX12-NEXT: v_dual_cndmask_b32 v1, 0, v3 :: v_dual_cndmask_b32 v0, 0, v214116; GFX12-NEXT: scratch_store_b64 off, v[0:1], s014117; GFX12-NEXT: s_endpgm14118entry:14119 %gep = getelementptr inbounds i64, ptr %out, i64 414120 %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !014121 ret void14122}14123 14124define amdgpu_kernel void @atomic_inc_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {14125; GCN1-LABEL: atomic_inc_i64_ret_offset:14126; GCN1: ; %bb.0: ; %entry14127; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD014128; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD114129; GCN1-NEXT: s_mov_b32 s14, -114130; GCN1-NEXT: s_mov_b32 s15, 0xe8f00014131; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x914132; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f14133; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd14134; GCN1-NEXT: s_add_u32 s12, s12, s1114135; GCN1-NEXT: s_addc_u32 s13, s13, 014136; GCN1-NEXT: s_waitcnt lgkmcnt(0)14137; GCN1-NEXT: s_add_u32 s0, s0, 3214138; GCN1-NEXT: s_addc_u32 s1, s1, 014139; GCN1-NEXT: s_cmp_eq_u32 s1, s614140; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 014141; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]14142; GCN1-NEXT: s_cbranch_vccz .LBB108_214143; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global14144; GCN1-NEXT: v_mov_b32_e32 v0, s014145; GCN1-NEXT: v_mov_b32_e32 v2, s414146; GCN1-NEXT: v_mov_b32_e32 v1, s114147; GCN1-NEXT: v_mov_b32_e32 v3, s514148; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc14149; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14150; GCN1-NEXT: buffer_wbinvl1_vol14151; GCN1-NEXT: s_cbranch_execz .LBB108_314152; GCN1-NEXT: s_branch .LBB108_414153; GCN1-NEXT: .LBB108_2:14154; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr114155; GCN1-NEXT: .LBB108_3: ; %atomicrmw.private14156; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 014157; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec14158; GCN1-NEXT: s_cselect_b32 s0, s0, -114159; GCN1-NEXT: v_mov_b32_e32 v2, s014160; GCN1-NEXT: s_add_i32 s0, s0, 414161; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen14162; GCN1-NEXT: v_mov_b32_e32 v3, s014163; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen14164; GCN1-NEXT: s_waitcnt vmcnt(1)14165; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v014166; GCN1-NEXT: s_waitcnt vmcnt(0)14167; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc14168; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[0:1]14169; GCN1-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc14170; GCN1-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc14171; GCN1-NEXT: buffer_store_dword v4, v2, s[12:15], 0 offen14172; GCN1-NEXT: buffer_store_dword v5, v3, s[12:15], 0 offen14173; GCN1-NEXT: .LBB108_4: ; %atomicrmw.end14174; GCN1-NEXT: v_mov_b32_e32 v2, s214175; GCN1-NEXT: v_mov_b32_e32 v3, s314176; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]14177; GCN1-NEXT: s_endpgm14178;14179; GCN2-LABEL: atomic_inc_i64_ret_offset:14180; GCN2: ; %bb.0: ; %entry14181; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD014182; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD114183; GCN2-NEXT: s_mov_b32 s90, -114184; GCN2-NEXT: s_mov_b32 s91, 0xe8000014185; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2414186; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc14187; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x3414188; GCN2-NEXT: s_add_u32 s88, s88, s1114189; GCN2-NEXT: s_addc_u32 s89, s89, 014190; GCN2-NEXT: s_waitcnt lgkmcnt(0)14191; GCN2-NEXT: s_add_u32 s0, s0, 3214192; GCN2-NEXT: s_addc_u32 s1, s1, 014193; GCN2-NEXT: s_cmp_eq_u32 s1, s614194; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 014195; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]14196; GCN2-NEXT: s_cbranch_vccz .LBB108_214197; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global14198; GCN2-NEXT: v_mov_b32_e32 v0, s014199; GCN2-NEXT: v_mov_b32_e32 v2, s414200; GCN2-NEXT: v_mov_b32_e32 v1, s114201; GCN2-NEXT: v_mov_b32_e32 v3, s514202; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc14203; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14204; GCN2-NEXT: buffer_wbinvl1_vol14205; GCN2-NEXT: s_cbranch_execz .LBB108_314206; GCN2-NEXT: s_branch .LBB108_414207; GCN2-NEXT: .LBB108_2:14208; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr114209; GCN2-NEXT: .LBB108_3: ; %atomicrmw.private14210; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 014211; GCN2-NEXT: s_cselect_b32 s0, s0, -114212; GCN2-NEXT: v_mov_b32_e32 v2, s014213; GCN2-NEXT: s_add_i32 s0, s0, 414214; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen14215; GCN2-NEXT: v_mov_b32_e32 v3, s014216; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen14217; GCN2-NEXT: s_waitcnt vmcnt(1)14218; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v014219; GCN2-NEXT: s_waitcnt vmcnt(0)14220; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc14221; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[0:1]14222; GCN2-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc14223; GCN2-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc14224; GCN2-NEXT: buffer_store_dword v4, v2, s[88:91], 0 offen14225; GCN2-NEXT: buffer_store_dword v5, v3, s[88:91], 0 offen14226; GCN2-NEXT: .LBB108_4: ; %atomicrmw.end14227; GCN2-NEXT: v_mov_b32_e32 v2, s214228; GCN2-NEXT: v_mov_b32_e32 v3, s314229; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]14230; GCN2-NEXT: s_endpgm14231;14232; GFX12-LABEL: atomic_inc_i64_ret_offset:14233; GFX12: ; %bb.0: ; %entry14234; GFX12-NEXT: s_clause 0x114235; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2414236; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x3414237; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base14238; GFX12-NEXT: s_wait_kmcnt 0x014239; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 3214240; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)14241; GFX12-NEXT: s_cmp_eq_u32 s1, s714242; GFX12-NEXT: s_cselect_b32 s6, -1, 014243; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s614244; GFX12-NEXT: s_cbranch_vccz .LBB108_214245; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global14246; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s114247; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s514248; GFX12-NEXT: flat_atomic_inc_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV14249; GFX12-NEXT: s_wait_loadcnt_dscnt 0x014250; GFX12-NEXT: global_inv scope:SCOPE_DEV14251; GFX12-NEXT: s_cbranch_execz .LBB108_314252; GFX12-NEXT: s_branch .LBB108_414253; GFX12-NEXT: .LBB108_2:14254; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr114255; GFX12-NEXT: .LBB108_3: ; %atomicrmw.private14256; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 014257; GFX12-NEXT: s_cselect_b32 s0, s0, -114258; GFX12-NEXT: scratch_load_b64 v[0:1], off, s014259; GFX12-NEXT: s_wait_loadcnt 0x014260; GFX12-NEXT: v_add_co_u32 v2, vcc_lo, v0, 114261; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)14262; GFX12-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo14263; GFX12-NEXT: v_cmp_gt_u64_e32 vcc_lo, s[4:5], v[0:1]14264; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)14265; GFX12-NEXT: v_dual_cndmask_b32 v3, 0, v3 :: v_dual_cndmask_b32 v2, 0, v214266; GFX12-NEXT: scratch_store_b64 off, v[2:3], s014267; GFX12-NEXT: .LBB108_4: ; %atomicrmw.end14268; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s314269; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]14270; GFX12-NEXT: s_endpgm14271entry:14272 %gep = getelementptr inbounds i64, ptr %out, i64 414273 %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !014274 store i64 %tmp0, ptr %out214275 ret void14276}14277 14278define amdgpu_kernel void @atomic_inc_i64_incr64_offset(ptr %out, i64 %in, i64 %index) {14279; GCN1-LABEL: atomic_inc_i64_incr64_offset:14280; GCN1: ; %bb.0: ; %entry14281; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD014282; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD114283; GCN1-NEXT: s_mov_b32 s14, -114284; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd14285; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x914286; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f14287; GCN1-NEXT: s_mov_b32 s15, 0xe8f00014288; GCN1-NEXT: s_add_u32 s12, s12, s1114289; GCN1-NEXT: s_addc_u32 s13, s13, 014290; GCN1-NEXT: s_waitcnt lgkmcnt(0)14291; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 314292; GCN1-NEXT: s_add_u32 s0, s0, s414293; GCN1-NEXT: s_addc_u32 s1, s1, s514294; GCN1-NEXT: s_add_u32 s0, s0, 3214295; GCN1-NEXT: s_addc_u32 s1, s1, 014296; GCN1-NEXT: s_cmp_eq_u32 s1, s814297; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 014298; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]14299; GCN1-NEXT: s_mov_b64 s[4:5], -114300; GCN1-NEXT: s_cbranch_vccnz .LBB109_314301; GCN1-NEXT: ; %bb.1: ; %Flow14302; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]14303; GCN1-NEXT: s_cbranch_vccz .LBB109_414304; GCN1-NEXT: .LBB109_2: ; %atomicrmw.phi14305; GCN1-NEXT: s_endpgm14306; GCN1-NEXT: .LBB109_3: ; %atomicrmw.global14307; GCN1-NEXT: v_mov_b32_e32 v0, s014308; GCN1-NEXT: v_mov_b32_e32 v2, s214309; GCN1-NEXT: v_mov_b32_e32 v1, s114310; GCN1-NEXT: v_mov_b32_e32 v3, s314311; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]14312; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14313; GCN1-NEXT: buffer_wbinvl1_vol14314; GCN1-NEXT: s_cbranch_execnz .LBB109_214315; GCN1-NEXT: .LBB109_4: ; %atomicrmw.private14316; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 014317; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec14318; GCN1-NEXT: s_cselect_b32 s0, s0, -114319; GCN1-NEXT: v_mov_b32_e32 v2, s014320; GCN1-NEXT: s_add_i32 s0, s0, 414321; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen14322; GCN1-NEXT: v_mov_b32_e32 v3, s014323; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen14324; GCN1-NEXT: s_waitcnt vmcnt(1)14325; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v014326; GCN1-NEXT: s_waitcnt vmcnt(0)14327; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc14328; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]14329; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v4, vcc14330; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v5, vcc14331; GCN1-NEXT: buffer_store_dword v1, v2, s[12:15], 0 offen14332; GCN1-NEXT: buffer_store_dword v0, v3, s[12:15], 0 offen14333; GCN1-NEXT: s_endpgm14334;14335; GCN2-LABEL: atomic_inc_i64_incr64_offset:14336; GCN2: ; %bb.0: ; %entry14337; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD014338; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD114339; GCN2-NEXT: s_mov_b32 s90, -114340; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3414341; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2414342; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc14343; GCN2-NEXT: s_mov_b32 s91, 0xe8000014344; GCN2-NEXT: s_add_u32 s88, s88, s1114345; GCN2-NEXT: s_addc_u32 s89, s89, 014346; GCN2-NEXT: s_waitcnt lgkmcnt(0)14347; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 314348; GCN2-NEXT: s_add_u32 s0, s0, s414349; GCN2-NEXT: s_addc_u32 s1, s1, s514350; GCN2-NEXT: s_add_u32 s0, s0, 3214351; GCN2-NEXT: s_addc_u32 s1, s1, 014352; GCN2-NEXT: s_cmp_eq_u32 s1, s814353; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 014354; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]14355; GCN2-NEXT: s_mov_b64 s[4:5], -114356; GCN2-NEXT: s_cbranch_vccnz .LBB109_314357; GCN2-NEXT: ; %bb.1: ; %Flow14358; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]14359; GCN2-NEXT: s_cbranch_vccz .LBB109_414360; GCN2-NEXT: .LBB109_2: ; %atomicrmw.phi14361; GCN2-NEXT: s_endpgm14362; GCN2-NEXT: .LBB109_3: ; %atomicrmw.global14363; GCN2-NEXT: v_mov_b32_e32 v0, s014364; GCN2-NEXT: v_mov_b32_e32 v2, s214365; GCN2-NEXT: v_mov_b32_e32 v1, s114366; GCN2-NEXT: v_mov_b32_e32 v3, s314367; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]14368; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14369; GCN2-NEXT: buffer_wbinvl1_vol14370; GCN2-NEXT: s_cbranch_execnz .LBB109_214371; GCN2-NEXT: .LBB109_4: ; %atomicrmw.private14372; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 014373; GCN2-NEXT: s_cselect_b32 s0, s0, -114374; GCN2-NEXT: v_mov_b32_e32 v2, s014375; GCN2-NEXT: s_add_i32 s0, s0, 414376; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen14377; GCN2-NEXT: v_mov_b32_e32 v3, s014378; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen14379; GCN2-NEXT: s_waitcnt vmcnt(1)14380; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v014381; GCN2-NEXT: s_waitcnt vmcnt(0)14382; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc14383; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]14384; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v4, vcc14385; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v5, vcc14386; GCN2-NEXT: buffer_store_dword v1, v2, s[88:91], 0 offen14387; GCN2-NEXT: buffer_store_dword v0, v3, s[88:91], 0 offen14388; GCN2-NEXT: s_endpgm14389;14390; GFX12-LABEL: atomic_inc_i64_incr64_offset:14391; GFX12: ; %bb.0: ; %entry14392; GFX12-NEXT: s_clause 0x114393; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x3414394; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2414395; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base14396; GFX12-NEXT: s_wait_kmcnt 0x014397; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 314398; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)14399; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]14400; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 3214401; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)14402; GFX12-NEXT: s_cmp_eq_u32 s1, s714403; GFX12-NEXT: s_cselect_b32 s4, -1, 014404; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s414405; GFX12-NEXT: s_mov_b32 s4, -114406; GFX12-NEXT: s_cbranch_vccnz .LBB109_314407; GFX12-NEXT: ; %bb.1: ; %Flow14408; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s414409; GFX12-NEXT: s_cbranch_vccz .LBB109_414410; GFX12-NEXT: .LBB109_2: ; %atomicrmw.phi14411; GFX12-NEXT: s_endpgm14412; GFX12-NEXT: .LBB109_3: ; %atomicrmw.global14413; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s114414; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s314415; GFX12-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3] scope:SCOPE_DEV14416; GFX12-NEXT: s_wait_storecnt_dscnt 0x014417; GFX12-NEXT: global_inv scope:SCOPE_DEV14418; GFX12-NEXT: s_cbranch_execnz .LBB109_214419; GFX12-NEXT: .LBB109_4: ; %atomicrmw.private14420; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 014421; GFX12-NEXT: s_cselect_b32 s0, s0, -114422; GFX12-NEXT: scratch_load_b64 v[0:1], off, s014423; GFX12-NEXT: s_wait_loadcnt 0x014424; GFX12-NEXT: v_add_co_u32 v2, vcc_lo, v0, 114425; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)14426; GFX12-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo14427; GFX12-NEXT: v_cmp_gt_u64_e32 vcc_lo, s[2:3], v[0:1]14428; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)14429; GFX12-NEXT: v_dual_cndmask_b32 v1, 0, v3 :: v_dual_cndmask_b32 v0, 0, v214430; GFX12-NEXT: scratch_store_b64 off, v[0:1], s014431; GFX12-NEXT: s_endpgm14432entry:14433 %ptr = getelementptr inbounds i64, ptr %out, i64 %index14434 %gep = getelementptr inbounds i64, ptr %ptr, i64 414435 %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !014436 ret void14437}14438 14439define amdgpu_kernel void @atomic_inc_i64_ret_incr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {14440; GCN1-LABEL: atomic_inc_i64_ret_incr64_offset:14441; GCN1: ; %bb.0: ; %entry14442; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD014443; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD114444; GCN1-NEXT: s_mov_b32 s18, -114445; GCN1-NEXT: s_mov_b32 s19, 0xe8f00014446; GCN1-NEXT: s_add_u32 s16, s16, s1114447; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x914448; GCN1-NEXT: s_load_dword s2, s[4:5], 0x4114449; GCN1-NEXT: s_addc_u32 s17, s17, 014450; GCN1-NEXT: s_waitcnt lgkmcnt(0)14451; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 314452; GCN1-NEXT: s_add_u32 s0, s8, s014453; GCN1-NEXT: s_addc_u32 s1, s9, s114454; GCN1-NEXT: s_add_u32 s0, s0, 3214455; GCN1-NEXT: s_addc_u32 s1, s1, 014456; GCN1-NEXT: s_cmp_eq_u32 s1, s214457; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 014458; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]14459; GCN1-NEXT: s_cbranch_vccz .LBB110_214460; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global14461; GCN1-NEXT: v_mov_b32_e32 v0, s014462; GCN1-NEXT: v_mov_b32_e32 v2, s1214463; GCN1-NEXT: v_mov_b32_e32 v1, s114464; GCN1-NEXT: v_mov_b32_e32 v3, s1314465; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc14466; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14467; GCN1-NEXT: buffer_wbinvl1_vol14468; GCN1-NEXT: s_cbranch_execz .LBB110_314469; GCN1-NEXT: s_branch .LBB110_414470; GCN1-NEXT: .LBB110_2:14471; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr114472; GCN1-NEXT: .LBB110_3: ; %atomicrmw.private14473; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 014474; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec14475; GCN1-NEXT: s_cselect_b32 s0, s0, -114476; GCN1-NEXT: v_mov_b32_e32 v2, s014477; GCN1-NEXT: s_add_i32 s0, s0, 414478; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen14479; GCN1-NEXT: v_mov_b32_e32 v3, s014480; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen14481; GCN1-NEXT: s_waitcnt vmcnt(1)14482; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v014483; GCN1-NEXT: s_waitcnt vmcnt(0)14484; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc14485; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[12:13], v[0:1]14486; GCN1-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc14487; GCN1-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc14488; GCN1-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen14489; GCN1-NEXT: buffer_store_dword v5, v3, s[16:19], 0 offen14490; GCN1-NEXT: .LBB110_4: ; %atomicrmw.end14491; GCN1-NEXT: v_mov_b32_e32 v2, s1014492; GCN1-NEXT: v_mov_b32_e32 v3, s1114493; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]14494; GCN1-NEXT: s_endpgm14495;14496; GCN2-LABEL: atomic_inc_i64_ret_incr64_offset:14497; GCN2: ; %bb.0: ; %entry14498; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD014499; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD114500; GCN2-NEXT: s_mov_b32 s90, -114501; GCN2-NEXT: s_mov_b32 s91, 0xe8000014502; GCN2-NEXT: s_add_u32 s88, s88, s1114503; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2414504; GCN2-NEXT: s_load_dword s2, s[4:5], 0x10414505; GCN2-NEXT: s_addc_u32 s89, s89, 014506; GCN2-NEXT: s_waitcnt lgkmcnt(0)14507; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 314508; GCN2-NEXT: s_add_u32 s0, s8, s014509; GCN2-NEXT: s_addc_u32 s1, s9, s114510; GCN2-NEXT: s_add_u32 s0, s0, 3214511; GCN2-NEXT: s_addc_u32 s1, s1, 014512; GCN2-NEXT: s_cmp_eq_u32 s1, s214513; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 014514; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]14515; GCN2-NEXT: s_cbranch_vccz .LBB110_214516; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global14517; GCN2-NEXT: v_mov_b32_e32 v0, s014518; GCN2-NEXT: v_mov_b32_e32 v2, s1214519; GCN2-NEXT: v_mov_b32_e32 v1, s114520; GCN2-NEXT: v_mov_b32_e32 v3, s1314521; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc14522; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14523; GCN2-NEXT: buffer_wbinvl1_vol14524; GCN2-NEXT: s_cbranch_execz .LBB110_314525; GCN2-NEXT: s_branch .LBB110_414526; GCN2-NEXT: .LBB110_2:14527; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr114528; GCN2-NEXT: .LBB110_3: ; %atomicrmw.private14529; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 014530; GCN2-NEXT: s_cselect_b32 s0, s0, -114531; GCN2-NEXT: v_mov_b32_e32 v2, s014532; GCN2-NEXT: s_add_i32 s0, s0, 414533; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen14534; GCN2-NEXT: v_mov_b32_e32 v3, s014535; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen14536; GCN2-NEXT: s_waitcnt vmcnt(1)14537; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v014538; GCN2-NEXT: s_waitcnt vmcnt(0)14539; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc14540; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[12:13], v[0:1]14541; GCN2-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc14542; GCN2-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc14543; GCN2-NEXT: buffer_store_dword v4, v2, s[88:91], 0 offen14544; GCN2-NEXT: buffer_store_dword v5, v3, s[88:91], 0 offen14545; GCN2-NEXT: .LBB110_4: ; %atomicrmw.end14546; GCN2-NEXT: v_mov_b32_e32 v2, s1014547; GCN2-NEXT: v_mov_b32_e32 v3, s1114548; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]14549; GCN2-NEXT: s_endpgm14550;14551; GFX12-LABEL: atomic_inc_i64_ret_incr64_offset:14552; GFX12: ; %bb.0: ; %entry14553; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x2414554; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base14555; GFX12-NEXT: s_wait_kmcnt 0x014556; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 314557; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)14558; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]14559; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 3214560; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)14561; GFX12-NEXT: s_cmp_eq_u32 s1, s914562; GFX12-NEXT: s_cselect_b32 s6, -1, 014563; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s614564; GFX12-NEXT: s_cbranch_vccz .LBB110_214565; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global14566; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s114567; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s514568; GFX12-NEXT: flat_atomic_inc_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV14569; GFX12-NEXT: s_wait_loadcnt_dscnt 0x014570; GFX12-NEXT: global_inv scope:SCOPE_DEV14571; GFX12-NEXT: s_cbranch_execz .LBB110_314572; GFX12-NEXT: s_branch .LBB110_414573; GFX12-NEXT: .LBB110_2:14574; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr114575; GFX12-NEXT: .LBB110_3: ; %atomicrmw.private14576; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 014577; GFX12-NEXT: s_cselect_b32 s0, s0, -114578; GFX12-NEXT: scratch_load_b64 v[0:1], off, s014579; GFX12-NEXT: s_wait_loadcnt 0x014580; GFX12-NEXT: v_add_co_u32 v2, vcc_lo, v0, 114581; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)14582; GFX12-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo14583; GFX12-NEXT: v_cmp_gt_u64_e32 vcc_lo, s[4:5], v[0:1]14584; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)14585; GFX12-NEXT: v_dual_cndmask_b32 v3, 0, v3 :: v_dual_cndmask_b32 v2, 0, v214586; GFX12-NEXT: scratch_store_b64 off, v[2:3], s014587; GFX12-NEXT: .LBB110_4: ; %atomicrmw.end14588; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s314589; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]14590; GFX12-NEXT: s_endpgm14591entry:14592 %ptr = getelementptr inbounds i64, ptr %out, i64 %index14593 %gep = getelementptr inbounds i64, ptr %ptr, i64 414594 %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !014595 store i64 %tmp0, ptr %out214596 ret void14597}14598 14599define amdgpu_kernel void @atomic_inc_i64(ptr %out, i64 %in) {14600; GCN1-LABEL: atomic_inc_i64:14601; GCN1: ; %bb.0: ; %entry14602; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD014603; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD114604; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x914605; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d14606; GCN1-NEXT: s_mov_b32 s14, -114607; GCN1-NEXT: s_mov_b32 s15, 0xe8f00014608; GCN1-NEXT: s_add_u32 s12, s12, s1114609; GCN1-NEXT: s_addc_u32 s13, s13, 014610; GCN1-NEXT: s_waitcnt lgkmcnt(0)14611; GCN1-NEXT: s_cmp_eq_u32 s1, s414612; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 014613; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]14614; GCN1-NEXT: s_mov_b64 s[4:5], -114615; GCN1-NEXT: s_cbranch_vccnz .LBB111_314616; GCN1-NEXT: ; %bb.1: ; %Flow14617; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]14618; GCN1-NEXT: s_cbranch_vccz .LBB111_414619; GCN1-NEXT: .LBB111_2: ; %atomicrmw.phi14620; GCN1-NEXT: s_endpgm14621; GCN1-NEXT: .LBB111_3: ; %atomicrmw.global14622; GCN1-NEXT: v_mov_b32_e32 v0, s014623; GCN1-NEXT: v_mov_b32_e32 v2, s214624; GCN1-NEXT: v_mov_b32_e32 v1, s114625; GCN1-NEXT: v_mov_b32_e32 v3, s314626; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]14627; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14628; GCN1-NEXT: buffer_wbinvl1_vol14629; GCN1-NEXT: s_cbranch_execnz .LBB111_214630; GCN1-NEXT: .LBB111_4: ; %atomicrmw.private14631; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 014632; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec14633; GCN1-NEXT: s_cselect_b32 s0, s0, -114634; GCN1-NEXT: v_mov_b32_e32 v2, s014635; GCN1-NEXT: s_add_i32 s0, s0, 414636; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen14637; GCN1-NEXT: v_mov_b32_e32 v3, s014638; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen14639; GCN1-NEXT: s_waitcnt vmcnt(1)14640; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v014641; GCN1-NEXT: s_waitcnt vmcnt(0)14642; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc14643; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]14644; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v4, vcc14645; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v5, vcc14646; GCN1-NEXT: buffer_store_dword v1, v2, s[12:15], 0 offen14647; GCN1-NEXT: buffer_store_dword v0, v3, s[12:15], 0 offen14648; GCN1-NEXT: s_endpgm14649;14650; GCN2-LABEL: atomic_inc_i64:14651; GCN2: ; %bb.0: ; %entry14652; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD014653; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD114654; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2414655; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf414656; GCN2-NEXT: s_mov_b32 s90, -114657; GCN2-NEXT: s_mov_b32 s91, 0xe8000014658; GCN2-NEXT: s_add_u32 s88, s88, s1114659; GCN2-NEXT: s_addc_u32 s89, s89, 014660; GCN2-NEXT: s_waitcnt lgkmcnt(0)14661; GCN2-NEXT: s_cmp_eq_u32 s1, s414662; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 014663; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]14664; GCN2-NEXT: s_mov_b64 s[4:5], -114665; GCN2-NEXT: s_cbranch_vccnz .LBB111_314666; GCN2-NEXT: ; %bb.1: ; %Flow14667; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]14668; GCN2-NEXT: s_cbranch_vccz .LBB111_414669; GCN2-NEXT: .LBB111_2: ; %atomicrmw.phi14670; GCN2-NEXT: s_endpgm14671; GCN2-NEXT: .LBB111_3: ; %atomicrmw.global14672; GCN2-NEXT: v_mov_b32_e32 v0, s014673; GCN2-NEXT: v_mov_b32_e32 v2, s214674; GCN2-NEXT: v_mov_b32_e32 v1, s114675; GCN2-NEXT: v_mov_b32_e32 v3, s314676; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]14677; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14678; GCN2-NEXT: buffer_wbinvl1_vol14679; GCN2-NEXT: s_cbranch_execnz .LBB111_214680; GCN2-NEXT: .LBB111_4: ; %atomicrmw.private14681; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 014682; GCN2-NEXT: s_cselect_b32 s0, s0, -114683; GCN2-NEXT: v_mov_b32_e32 v2, s014684; GCN2-NEXT: s_add_i32 s0, s0, 414685; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen14686; GCN2-NEXT: v_mov_b32_e32 v3, s014687; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen14688; GCN2-NEXT: s_waitcnt vmcnt(1)14689; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v014690; GCN2-NEXT: s_waitcnt vmcnt(0)14691; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc14692; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]14693; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v4, vcc14694; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v5, vcc14695; GCN2-NEXT: buffer_store_dword v1, v2, s[88:91], 0 offen14696; GCN2-NEXT: buffer_store_dword v0, v3, s[88:91], 0 offen14697; GCN2-NEXT: s_endpgm14698;14699; GFX12-LABEL: atomic_inc_i64:14700; GFX12: ; %bb.0: ; %entry14701; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2414702; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base14703; GFX12-NEXT: s_wait_kmcnt 0x014704; GFX12-NEXT: s_cmp_eq_u32 s1, s714705; GFX12-NEXT: s_cselect_b32 s4, -1, 014706; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)14707; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s414708; GFX12-NEXT: s_mov_b32 s4, -114709; GFX12-NEXT: s_cbranch_vccnz .LBB111_314710; GFX12-NEXT: ; %bb.1: ; %Flow14711; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s414712; GFX12-NEXT: s_cbranch_vccz .LBB111_414713; GFX12-NEXT: .LBB111_2: ; %atomicrmw.phi14714; GFX12-NEXT: s_endpgm14715; GFX12-NEXT: .LBB111_3: ; %atomicrmw.global14716; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s114717; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s314718; GFX12-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3] scope:SCOPE_DEV14719; GFX12-NEXT: s_wait_storecnt_dscnt 0x014720; GFX12-NEXT: global_inv scope:SCOPE_DEV14721; GFX12-NEXT: s_cbranch_execnz .LBB111_214722; GFX12-NEXT: .LBB111_4: ; %atomicrmw.private14723; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 014724; GFX12-NEXT: s_cselect_b32 s0, s0, -114725; GFX12-NEXT: scratch_load_b64 v[0:1], off, s014726; GFX12-NEXT: s_wait_loadcnt 0x014727; GFX12-NEXT: v_add_co_u32 v2, vcc_lo, v0, 114728; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)14729; GFX12-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo14730; GFX12-NEXT: v_cmp_gt_u64_e32 vcc_lo, s[2:3], v[0:1]14731; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)14732; GFX12-NEXT: v_dual_cndmask_b32 v1, 0, v3 :: v_dual_cndmask_b32 v0, 0, v214733; GFX12-NEXT: scratch_store_b64 off, v[0:1], s014734; GFX12-NEXT: s_endpgm14735entry:14736 %tmp0 = atomicrmw volatile uinc_wrap ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !014737 ret void14738}14739 14740define amdgpu_kernel void @atomic_inc_i64_ret(ptr %out, ptr %out2, i64 %in) {14741; GCN1-LABEL: atomic_inc_i64_ret:14742; GCN1: ; %bb.0: ; %entry14743; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD014744; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD114745; GCN1-NEXT: s_mov_b32 s14, -114746; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f14747; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x914748; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd14749; GCN1-NEXT: s_mov_b32 s15, 0xe8f00014750; GCN1-NEXT: s_add_u32 s12, s12, s1114751; GCN1-NEXT: s_addc_u32 s13, s13, 014752; GCN1-NEXT: s_waitcnt lgkmcnt(0)14753; GCN1-NEXT: s_cmp_eq_u32 s1, s614754; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 014755; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]14756; GCN1-NEXT: s_cbranch_vccz .LBB112_214757; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global14758; GCN1-NEXT: v_mov_b32_e32 v0, s014759; GCN1-NEXT: v_mov_b32_e32 v2, s414760; GCN1-NEXT: v_mov_b32_e32 v1, s114761; GCN1-NEXT: v_mov_b32_e32 v3, s514762; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc14763; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14764; GCN1-NEXT: buffer_wbinvl1_vol14765; GCN1-NEXT: s_cbranch_execz .LBB112_314766; GCN1-NEXT: s_branch .LBB112_414767; GCN1-NEXT: .LBB112_2:14768; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr114769; GCN1-NEXT: .LBB112_3: ; %atomicrmw.private14770; GCN1-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 014771; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec14772; GCN1-NEXT: s_cselect_b32 s0, s0, -114773; GCN1-NEXT: v_mov_b32_e32 v2, s014774; GCN1-NEXT: s_add_i32 s0, s0, 414775; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen14776; GCN1-NEXT: v_mov_b32_e32 v3, s014777; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen14778; GCN1-NEXT: s_waitcnt vmcnt(1)14779; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v014780; GCN1-NEXT: s_waitcnt vmcnt(0)14781; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc14782; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[0:1]14783; GCN1-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc14784; GCN1-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc14785; GCN1-NEXT: buffer_store_dword v4, v2, s[12:15], 0 offen14786; GCN1-NEXT: buffer_store_dword v5, v3, s[12:15], 0 offen14787; GCN1-NEXT: .LBB112_4: ; %atomicrmw.end14788; GCN1-NEXT: v_mov_b32_e32 v2, s214789; GCN1-NEXT: v_mov_b32_e32 v3, s314790; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]14791; GCN1-NEXT: s_endpgm14792;14793; GCN2-LABEL: atomic_inc_i64_ret:14794; GCN2: ; %bb.0: ; %entry14795; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD014796; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD114797; GCN2-NEXT: s_mov_b32 s90, -114798; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc14799; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2414800; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x3414801; GCN2-NEXT: s_mov_b32 s91, 0xe8000014802; GCN2-NEXT: s_add_u32 s88, s88, s1114803; GCN2-NEXT: s_addc_u32 s89, s89, 014804; GCN2-NEXT: s_waitcnt lgkmcnt(0)14805; GCN2-NEXT: s_cmp_eq_u32 s1, s614806; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 014807; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]14808; GCN2-NEXT: s_cbranch_vccz .LBB112_214809; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global14810; GCN2-NEXT: v_mov_b32_e32 v0, s014811; GCN2-NEXT: v_mov_b32_e32 v2, s414812; GCN2-NEXT: v_mov_b32_e32 v1, s114813; GCN2-NEXT: v_mov_b32_e32 v3, s514814; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc14815; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14816; GCN2-NEXT: buffer_wbinvl1_vol14817; GCN2-NEXT: s_cbranch_execz .LBB112_314818; GCN2-NEXT: s_branch .LBB112_414819; GCN2-NEXT: .LBB112_2:14820; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr114821; GCN2-NEXT: .LBB112_3: ; %atomicrmw.private14822; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 014823; GCN2-NEXT: s_cselect_b32 s0, s0, -114824; GCN2-NEXT: v_mov_b32_e32 v2, s014825; GCN2-NEXT: s_add_i32 s0, s0, 414826; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen14827; GCN2-NEXT: v_mov_b32_e32 v3, s014828; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen14829; GCN2-NEXT: s_waitcnt vmcnt(1)14830; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v014831; GCN2-NEXT: s_waitcnt vmcnt(0)14832; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc14833; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[0:1]14834; GCN2-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc14835; GCN2-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc14836; GCN2-NEXT: buffer_store_dword v4, v2, s[88:91], 0 offen14837; GCN2-NEXT: buffer_store_dword v5, v3, s[88:91], 0 offen14838; GCN2-NEXT: .LBB112_4: ; %atomicrmw.end14839; GCN2-NEXT: v_mov_b32_e32 v2, s214840; GCN2-NEXT: v_mov_b32_e32 v3, s314841; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]14842; GCN2-NEXT: s_endpgm14843;14844; GFX12-LABEL: atomic_inc_i64_ret:14845; GFX12: ; %bb.0: ; %entry14846; GFX12-NEXT: s_clause 0x114847; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2414848; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x3414849; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base14850; GFX12-NEXT: s_wait_kmcnt 0x014851; GFX12-NEXT: s_cmp_eq_u32 s1, s714852; GFX12-NEXT: s_cselect_b32 s6, -1, 014853; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)14854; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s614855; GFX12-NEXT: s_cbranch_vccz .LBB112_214856; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global14857; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s114858; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s514859; GFX12-NEXT: flat_atomic_inc_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV14860; GFX12-NEXT: s_wait_loadcnt_dscnt 0x014861; GFX12-NEXT: global_inv scope:SCOPE_DEV14862; GFX12-NEXT: s_cbranch_execz .LBB112_314863; GFX12-NEXT: s_branch .LBB112_414864; GFX12-NEXT: .LBB112_2:14865; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr114866; GFX12-NEXT: .LBB112_3: ; %atomicrmw.private14867; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 014868; GFX12-NEXT: s_cselect_b32 s0, s0, -114869; GFX12-NEXT: scratch_load_b64 v[0:1], off, s014870; GFX12-NEXT: s_wait_loadcnt 0x014871; GFX12-NEXT: v_add_co_u32 v2, vcc_lo, v0, 114872; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)14873; GFX12-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo14874; GFX12-NEXT: v_cmp_gt_u64_e32 vcc_lo, s[4:5], v[0:1]14875; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)14876; GFX12-NEXT: v_dual_cndmask_b32 v3, 0, v3 :: v_dual_cndmask_b32 v2, 0, v214877; GFX12-NEXT: scratch_store_b64 off, v[2:3], s014878; GFX12-NEXT: .LBB112_4: ; %atomicrmw.end14879; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s314880; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]14881; GFX12-NEXT: s_endpgm14882entry:14883 %tmp0 = atomicrmw volatile uinc_wrap ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !014884 store i64 %tmp0, ptr %out214885 ret void14886}14887 14888define amdgpu_kernel void @atomic_inc_i64_incr64(ptr %out, i64 %in, i64 %index) {14889; GCN1-LABEL: atomic_inc_i64_incr64:14890; GCN1: ; %bb.0: ; %entry14891; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD014892; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD114893; GCN1-NEXT: s_mov_b32 s14, -114894; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd14895; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x914896; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f14897; GCN1-NEXT: s_mov_b32 s15, 0xe8f00014898; GCN1-NEXT: s_add_u32 s12, s12, s1114899; GCN1-NEXT: s_addc_u32 s13, s13, 014900; GCN1-NEXT: s_waitcnt lgkmcnt(0)14901; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 314902; GCN1-NEXT: s_add_u32 s0, s0, s414903; GCN1-NEXT: s_addc_u32 s1, s1, s514904; GCN1-NEXT: s_cmp_eq_u32 s1, s814905; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 014906; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]14907; GCN1-NEXT: s_mov_b64 s[4:5], -114908; GCN1-NEXT: s_cbranch_vccnz .LBB113_314909; GCN1-NEXT: ; %bb.1: ; %Flow14910; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]14911; GCN1-NEXT: s_cbranch_vccz .LBB113_414912; GCN1-NEXT: .LBB113_2: ; %atomicrmw.phi14913; GCN1-NEXT: s_endpgm14914; GCN1-NEXT: .LBB113_3: ; %atomicrmw.global14915; GCN1-NEXT: v_mov_b32_e32 v0, s014916; GCN1-NEXT: v_mov_b32_e32 v2, s214917; GCN1-NEXT: v_mov_b32_e32 v1, s114918; GCN1-NEXT: v_mov_b32_e32 v3, s314919; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]14920; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14921; GCN1-NEXT: buffer_wbinvl1_vol14922; GCN1-NEXT: s_cbranch_execnz .LBB113_214923; GCN1-NEXT: .LBB113_4: ; %atomicrmw.private14924; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 014925; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec14926; GCN1-NEXT: s_cselect_b32 s0, s0, -114927; GCN1-NEXT: v_mov_b32_e32 v2, s014928; GCN1-NEXT: s_add_i32 s0, s0, 414929; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen14930; GCN1-NEXT: v_mov_b32_e32 v3, s014931; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen14932; GCN1-NEXT: s_waitcnt vmcnt(1)14933; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v014934; GCN1-NEXT: s_waitcnt vmcnt(0)14935; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc14936; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]14937; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v4, vcc14938; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v5, vcc14939; GCN1-NEXT: buffer_store_dword v1, v2, s[12:15], 0 offen14940; GCN1-NEXT: buffer_store_dword v0, v3, s[12:15], 0 offen14941; GCN1-NEXT: s_endpgm14942;14943; GCN2-LABEL: atomic_inc_i64_incr64:14944; GCN2: ; %bb.0: ; %entry14945; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD014946; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD114947; GCN2-NEXT: s_mov_b32 s90, -114948; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3414949; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2414950; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc14951; GCN2-NEXT: s_mov_b32 s91, 0xe8000014952; GCN2-NEXT: s_add_u32 s88, s88, s1114953; GCN2-NEXT: s_addc_u32 s89, s89, 014954; GCN2-NEXT: s_waitcnt lgkmcnt(0)14955; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 314956; GCN2-NEXT: s_add_u32 s0, s0, s414957; GCN2-NEXT: s_addc_u32 s1, s1, s514958; GCN2-NEXT: s_cmp_eq_u32 s1, s814959; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 014960; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]14961; GCN2-NEXT: s_mov_b64 s[4:5], -114962; GCN2-NEXT: s_cbranch_vccnz .LBB113_314963; GCN2-NEXT: ; %bb.1: ; %Flow14964; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]14965; GCN2-NEXT: s_cbranch_vccz .LBB113_414966; GCN2-NEXT: .LBB113_2: ; %atomicrmw.phi14967; GCN2-NEXT: s_endpgm14968; GCN2-NEXT: .LBB113_3: ; %atomicrmw.global14969; GCN2-NEXT: v_mov_b32_e32 v0, s014970; GCN2-NEXT: v_mov_b32_e32 v2, s214971; GCN2-NEXT: v_mov_b32_e32 v1, s114972; GCN2-NEXT: v_mov_b32_e32 v3, s314973; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]14974; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14975; GCN2-NEXT: buffer_wbinvl1_vol14976; GCN2-NEXT: s_cbranch_execnz .LBB113_214977; GCN2-NEXT: .LBB113_4: ; %atomicrmw.private14978; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 014979; GCN2-NEXT: s_cselect_b32 s0, s0, -114980; GCN2-NEXT: v_mov_b32_e32 v2, s014981; GCN2-NEXT: s_add_i32 s0, s0, 414982; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen14983; GCN2-NEXT: v_mov_b32_e32 v3, s014984; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen14985; GCN2-NEXT: s_waitcnt vmcnt(1)14986; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v014987; GCN2-NEXT: s_waitcnt vmcnt(0)14988; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc14989; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]14990; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v4, vcc14991; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v5, vcc14992; GCN2-NEXT: buffer_store_dword v1, v2, s[88:91], 0 offen14993; GCN2-NEXT: buffer_store_dword v0, v3, s[88:91], 0 offen14994; GCN2-NEXT: s_endpgm14995;14996; GFX12-LABEL: atomic_inc_i64_incr64:14997; GFX12: ; %bb.0: ; %entry14998; GFX12-NEXT: s_clause 0x114999; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x3415000; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2415001; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base15002; GFX12-NEXT: s_wait_kmcnt 0x015003; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 315004; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)15005; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]15006; GFX12-NEXT: s_cmp_eq_u32 s1, s715007; GFX12-NEXT: s_cselect_b32 s4, -1, 015008; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)15009; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s415010; GFX12-NEXT: s_mov_b32 s4, -115011; GFX12-NEXT: s_cbranch_vccnz .LBB113_315012; GFX12-NEXT: ; %bb.1: ; %Flow15013; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s415014; GFX12-NEXT: s_cbranch_vccz .LBB113_415015; GFX12-NEXT: .LBB113_2: ; %atomicrmw.phi15016; GFX12-NEXT: s_endpgm15017; GFX12-NEXT: .LBB113_3: ; %atomicrmw.global15018; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s115019; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s315020; GFX12-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3] scope:SCOPE_DEV15021; GFX12-NEXT: s_wait_storecnt_dscnt 0x015022; GFX12-NEXT: global_inv scope:SCOPE_DEV15023; GFX12-NEXT: s_cbranch_execnz .LBB113_215024; GFX12-NEXT: .LBB113_4: ; %atomicrmw.private15025; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 015026; GFX12-NEXT: s_cselect_b32 s0, s0, -115027; GFX12-NEXT: scratch_load_b64 v[0:1], off, s015028; GFX12-NEXT: s_wait_loadcnt 0x015029; GFX12-NEXT: v_add_co_u32 v2, vcc_lo, v0, 115030; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)15031; GFX12-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo15032; GFX12-NEXT: v_cmp_gt_u64_e32 vcc_lo, s[2:3], v[0:1]15033; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)15034; GFX12-NEXT: v_dual_cndmask_b32 v1, 0, v3 :: v_dual_cndmask_b32 v0, 0, v215035; GFX12-NEXT: scratch_store_b64 off, v[0:1], s015036; GFX12-NEXT: s_endpgm15037entry:15038 %ptr = getelementptr inbounds i64, ptr %out, i64 %index15039 %tmp0 = atomicrmw volatile uinc_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !015040 ret void15041}15042 15043define amdgpu_kernel void @atomic_inc_i64_ret_incr64(ptr %out, ptr %out2, i64 %in, i64 %index) {15044; GCN1-LABEL: atomic_inc_i64_ret_incr64:15045; GCN1: ; %bb.0: ; %entry15046; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD015047; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD115048; GCN1-NEXT: s_mov_b32 s18, -115049; GCN1-NEXT: s_mov_b32 s19, 0xe8f00015050; GCN1-NEXT: s_add_u32 s16, s16, s1115051; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x915052; GCN1-NEXT: s_load_dword s2, s[4:5], 0x4115053; GCN1-NEXT: s_addc_u32 s17, s17, 015054; GCN1-NEXT: s_waitcnt lgkmcnt(0)15055; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 315056; GCN1-NEXT: s_add_u32 s0, s8, s015057; GCN1-NEXT: s_addc_u32 s1, s9, s115058; GCN1-NEXT: s_cmp_eq_u32 s1, s215059; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 015060; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]15061; GCN1-NEXT: s_cbranch_vccz .LBB114_215062; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global15063; GCN1-NEXT: v_mov_b32_e32 v0, s015064; GCN1-NEXT: v_mov_b32_e32 v2, s1215065; GCN1-NEXT: v_mov_b32_e32 v1, s115066; GCN1-NEXT: v_mov_b32_e32 v3, s1315067; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc15068; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)15069; GCN1-NEXT: buffer_wbinvl1_vol15070; GCN1-NEXT: s_cbranch_execz .LBB114_315071; GCN1-NEXT: s_branch .LBB114_415072; GCN1-NEXT: .LBB114_2:15073; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr115074; GCN1-NEXT: .LBB114_3: ; %atomicrmw.private15075; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 015076; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec15077; GCN1-NEXT: s_cselect_b32 s0, s0, -115078; GCN1-NEXT: v_mov_b32_e32 v2, s015079; GCN1-NEXT: s_add_i32 s0, s0, 415080; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen15081; GCN1-NEXT: v_mov_b32_e32 v3, s015082; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen15083; GCN1-NEXT: s_waitcnt vmcnt(1)15084; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v015085; GCN1-NEXT: s_waitcnt vmcnt(0)15086; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc15087; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[12:13], v[0:1]15088; GCN1-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc15089; GCN1-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc15090; GCN1-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen15091; GCN1-NEXT: buffer_store_dword v5, v3, s[16:19], 0 offen15092; GCN1-NEXT: .LBB114_4: ; %atomicrmw.end15093; GCN1-NEXT: v_mov_b32_e32 v2, s1015094; GCN1-NEXT: v_mov_b32_e32 v3, s1115095; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]15096; GCN1-NEXT: s_endpgm15097;15098; GCN2-LABEL: atomic_inc_i64_ret_incr64:15099; GCN2: ; %bb.0: ; %entry15100; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD015101; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD115102; GCN2-NEXT: s_mov_b32 s90, -115103; GCN2-NEXT: s_mov_b32 s91, 0xe8000015104; GCN2-NEXT: s_add_u32 s88, s88, s1115105; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2415106; GCN2-NEXT: s_load_dword s2, s[4:5], 0x10415107; GCN2-NEXT: s_addc_u32 s89, s89, 015108; GCN2-NEXT: s_waitcnt lgkmcnt(0)15109; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 315110; GCN2-NEXT: s_add_u32 s0, s8, s015111; GCN2-NEXT: s_addc_u32 s1, s9, s115112; GCN2-NEXT: s_cmp_eq_u32 s1, s215113; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 015114; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]15115; GCN2-NEXT: s_cbranch_vccz .LBB114_215116; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global15117; GCN2-NEXT: v_mov_b32_e32 v0, s015118; GCN2-NEXT: v_mov_b32_e32 v2, s1215119; GCN2-NEXT: v_mov_b32_e32 v1, s115120; GCN2-NEXT: v_mov_b32_e32 v3, s1315121; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc15122; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)15123; GCN2-NEXT: buffer_wbinvl1_vol15124; GCN2-NEXT: s_cbranch_execz .LBB114_315125; GCN2-NEXT: s_branch .LBB114_415126; GCN2-NEXT: .LBB114_2:15127; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr115128; GCN2-NEXT: .LBB114_3: ; %atomicrmw.private15129; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 015130; GCN2-NEXT: s_cselect_b32 s0, s0, -115131; GCN2-NEXT: v_mov_b32_e32 v2, s015132; GCN2-NEXT: s_add_i32 s0, s0, 415133; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen15134; GCN2-NEXT: v_mov_b32_e32 v3, s015135; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen15136; GCN2-NEXT: s_waitcnt vmcnt(1)15137; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v015138; GCN2-NEXT: s_waitcnt vmcnt(0)15139; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc15140; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[12:13], v[0:1]15141; GCN2-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc15142; GCN2-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc15143; GCN2-NEXT: buffer_store_dword v4, v2, s[88:91], 0 offen15144; GCN2-NEXT: buffer_store_dword v5, v3, s[88:91], 0 offen15145; GCN2-NEXT: .LBB114_4: ; %atomicrmw.end15146; GCN2-NEXT: v_mov_b32_e32 v2, s1015147; GCN2-NEXT: v_mov_b32_e32 v3, s1115148; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]15149; GCN2-NEXT: s_endpgm15150;15151; GFX12-LABEL: atomic_inc_i64_ret_incr64:15152; GFX12: ; %bb.0: ; %entry15153; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x2415154; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base15155; GFX12-NEXT: s_wait_kmcnt 0x015156; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 315157; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)15158; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]15159; GFX12-NEXT: s_cmp_eq_u32 s1, s915160; GFX12-NEXT: s_cselect_b32 s6, -1, 015161; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)15162; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s615163; GFX12-NEXT: s_cbranch_vccz .LBB114_215164; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global15165; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s115166; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s515167; GFX12-NEXT: flat_atomic_inc_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV15168; GFX12-NEXT: s_wait_loadcnt_dscnt 0x015169; GFX12-NEXT: global_inv scope:SCOPE_DEV15170; GFX12-NEXT: s_cbranch_execz .LBB114_315171; GFX12-NEXT: s_branch .LBB114_415172; GFX12-NEXT: .LBB114_2:15173; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr115174; GFX12-NEXT: .LBB114_3: ; %atomicrmw.private15175; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 015176; GFX12-NEXT: s_cselect_b32 s0, s0, -115177; GFX12-NEXT: scratch_load_b64 v[0:1], off, s015178; GFX12-NEXT: s_wait_loadcnt 0x015179; GFX12-NEXT: v_add_co_u32 v2, vcc_lo, v0, 115180; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)15181; GFX12-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo15182; GFX12-NEXT: v_cmp_gt_u64_e32 vcc_lo, s[4:5], v[0:1]15183; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)15184; GFX12-NEXT: v_dual_cndmask_b32 v3, 0, v3 :: v_dual_cndmask_b32 v2, 0, v215185; GFX12-NEXT: scratch_store_b64 off, v[2:3], s015186; GFX12-NEXT: .LBB114_4: ; %atomicrmw.end15187; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s315188; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]15189; GFX12-NEXT: s_endpgm15190entry:15191 %ptr = getelementptr inbounds i64, ptr %out, i64 %index15192 %tmp0 = atomicrmw volatile uinc_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !015193 store i64 %tmp0, ptr %out215194 ret void15195}15196 15197define amdgpu_kernel void @atomic_dec_i64_offset(ptr %out, i64 %in) {15198; GCN1-LABEL: atomic_dec_i64_offset:15199; GCN1: ; %bb.0: ; %entry15200; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD015201; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD115202; GCN1-NEXT: s_mov_b32 s14, -115203; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x915204; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d15205; GCN1-NEXT: s_mov_b32 s15, 0xe8f00015206; GCN1-NEXT: s_add_u32 s12, s12, s1115207; GCN1-NEXT: s_addc_u32 s13, s13, 015208; GCN1-NEXT: s_waitcnt lgkmcnt(0)15209; GCN1-NEXT: s_add_u32 s0, s0, 3215210; GCN1-NEXT: s_addc_u32 s1, s1, 015211; GCN1-NEXT: s_cmp_eq_u32 s1, s415212; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 015213; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]15214; GCN1-NEXT: s_mov_b64 s[4:5], -115215; GCN1-NEXT: s_cbranch_vccnz .LBB115_315216; GCN1-NEXT: ; %bb.1: ; %Flow15217; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]15218; GCN1-NEXT: s_cbranch_vccz .LBB115_415219; GCN1-NEXT: .LBB115_2: ; %atomicrmw.phi15220; GCN1-NEXT: s_endpgm15221; GCN1-NEXT: .LBB115_3: ; %atomicrmw.global15222; GCN1-NEXT: v_mov_b32_e32 v0, s015223; GCN1-NEXT: v_mov_b32_e32 v2, s215224; GCN1-NEXT: v_mov_b32_e32 v1, s115225; GCN1-NEXT: v_mov_b32_e32 v3, s315226; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]15227; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)15228; GCN1-NEXT: buffer_wbinvl1_vol15229; GCN1-NEXT: s_cbranch_execnz .LBB115_215230; GCN1-NEXT: .LBB115_4: ; %atomicrmw.private15231; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 015232; GCN1-NEXT: v_mov_b32_e32 v4, s315233; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec15234; GCN1-NEXT: s_cselect_b32 s0, s0, -115235; GCN1-NEXT: v_mov_b32_e32 v2, s015236; GCN1-NEXT: s_add_i32 s0, s0, 415237; GCN1-NEXT: v_mov_b32_e32 v3, s015238; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen15239; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen15240; GCN1-NEXT: v_mov_b32_e32 v5, s215241; GCN1-NEXT: s_waitcnt vmcnt(0)15242; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]15243; GCN1-NEXT: v_cmp_lt_u64_e64 s[0:1], s[2:3], v[0:1]15244; GCN1-NEXT: v_add_i32_e64 v0, s[2:3], -1, v015245; GCN1-NEXT: s_or_b64 vcc, vcc, s[0:1]15246; GCN1-NEXT: v_addc_u32_e64 v1, s[2:3], -1, v1, s[2:3]15247; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc15248; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc15249; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen15250; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen15251; GCN1-NEXT: s_endpgm15252;15253; GCN2-LABEL: atomic_dec_i64_offset:15254; GCN2: ; %bb.0: ; %entry15255; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD015256; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD115257; GCN2-NEXT: s_mov_b32 s90, -115258; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2415259; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf415260; GCN2-NEXT: s_mov_b32 s91, 0xe8000015261; GCN2-NEXT: s_add_u32 s88, s88, s1115262; GCN2-NEXT: s_addc_u32 s89, s89, 015263; GCN2-NEXT: s_waitcnt lgkmcnt(0)15264; GCN2-NEXT: s_add_u32 s0, s0, 3215265; GCN2-NEXT: s_addc_u32 s1, s1, 015266; GCN2-NEXT: s_cmp_eq_u32 s1, s415267; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 015268; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]15269; GCN2-NEXT: s_mov_b64 s[4:5], -115270; GCN2-NEXT: s_cbranch_vccnz .LBB115_315271; GCN2-NEXT: ; %bb.1: ; %Flow15272; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]15273; GCN2-NEXT: s_cbranch_vccz .LBB115_415274; GCN2-NEXT: .LBB115_2: ; %atomicrmw.phi15275; GCN2-NEXT: s_endpgm15276; GCN2-NEXT: .LBB115_3: ; %atomicrmw.global15277; GCN2-NEXT: v_mov_b32_e32 v0, s015278; GCN2-NEXT: v_mov_b32_e32 v2, s215279; GCN2-NEXT: v_mov_b32_e32 v1, s115280; GCN2-NEXT: v_mov_b32_e32 v3, s315281; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]15282; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)15283; GCN2-NEXT: buffer_wbinvl1_vol15284; GCN2-NEXT: s_cbranch_execnz .LBB115_215285; GCN2-NEXT: .LBB115_4: ; %atomicrmw.private15286; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 015287; GCN2-NEXT: s_cselect_b32 s0, s0, -115288; GCN2-NEXT: v_mov_b32_e32 v2, s015289; GCN2-NEXT: s_add_i32 s0, s0, 415290; GCN2-NEXT: v_mov_b32_e32 v3, s015291; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen15292; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen15293; GCN2-NEXT: v_mov_b32_e32 v4, s315294; GCN2-NEXT: v_mov_b32_e32 v5, s215295; GCN2-NEXT: s_waitcnt vmcnt(0)15296; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]15297; GCN2-NEXT: v_cmp_lt_u64_e64 s[0:1], s[2:3], v[0:1]15298; GCN2-NEXT: v_add_u32_e64 v0, s[2:3], -1, v015299; GCN2-NEXT: s_or_b64 vcc, vcc, s[0:1]15300; GCN2-NEXT: v_addc_u32_e64 v1, s[2:3], -1, v1, s[2:3]15301; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc15302; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc15303; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen15304; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen15305; GCN2-NEXT: s_endpgm15306;15307; GFX12-LABEL: atomic_dec_i64_offset:15308; GFX12: ; %bb.0: ; %entry15309; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2415310; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base15311; GFX12-NEXT: s_wait_kmcnt 0x015312; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 3215313; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)15314; GFX12-NEXT: s_cmp_eq_u32 s1, s715315; GFX12-NEXT: s_cselect_b32 s4, -1, 015316; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s415317; GFX12-NEXT: s_mov_b32 s4, -115318; GFX12-NEXT: s_cbranch_vccnz .LBB115_315319; GFX12-NEXT: ; %bb.1: ; %Flow15320; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s415321; GFX12-NEXT: s_cbranch_vccz .LBB115_415322; GFX12-NEXT: .LBB115_2: ; %atomicrmw.phi15323; GFX12-NEXT: s_endpgm15324; GFX12-NEXT: .LBB115_3: ; %atomicrmw.global15325; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s115326; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s315327; GFX12-NEXT: flat_atomic_dec_u64 v[0:1], v[2:3] scope:SCOPE_DEV15328; GFX12-NEXT: s_wait_storecnt_dscnt 0x015329; GFX12-NEXT: global_inv scope:SCOPE_DEV15330; GFX12-NEXT: s_cbranch_execnz .LBB115_215331; GFX12-NEXT: .LBB115_4: ; %atomicrmw.private15332; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 015333; GFX12-NEXT: s_cselect_b32 s4, s0, -115334; GFX12-NEXT: scratch_load_b64 v[0:1], off, s415335; GFX12-NEXT: s_wait_loadcnt 0x015336; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]15337; GFX12-NEXT: v_cmp_lt_u64_e64 s0, s[2:3], v[0:1]15338; GFX12-NEXT: v_add_co_u32 v0, s1, v0, -115339; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)15340; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, s115341; GFX12-NEXT: s_or_b32 s0, vcc_lo, s015342; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)15343; GFX12-NEXT: v_cndmask_b32_e64 v1, v1, s3, s015344; GFX12-NEXT: v_cndmask_b32_e64 v0, v0, s2, s015345; GFX12-NEXT: scratch_store_b64 off, v[0:1], s415346; GFX12-NEXT: s_endpgm15347entry:15348 %gep = getelementptr inbounds i64, ptr %out, i64 415349 %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !015350 ret void15351}15352 15353define amdgpu_kernel void @atomic_dec_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {15354; GCN1-LABEL: atomic_dec_i64_ret_offset:15355; GCN1: ; %bb.0: ; %entry15356; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD015357; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD115358; GCN1-NEXT: s_mov_b32 s14, -115359; GCN1-NEXT: s_mov_b32 s15, 0xe8f00015360; GCN1-NEXT: s_add_u32 s12, s12, s1115361; GCN1-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x915362; GCN1-NEXT: s_load_dword s6, s[4:5], 0x3f15363; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd15364; GCN1-NEXT: s_addc_u32 s13, s13, 015365; GCN1-NEXT: s_waitcnt lgkmcnt(0)15366; GCN1-NEXT: s_add_u32 s2, s8, 3215367; GCN1-NEXT: s_addc_u32 s3, s9, 015368; GCN1-NEXT: s_cmp_eq_u32 s3, s615369; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 015370; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]15371; GCN1-NEXT: s_cbranch_vccz .LBB116_215372; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global15373; GCN1-NEXT: v_mov_b32_e32 v0, s215374; GCN1-NEXT: v_mov_b32_e32 v3, s115375; GCN1-NEXT: v_mov_b32_e32 v1, s315376; GCN1-NEXT: v_mov_b32_e32 v2, s015377; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc15378; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)15379; GCN1-NEXT: buffer_wbinvl1_vol15380; GCN1-NEXT: s_cbranch_execz .LBB116_315381; GCN1-NEXT: s_branch .LBB116_415382; GCN1-NEXT: .LBB116_2:15383; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr115384; GCN1-NEXT: .LBB116_3: ; %atomicrmw.private15385; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[2:3], 015386; GCN1-NEXT: v_mov_b32_e32 v4, s115387; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec15388; GCN1-NEXT: s_cselect_b32 s2, s2, -115389; GCN1-NEXT: v_mov_b32_e32 v2, s215390; GCN1-NEXT: s_add_i32 s2, s2, 415391; GCN1-NEXT: v_mov_b32_e32 v3, s215392; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen15393; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen15394; GCN1-NEXT: v_mov_b32_e32 v5, s015395; GCN1-NEXT: s_waitcnt vmcnt(1)15396; GCN1-NEXT: v_add_i32_e64 v6, s[2:3], -1, v015397; GCN1-NEXT: s_waitcnt vmcnt(0)15398; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]15399; GCN1-NEXT: v_cmp_lt_u64_e64 s[0:1], s[0:1], v[0:1]15400; GCN1-NEXT: v_addc_u32_e64 v7, s[2:3], -1, v1, s[2:3]15401; GCN1-NEXT: s_or_b64 vcc, vcc, s[0:1]15402; GCN1-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc15403; GCN1-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc15404; GCN1-NEXT: buffer_store_dword v5, v2, s[12:15], 0 offen15405; GCN1-NEXT: buffer_store_dword v4, v3, s[12:15], 0 offen15406; GCN1-NEXT: .LBB116_4: ; %atomicrmw.end15407; GCN1-NEXT: v_mov_b32_e32 v2, s1015408; GCN1-NEXT: v_mov_b32_e32 v3, s1115409; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]15410; GCN1-NEXT: s_endpgm15411;15412; GCN2-LABEL: atomic_dec_i64_ret_offset:15413; GCN2: ; %bb.0: ; %entry15414; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD015415; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD115416; GCN2-NEXT: s_mov_b32 s90, -115417; GCN2-NEXT: s_mov_b32 s91, 0xe8000015418; GCN2-NEXT: s_add_u32 s88, s88, s1115419; GCN2-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x2415420; GCN2-NEXT: s_load_dword s6, s[4:5], 0xfc15421; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x3415422; GCN2-NEXT: s_addc_u32 s89, s89, 015423; GCN2-NEXT: s_waitcnt lgkmcnt(0)15424; GCN2-NEXT: s_add_u32 s2, s8, 3215425; GCN2-NEXT: s_addc_u32 s3, s9, 015426; GCN2-NEXT: s_cmp_eq_u32 s3, s615427; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 015428; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]15429; GCN2-NEXT: s_cbranch_vccz .LBB116_215430; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global15431; GCN2-NEXT: v_mov_b32_e32 v0, s215432; GCN2-NEXT: v_mov_b32_e32 v3, s115433; GCN2-NEXT: v_mov_b32_e32 v1, s315434; GCN2-NEXT: v_mov_b32_e32 v2, s015435; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc15436; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)15437; GCN2-NEXT: buffer_wbinvl1_vol15438; GCN2-NEXT: s_cbranch_execz .LBB116_315439; GCN2-NEXT: s_branch .LBB116_415440; GCN2-NEXT: .LBB116_2:15441; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr115442; GCN2-NEXT: .LBB116_3: ; %atomicrmw.private15443; GCN2-NEXT: s_cmp_lg_u64 s[2:3], 015444; GCN2-NEXT: s_cselect_b32 s2, s2, -115445; GCN2-NEXT: v_mov_b32_e32 v2, s215446; GCN2-NEXT: s_add_i32 s2, s2, 415447; GCN2-NEXT: v_mov_b32_e32 v3, s215448; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen15449; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen15450; GCN2-NEXT: v_mov_b32_e32 v4, s115451; GCN2-NEXT: v_mov_b32_e32 v5, s015452; GCN2-NEXT: s_waitcnt vmcnt(1)15453; GCN2-NEXT: v_add_u32_e64 v6, s[2:3], -1, v015454; GCN2-NEXT: s_waitcnt vmcnt(0)15455; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]15456; GCN2-NEXT: v_cmp_lt_u64_e64 s[0:1], s[0:1], v[0:1]15457; GCN2-NEXT: v_addc_u32_e64 v7, s[2:3], -1, v1, s[2:3]15458; GCN2-NEXT: s_or_b64 vcc, vcc, s[0:1]15459; GCN2-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc15460; GCN2-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc15461; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen15462; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen15463; GCN2-NEXT: .LBB116_4: ; %atomicrmw.end15464; GCN2-NEXT: v_mov_b32_e32 v2, s1015465; GCN2-NEXT: v_mov_b32_e32 v3, s1115466; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]15467; GCN2-NEXT: s_endpgm15468;15469; GFX12-LABEL: atomic_dec_i64_ret_offset:15470; GFX12: ; %bb.0: ; %entry15471; GFX12-NEXT: s_clause 0x115472; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2415473; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x3415474; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base15475; GFX12-NEXT: s_wait_kmcnt 0x015476; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 3215477; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)15478; GFX12-NEXT: s_cmp_eq_u32 s1, s715479; GFX12-NEXT: s_cselect_b32 s6, -1, 015480; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s615481; GFX12-NEXT: s_cbranch_vccz .LBB116_215482; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global15483; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s115484; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s515485; GFX12-NEXT: flat_atomic_dec_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV15486; GFX12-NEXT: s_wait_loadcnt_dscnt 0x015487; GFX12-NEXT: global_inv scope:SCOPE_DEV15488; GFX12-NEXT: s_cbranch_execz .LBB116_315489; GFX12-NEXT: s_branch .LBB116_415490; GFX12-NEXT: .LBB116_2:15491; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr115492; GFX12-NEXT: .LBB116_3: ; %atomicrmw.private15493; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 015494; GFX12-NEXT: s_cselect_b32 s6, s0, -115495; GFX12-NEXT: scratch_load_b64 v[0:1], off, s615496; GFX12-NEXT: s_wait_loadcnt 0x015497; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]15498; GFX12-NEXT: v_cmp_lt_u64_e64 s0, s[4:5], v[0:1]15499; GFX12-NEXT: v_add_co_u32 v2, s1, v0, -115500; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)15501; GFX12-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v1, s115502; GFX12-NEXT: s_or_b32 s0, vcc_lo, s015503; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)15504; GFX12-NEXT: v_cndmask_b32_e64 v3, v3, s5, s015505; GFX12-NEXT: v_cndmask_b32_e64 v2, v2, s4, s015506; GFX12-NEXT: scratch_store_b64 off, v[2:3], s615507; GFX12-NEXT: .LBB116_4: ; %atomicrmw.end15508; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s315509; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]15510; GFX12-NEXT: s_endpgm15511entry:15512 %gep = getelementptr inbounds i64, ptr %out, i64 415513 %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !015514 store i64 %tmp0, ptr %out215515 ret void15516}15517 15518define amdgpu_kernel void @atomic_dec_i64_decr64_offset(ptr %out, i64 %in, i64 %index) {15519; GCN1-LABEL: atomic_dec_i64_decr64_offset:15520; GCN1: ; %bb.0: ; %entry15521; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD015522; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD115523; GCN1-NEXT: s_mov_b32 s14, -115524; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd15525; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x915526; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f15527; GCN1-NEXT: s_mov_b32 s15, 0xe8f00015528; GCN1-NEXT: s_add_u32 s12, s12, s1115529; GCN1-NEXT: s_addc_u32 s13, s13, 015530; GCN1-NEXT: s_waitcnt lgkmcnt(0)15531; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 315532; GCN1-NEXT: s_add_u32 s0, s0, s415533; GCN1-NEXT: s_addc_u32 s1, s1, s515534; GCN1-NEXT: s_add_u32 s0, s0, 3215535; GCN1-NEXT: s_addc_u32 s1, s1, 015536; GCN1-NEXT: s_cmp_eq_u32 s1, s815537; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 015538; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]15539; GCN1-NEXT: s_mov_b64 s[4:5], -115540; GCN1-NEXT: s_cbranch_vccnz .LBB117_315541; GCN1-NEXT: ; %bb.1: ; %Flow15542; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]15543; GCN1-NEXT: s_cbranch_vccz .LBB117_415544; GCN1-NEXT: .LBB117_2: ; %atomicrmw.phi15545; GCN1-NEXT: s_endpgm15546; GCN1-NEXT: .LBB117_3: ; %atomicrmw.global15547; GCN1-NEXT: v_mov_b32_e32 v0, s015548; GCN1-NEXT: v_mov_b32_e32 v2, s215549; GCN1-NEXT: v_mov_b32_e32 v1, s115550; GCN1-NEXT: v_mov_b32_e32 v3, s315551; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]15552; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)15553; GCN1-NEXT: buffer_wbinvl1_vol15554; GCN1-NEXT: s_cbranch_execnz .LBB117_215555; GCN1-NEXT: .LBB117_4: ; %atomicrmw.private15556; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 015557; GCN1-NEXT: v_mov_b32_e32 v4, s315558; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec15559; GCN1-NEXT: s_cselect_b32 s0, s0, -115560; GCN1-NEXT: v_mov_b32_e32 v2, s015561; GCN1-NEXT: s_add_i32 s0, s0, 415562; GCN1-NEXT: v_mov_b32_e32 v3, s015563; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen15564; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen15565; GCN1-NEXT: v_mov_b32_e32 v5, s215566; GCN1-NEXT: s_waitcnt vmcnt(0)15567; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]15568; GCN1-NEXT: v_cmp_lt_u64_e64 s[0:1], s[2:3], v[0:1]15569; GCN1-NEXT: v_add_i32_e64 v0, s[2:3], -1, v015570; GCN1-NEXT: s_or_b64 vcc, vcc, s[0:1]15571; GCN1-NEXT: v_addc_u32_e64 v1, s[2:3], -1, v1, s[2:3]15572; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc15573; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc15574; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen15575; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen15576; GCN1-NEXT: s_endpgm15577;15578; GCN2-LABEL: atomic_dec_i64_decr64_offset:15579; GCN2: ; %bb.0: ; %entry15580; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD015581; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD115582; GCN2-NEXT: s_mov_b32 s90, -115583; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3415584; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2415585; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc15586; GCN2-NEXT: s_mov_b32 s91, 0xe8000015587; GCN2-NEXT: s_add_u32 s88, s88, s1115588; GCN2-NEXT: s_addc_u32 s89, s89, 015589; GCN2-NEXT: s_waitcnt lgkmcnt(0)15590; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 315591; GCN2-NEXT: s_add_u32 s0, s0, s415592; GCN2-NEXT: s_addc_u32 s1, s1, s515593; GCN2-NEXT: s_add_u32 s0, s0, 3215594; GCN2-NEXT: s_addc_u32 s1, s1, 015595; GCN2-NEXT: s_cmp_eq_u32 s1, s815596; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 015597; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]15598; GCN2-NEXT: s_mov_b64 s[4:5], -115599; GCN2-NEXT: s_cbranch_vccnz .LBB117_315600; GCN2-NEXT: ; %bb.1: ; %Flow15601; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]15602; GCN2-NEXT: s_cbranch_vccz .LBB117_415603; GCN2-NEXT: .LBB117_2: ; %atomicrmw.phi15604; GCN2-NEXT: s_endpgm15605; GCN2-NEXT: .LBB117_3: ; %atomicrmw.global15606; GCN2-NEXT: v_mov_b32_e32 v0, s015607; GCN2-NEXT: v_mov_b32_e32 v2, s215608; GCN2-NEXT: v_mov_b32_e32 v1, s115609; GCN2-NEXT: v_mov_b32_e32 v3, s315610; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]15611; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)15612; GCN2-NEXT: buffer_wbinvl1_vol15613; GCN2-NEXT: s_cbranch_execnz .LBB117_215614; GCN2-NEXT: .LBB117_4: ; %atomicrmw.private15615; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 015616; GCN2-NEXT: s_cselect_b32 s0, s0, -115617; GCN2-NEXT: v_mov_b32_e32 v2, s015618; GCN2-NEXT: s_add_i32 s0, s0, 415619; GCN2-NEXT: v_mov_b32_e32 v3, s015620; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen15621; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen15622; GCN2-NEXT: v_mov_b32_e32 v4, s315623; GCN2-NEXT: v_mov_b32_e32 v5, s215624; GCN2-NEXT: s_waitcnt vmcnt(0)15625; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]15626; GCN2-NEXT: v_cmp_lt_u64_e64 s[0:1], s[2:3], v[0:1]15627; GCN2-NEXT: v_add_u32_e64 v0, s[2:3], -1, v015628; GCN2-NEXT: s_or_b64 vcc, vcc, s[0:1]15629; GCN2-NEXT: v_addc_u32_e64 v1, s[2:3], -1, v1, s[2:3]15630; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc15631; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc15632; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen15633; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen15634; GCN2-NEXT: s_endpgm15635;15636; GFX12-LABEL: atomic_dec_i64_decr64_offset:15637; GFX12: ; %bb.0: ; %entry15638; GFX12-NEXT: s_clause 0x115639; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x3415640; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2415641; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base15642; GFX12-NEXT: s_wait_kmcnt 0x015643; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 315644; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)15645; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]15646; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 3215647; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)15648; GFX12-NEXT: s_cmp_eq_u32 s1, s715649; GFX12-NEXT: s_cselect_b32 s4, -1, 015650; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s415651; GFX12-NEXT: s_mov_b32 s4, -115652; GFX12-NEXT: s_cbranch_vccnz .LBB117_315653; GFX12-NEXT: ; %bb.1: ; %Flow15654; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s415655; GFX12-NEXT: s_cbranch_vccz .LBB117_415656; GFX12-NEXT: .LBB117_2: ; %atomicrmw.phi15657; GFX12-NEXT: s_endpgm15658; GFX12-NEXT: .LBB117_3: ; %atomicrmw.global15659; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s115660; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s315661; GFX12-NEXT: flat_atomic_dec_u64 v[0:1], v[2:3] scope:SCOPE_DEV15662; GFX12-NEXT: s_wait_storecnt_dscnt 0x015663; GFX12-NEXT: global_inv scope:SCOPE_DEV15664; GFX12-NEXT: s_cbranch_execnz .LBB117_215665; GFX12-NEXT: .LBB117_4: ; %atomicrmw.private15666; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 015667; GFX12-NEXT: s_cselect_b32 s4, s0, -115668; GFX12-NEXT: scratch_load_b64 v[0:1], off, s415669; GFX12-NEXT: s_wait_loadcnt 0x015670; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]15671; GFX12-NEXT: v_cmp_lt_u64_e64 s0, s[2:3], v[0:1]15672; GFX12-NEXT: v_add_co_u32 v0, s1, v0, -115673; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)15674; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, s115675; GFX12-NEXT: s_or_b32 s0, vcc_lo, s015676; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)15677; GFX12-NEXT: v_cndmask_b32_e64 v1, v1, s3, s015678; GFX12-NEXT: v_cndmask_b32_e64 v0, v0, s2, s015679; GFX12-NEXT: scratch_store_b64 off, v[0:1], s415680; GFX12-NEXT: s_endpgm15681entry:15682 %ptr = getelementptr inbounds i64, ptr %out, i64 %index15683 %gep = getelementptr inbounds i64, ptr %ptr, i64 415684 %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !015685 ret void15686}15687 15688define amdgpu_kernel void @atomic_dec_i64_ret_decr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {15689; GCN1-LABEL: atomic_dec_i64_ret_decr64_offset:15690; GCN1: ; %bb.0: ; %entry15691; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD015692; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD115693; GCN1-NEXT: s_mov_b32 s18, -115694; GCN1-NEXT: s_mov_b32 s19, 0xe8f00015695; GCN1-NEXT: s_add_u32 s16, s16, s1115696; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x915697; GCN1-NEXT: s_load_dword s2, s[4:5], 0x4115698; GCN1-NEXT: s_addc_u32 s17, s17, 015699; GCN1-NEXT: s_waitcnt lgkmcnt(0)15700; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 315701; GCN1-NEXT: s_add_u32 s0, s8, s015702; GCN1-NEXT: s_addc_u32 s1, s9, s115703; GCN1-NEXT: s_add_u32 s0, s0, 3215704; GCN1-NEXT: s_addc_u32 s1, s1, 015705; GCN1-NEXT: s_cmp_eq_u32 s1, s215706; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 015707; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]15708; GCN1-NEXT: s_cbranch_vccz .LBB118_215709; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global15710; GCN1-NEXT: v_mov_b32_e32 v0, s015711; GCN1-NEXT: v_mov_b32_e32 v2, s1215712; GCN1-NEXT: v_mov_b32_e32 v1, s115713; GCN1-NEXT: v_mov_b32_e32 v3, s1315714; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc15715; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)15716; GCN1-NEXT: buffer_wbinvl1_vol15717; GCN1-NEXT: s_cbranch_execz .LBB118_315718; GCN1-NEXT: s_branch .LBB118_415719; GCN1-NEXT: .LBB118_2:15720; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr115721; GCN1-NEXT: .LBB118_3: ; %atomicrmw.private15722; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 015723; GCN1-NEXT: v_mov_b32_e32 v5, s1215724; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec15725; GCN1-NEXT: s_cselect_b32 s0, s0, -115726; GCN1-NEXT: v_mov_b32_e32 v2, s015727; GCN1-NEXT: s_add_i32 s0, s0, 415728; GCN1-NEXT: v_mov_b32_e32 v3, s015729; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen15730; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen15731; GCN1-NEXT: v_mov_b32_e32 v4, s1315732; GCN1-NEXT: s_waitcnt vmcnt(1)15733; GCN1-NEXT: v_add_i32_e64 v6, s[2:3], -1, v015734; GCN1-NEXT: s_waitcnt vmcnt(0)15735; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]15736; GCN1-NEXT: v_cmp_lt_u64_e64 s[0:1], s[12:13], v[0:1]15737; GCN1-NEXT: v_addc_u32_e64 v7, s[2:3], -1, v1, s[2:3]15738; GCN1-NEXT: s_or_b64 vcc, vcc, s[0:1]15739; GCN1-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc15740; GCN1-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc15741; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen15742; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen15743; GCN1-NEXT: .LBB118_4: ; %atomicrmw.end15744; GCN1-NEXT: v_mov_b32_e32 v2, s1015745; GCN1-NEXT: v_mov_b32_e32 v3, s1115746; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]15747; GCN1-NEXT: s_endpgm15748;15749; GCN2-LABEL: atomic_dec_i64_ret_decr64_offset:15750; GCN2: ; %bb.0: ; %entry15751; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD015752; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD115753; GCN2-NEXT: s_mov_b32 s90, -115754; GCN2-NEXT: s_mov_b32 s91, 0xe8000015755; GCN2-NEXT: s_add_u32 s88, s88, s1115756; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2415757; GCN2-NEXT: s_load_dword s2, s[4:5], 0x10415758; GCN2-NEXT: s_addc_u32 s89, s89, 015759; GCN2-NEXT: s_waitcnt lgkmcnt(0)15760; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 315761; GCN2-NEXT: s_add_u32 s0, s8, s015762; GCN2-NEXT: s_addc_u32 s1, s9, s115763; GCN2-NEXT: s_add_u32 s0, s0, 3215764; GCN2-NEXT: s_addc_u32 s1, s1, 015765; GCN2-NEXT: s_cmp_eq_u32 s1, s215766; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 015767; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]15768; GCN2-NEXT: s_cbranch_vccz .LBB118_215769; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global15770; GCN2-NEXT: v_mov_b32_e32 v0, s015771; GCN2-NEXT: v_mov_b32_e32 v2, s1215772; GCN2-NEXT: v_mov_b32_e32 v1, s115773; GCN2-NEXT: v_mov_b32_e32 v3, s1315774; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc15775; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)15776; GCN2-NEXT: buffer_wbinvl1_vol15777; GCN2-NEXT: s_cbranch_execz .LBB118_315778; GCN2-NEXT: s_branch .LBB118_415779; GCN2-NEXT: .LBB118_2:15780; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr115781; GCN2-NEXT: .LBB118_3: ; %atomicrmw.private15782; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 015783; GCN2-NEXT: s_cselect_b32 s0, s0, -115784; GCN2-NEXT: v_mov_b32_e32 v2, s015785; GCN2-NEXT: s_add_i32 s0, s0, 415786; GCN2-NEXT: v_mov_b32_e32 v3, s015787; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen15788; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen15789; GCN2-NEXT: v_mov_b32_e32 v5, s1215790; GCN2-NEXT: v_mov_b32_e32 v4, s1315791; GCN2-NEXT: s_waitcnt vmcnt(1)15792; GCN2-NEXT: v_add_u32_e64 v6, s[2:3], -1, v015793; GCN2-NEXT: s_waitcnt vmcnt(0)15794; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]15795; GCN2-NEXT: v_cmp_lt_u64_e64 s[0:1], s[12:13], v[0:1]15796; GCN2-NEXT: v_addc_u32_e64 v7, s[2:3], -1, v1, s[2:3]15797; GCN2-NEXT: s_or_b64 vcc, vcc, s[0:1]15798; GCN2-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc15799; GCN2-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc15800; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen15801; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen15802; GCN2-NEXT: .LBB118_4: ; %atomicrmw.end15803; GCN2-NEXT: v_mov_b32_e32 v2, s1015804; GCN2-NEXT: v_mov_b32_e32 v3, s1115805; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]15806; GCN2-NEXT: s_endpgm15807;15808; GFX12-LABEL: atomic_dec_i64_ret_decr64_offset:15809; GFX12: ; %bb.0: ; %entry15810; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x2415811; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base15812; GFX12-NEXT: s_wait_kmcnt 0x015813; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 315814; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)15815; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]15816; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], 3215817; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)15818; GFX12-NEXT: s_cmp_eq_u32 s1, s915819; GFX12-NEXT: s_cselect_b32 s6, -1, 015820; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s615821; GFX12-NEXT: s_cbranch_vccz .LBB118_215822; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global15823; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s115824; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s515825; GFX12-NEXT: flat_atomic_dec_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV15826; GFX12-NEXT: s_wait_loadcnt_dscnt 0x015827; GFX12-NEXT: global_inv scope:SCOPE_DEV15828; GFX12-NEXT: s_cbranch_execz .LBB118_315829; GFX12-NEXT: s_branch .LBB118_415830; GFX12-NEXT: .LBB118_2:15831; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr115832; GFX12-NEXT: .LBB118_3: ; %atomicrmw.private15833; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 015834; GFX12-NEXT: s_cselect_b32 s6, s0, -115835; GFX12-NEXT: scratch_load_b64 v[0:1], off, s615836; GFX12-NEXT: s_wait_loadcnt 0x015837; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]15838; GFX12-NEXT: v_cmp_lt_u64_e64 s0, s[4:5], v[0:1]15839; GFX12-NEXT: v_add_co_u32 v2, s1, v0, -115840; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)15841; GFX12-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v1, s115842; GFX12-NEXT: s_or_b32 s0, vcc_lo, s015843; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)15844; GFX12-NEXT: v_cndmask_b32_e64 v3, v3, s5, s015845; GFX12-NEXT: v_cndmask_b32_e64 v2, v2, s4, s015846; GFX12-NEXT: scratch_store_b64 off, v[2:3], s615847; GFX12-NEXT: .LBB118_4: ; %atomicrmw.end15848; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s315849; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]15850; GFX12-NEXT: s_endpgm15851entry:15852 %ptr = getelementptr inbounds i64, ptr %out, i64 %index15853 %gep = getelementptr inbounds i64, ptr %ptr, i64 415854 %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !015855 store i64 %tmp0, ptr %out215856 ret void15857}15858 15859define amdgpu_kernel void @atomic_dec_i64(ptr %out, i64 %in) {15860; GCN1-LABEL: atomic_dec_i64:15861; GCN1: ; %bb.0: ; %entry15862; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD015863; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD115864; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x915865; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d15866; GCN1-NEXT: s_mov_b32 s14, -115867; GCN1-NEXT: s_mov_b32 s15, 0xe8f00015868; GCN1-NEXT: s_add_u32 s12, s12, s1115869; GCN1-NEXT: s_addc_u32 s13, s13, 015870; GCN1-NEXT: s_waitcnt lgkmcnt(0)15871; GCN1-NEXT: s_cmp_eq_u32 s1, s415872; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 015873; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]15874; GCN1-NEXT: s_mov_b64 s[4:5], -115875; GCN1-NEXT: s_cbranch_vccnz .LBB119_315876; GCN1-NEXT: ; %bb.1: ; %Flow15877; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]15878; GCN1-NEXT: s_cbranch_vccz .LBB119_415879; GCN1-NEXT: .LBB119_2: ; %atomicrmw.phi15880; GCN1-NEXT: s_endpgm15881; GCN1-NEXT: .LBB119_3: ; %atomicrmw.global15882; GCN1-NEXT: v_mov_b32_e32 v0, s015883; GCN1-NEXT: v_mov_b32_e32 v2, s215884; GCN1-NEXT: v_mov_b32_e32 v1, s115885; GCN1-NEXT: v_mov_b32_e32 v3, s315886; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]15887; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)15888; GCN1-NEXT: buffer_wbinvl1_vol15889; GCN1-NEXT: s_cbranch_execnz .LBB119_215890; GCN1-NEXT: .LBB119_4: ; %atomicrmw.private15891; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 015892; GCN1-NEXT: v_mov_b32_e32 v4, s315893; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec15894; GCN1-NEXT: s_cselect_b32 s0, s0, -115895; GCN1-NEXT: v_mov_b32_e32 v2, s015896; GCN1-NEXT: s_add_i32 s0, s0, 415897; GCN1-NEXT: v_mov_b32_e32 v3, s015898; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen15899; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen15900; GCN1-NEXT: v_mov_b32_e32 v5, s215901; GCN1-NEXT: s_waitcnt vmcnt(0)15902; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]15903; GCN1-NEXT: v_cmp_lt_u64_e64 s[0:1], s[2:3], v[0:1]15904; GCN1-NEXT: v_add_i32_e64 v0, s[2:3], -1, v015905; GCN1-NEXT: s_or_b64 vcc, vcc, s[0:1]15906; GCN1-NEXT: v_addc_u32_e64 v1, s[2:3], -1, v1, s[2:3]15907; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc15908; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc15909; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen15910; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen15911; GCN1-NEXT: s_endpgm15912;15913; GCN2-LABEL: atomic_dec_i64:15914; GCN2: ; %bb.0: ; %entry15915; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD015916; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD115917; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2415918; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf415919; GCN2-NEXT: s_mov_b32 s90, -115920; GCN2-NEXT: s_mov_b32 s91, 0xe8000015921; GCN2-NEXT: s_add_u32 s88, s88, s1115922; GCN2-NEXT: s_addc_u32 s89, s89, 015923; GCN2-NEXT: s_waitcnt lgkmcnt(0)15924; GCN2-NEXT: s_cmp_eq_u32 s1, s415925; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 015926; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]15927; GCN2-NEXT: s_mov_b64 s[4:5], -115928; GCN2-NEXT: s_cbranch_vccnz .LBB119_315929; GCN2-NEXT: ; %bb.1: ; %Flow15930; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]15931; GCN2-NEXT: s_cbranch_vccz .LBB119_415932; GCN2-NEXT: .LBB119_2: ; %atomicrmw.phi15933; GCN2-NEXT: s_endpgm15934; GCN2-NEXT: .LBB119_3: ; %atomicrmw.global15935; GCN2-NEXT: v_mov_b32_e32 v0, s015936; GCN2-NEXT: v_mov_b32_e32 v2, s215937; GCN2-NEXT: v_mov_b32_e32 v1, s115938; GCN2-NEXT: v_mov_b32_e32 v3, s315939; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]15940; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)15941; GCN2-NEXT: buffer_wbinvl1_vol15942; GCN2-NEXT: s_cbranch_execnz .LBB119_215943; GCN2-NEXT: .LBB119_4: ; %atomicrmw.private15944; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 015945; GCN2-NEXT: s_cselect_b32 s0, s0, -115946; GCN2-NEXT: v_mov_b32_e32 v2, s015947; GCN2-NEXT: s_add_i32 s0, s0, 415948; GCN2-NEXT: v_mov_b32_e32 v3, s015949; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen15950; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen15951; GCN2-NEXT: v_mov_b32_e32 v4, s315952; GCN2-NEXT: v_mov_b32_e32 v5, s215953; GCN2-NEXT: s_waitcnt vmcnt(0)15954; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]15955; GCN2-NEXT: v_cmp_lt_u64_e64 s[0:1], s[2:3], v[0:1]15956; GCN2-NEXT: v_add_u32_e64 v0, s[2:3], -1, v015957; GCN2-NEXT: s_or_b64 vcc, vcc, s[0:1]15958; GCN2-NEXT: v_addc_u32_e64 v1, s[2:3], -1, v1, s[2:3]15959; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc15960; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc15961; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen15962; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen15963; GCN2-NEXT: s_endpgm15964;15965; GFX12-LABEL: atomic_dec_i64:15966; GFX12: ; %bb.0: ; %entry15967; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2415968; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base15969; GFX12-NEXT: s_wait_kmcnt 0x015970; GFX12-NEXT: s_cmp_eq_u32 s1, s715971; GFX12-NEXT: s_cselect_b32 s4, -1, 015972; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)15973; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s415974; GFX12-NEXT: s_mov_b32 s4, -115975; GFX12-NEXT: s_cbranch_vccnz .LBB119_315976; GFX12-NEXT: ; %bb.1: ; %Flow15977; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s415978; GFX12-NEXT: s_cbranch_vccz .LBB119_415979; GFX12-NEXT: .LBB119_2: ; %atomicrmw.phi15980; GFX12-NEXT: s_endpgm15981; GFX12-NEXT: .LBB119_3: ; %atomicrmw.global15982; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s115983; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s315984; GFX12-NEXT: flat_atomic_dec_u64 v[0:1], v[2:3] scope:SCOPE_DEV15985; GFX12-NEXT: s_wait_storecnt_dscnt 0x015986; GFX12-NEXT: global_inv scope:SCOPE_DEV15987; GFX12-NEXT: s_cbranch_execnz .LBB119_215988; GFX12-NEXT: .LBB119_4: ; %atomicrmw.private15989; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 015990; GFX12-NEXT: s_cselect_b32 s4, s0, -115991; GFX12-NEXT: scratch_load_b64 v[0:1], off, s415992; GFX12-NEXT: s_wait_loadcnt 0x015993; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]15994; GFX12-NEXT: v_cmp_lt_u64_e64 s0, s[2:3], v[0:1]15995; GFX12-NEXT: v_add_co_u32 v0, s1, v0, -115996; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)15997; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, s115998; GFX12-NEXT: s_or_b32 s0, vcc_lo, s015999; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)16000; GFX12-NEXT: v_cndmask_b32_e64 v1, v1, s3, s016001; GFX12-NEXT: v_cndmask_b32_e64 v0, v0, s2, s016002; GFX12-NEXT: scratch_store_b64 off, v[0:1], s416003; GFX12-NEXT: s_endpgm16004entry:16005 %tmp0 = atomicrmw volatile udec_wrap ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !016006 ret void16007}16008 16009define amdgpu_kernel void @atomic_dec_i64_ret(ptr %out, ptr %out2, i64 %in) {16010; GCN1-LABEL: atomic_dec_i64_ret:16011; GCN1: ; %bb.0: ; %entry16012; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD016013; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD116014; GCN1-NEXT: s_mov_b32 s14, -116015; GCN1-NEXT: s_mov_b32 s15, 0xe8f00016016; GCN1-NEXT: s_add_u32 s12, s12, s1116017; GCN1-NEXT: s_load_dword s2, s[4:5], 0x3f16018; GCN1-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x916019; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd16020; GCN1-NEXT: s_addc_u32 s13, s13, 016021; GCN1-NEXT: s_waitcnt lgkmcnt(0)16022; GCN1-NEXT: s_cmp_eq_u32 s9, s216023; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 016024; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]16025; GCN1-NEXT: s_cbranch_vccz .LBB120_216026; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global16027; GCN1-NEXT: v_mov_b32_e32 v0, s816028; GCN1-NEXT: v_mov_b32_e32 v3, s116029; GCN1-NEXT: v_mov_b32_e32 v1, s916030; GCN1-NEXT: v_mov_b32_e32 v2, s016031; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc16032; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)16033; GCN1-NEXT: buffer_wbinvl1_vol16034; GCN1-NEXT: s_cbranch_execz .LBB120_316035; GCN1-NEXT: s_branch .LBB120_416036; GCN1-NEXT: .LBB120_2:16037; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr116038; GCN1-NEXT: .LBB120_3: ; %atomicrmw.private16039; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[8:9], 016040; GCN1-NEXT: v_mov_b32_e32 v4, s116041; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec16042; GCN1-NEXT: s_cselect_b32 s2, s8, -116043; GCN1-NEXT: v_mov_b32_e32 v2, s216044; GCN1-NEXT: s_add_i32 s2, s2, 416045; GCN1-NEXT: v_mov_b32_e32 v3, s216046; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen16047; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen16048; GCN1-NEXT: v_mov_b32_e32 v5, s016049; GCN1-NEXT: s_waitcnt vmcnt(1)16050; GCN1-NEXT: v_add_i32_e64 v6, s[2:3], -1, v016051; GCN1-NEXT: s_waitcnt vmcnt(0)16052; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]16053; GCN1-NEXT: v_cmp_lt_u64_e64 s[0:1], s[0:1], v[0:1]16054; GCN1-NEXT: v_addc_u32_e64 v7, s[2:3], -1, v1, s[2:3]16055; GCN1-NEXT: s_or_b64 vcc, vcc, s[0:1]16056; GCN1-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc16057; GCN1-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc16058; GCN1-NEXT: buffer_store_dword v5, v2, s[12:15], 0 offen16059; GCN1-NEXT: buffer_store_dword v4, v3, s[12:15], 0 offen16060; GCN1-NEXT: .LBB120_4: ; %atomicrmw.end16061; GCN1-NEXT: v_mov_b32_e32 v2, s1016062; GCN1-NEXT: v_mov_b32_e32 v3, s1116063; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]16064; GCN1-NEXT: s_endpgm16065;16066; GCN2-LABEL: atomic_dec_i64_ret:16067; GCN2: ; %bb.0: ; %entry16068; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD016069; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD116070; GCN2-NEXT: s_mov_b32 s90, -116071; GCN2-NEXT: s_mov_b32 s91, 0xe8000016072; GCN2-NEXT: s_add_u32 s88, s88, s1116073; GCN2-NEXT: s_load_dword s2, s[4:5], 0xfc16074; GCN2-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x2416075; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x3416076; GCN2-NEXT: s_addc_u32 s89, s89, 016077; GCN2-NEXT: s_waitcnt lgkmcnt(0)16078; GCN2-NEXT: s_cmp_eq_u32 s9, s216079; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 016080; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]16081; GCN2-NEXT: s_cbranch_vccz .LBB120_216082; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global16083; GCN2-NEXT: v_mov_b32_e32 v0, s816084; GCN2-NEXT: v_mov_b32_e32 v3, s116085; GCN2-NEXT: v_mov_b32_e32 v1, s916086; GCN2-NEXT: v_mov_b32_e32 v2, s016087; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc16088; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)16089; GCN2-NEXT: buffer_wbinvl1_vol16090; GCN2-NEXT: s_cbranch_execz .LBB120_316091; GCN2-NEXT: s_branch .LBB120_416092; GCN2-NEXT: .LBB120_2:16093; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr116094; GCN2-NEXT: .LBB120_3: ; %atomicrmw.private16095; GCN2-NEXT: s_cmp_lg_u64 s[8:9], 016096; GCN2-NEXT: s_cselect_b32 s2, s8, -116097; GCN2-NEXT: v_mov_b32_e32 v2, s216098; GCN2-NEXT: s_add_i32 s2, s2, 416099; GCN2-NEXT: v_mov_b32_e32 v3, s216100; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen16101; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen16102; GCN2-NEXT: v_mov_b32_e32 v4, s116103; GCN2-NEXT: v_mov_b32_e32 v5, s016104; GCN2-NEXT: s_waitcnt vmcnt(1)16105; GCN2-NEXT: v_add_u32_e64 v6, s[2:3], -1, v016106; GCN2-NEXT: s_waitcnt vmcnt(0)16107; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]16108; GCN2-NEXT: v_cmp_lt_u64_e64 s[0:1], s[0:1], v[0:1]16109; GCN2-NEXT: v_addc_u32_e64 v7, s[2:3], -1, v1, s[2:3]16110; GCN2-NEXT: s_or_b64 vcc, vcc, s[0:1]16111; GCN2-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc16112; GCN2-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc16113; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen16114; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen16115; GCN2-NEXT: .LBB120_4: ; %atomicrmw.end16116; GCN2-NEXT: v_mov_b32_e32 v2, s1016117; GCN2-NEXT: v_mov_b32_e32 v3, s1116118; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]16119; GCN2-NEXT: s_endpgm16120;16121; GFX12-LABEL: atomic_dec_i64_ret:16122; GFX12: ; %bb.0: ; %entry16123; GFX12-NEXT: s_clause 0x116124; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2416125; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x3416126; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base16127; GFX12-NEXT: s_wait_kmcnt 0x016128; GFX12-NEXT: s_cmp_eq_u32 s1, s716129; GFX12-NEXT: s_cselect_b32 s6, -1, 016130; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)16131; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s616132; GFX12-NEXT: s_cbranch_vccz .LBB120_216133; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global16134; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s116135; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s516136; GFX12-NEXT: flat_atomic_dec_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV16137; GFX12-NEXT: s_wait_loadcnt_dscnt 0x016138; GFX12-NEXT: global_inv scope:SCOPE_DEV16139; GFX12-NEXT: s_cbranch_execz .LBB120_316140; GFX12-NEXT: s_branch .LBB120_416141; GFX12-NEXT: .LBB120_2:16142; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr116143; GFX12-NEXT: .LBB120_3: ; %atomicrmw.private16144; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 016145; GFX12-NEXT: s_cselect_b32 s6, s0, -116146; GFX12-NEXT: scratch_load_b64 v[0:1], off, s616147; GFX12-NEXT: s_wait_loadcnt 0x016148; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]16149; GFX12-NEXT: v_cmp_lt_u64_e64 s0, s[4:5], v[0:1]16150; GFX12-NEXT: v_add_co_u32 v2, s1, v0, -116151; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)16152; GFX12-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v1, s116153; GFX12-NEXT: s_or_b32 s0, vcc_lo, s016154; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)16155; GFX12-NEXT: v_cndmask_b32_e64 v3, v3, s5, s016156; GFX12-NEXT: v_cndmask_b32_e64 v2, v2, s4, s016157; GFX12-NEXT: scratch_store_b64 off, v[2:3], s616158; GFX12-NEXT: .LBB120_4: ; %atomicrmw.end16159; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s316160; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]16161; GFX12-NEXT: s_endpgm16162entry:16163 %tmp0 = atomicrmw volatile udec_wrap ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !016164 store i64 %tmp0, ptr %out216165 ret void16166}16167 16168define amdgpu_kernel void @atomic_dec_i64_decr64(ptr %out, i64 %in, i64 %index) {16169; GCN1-LABEL: atomic_dec_i64_decr64:16170; GCN1: ; %bb.0: ; %entry16171; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD016172; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD116173; GCN1-NEXT: s_mov_b32 s14, -116174; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd16175; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x916176; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f16177; GCN1-NEXT: s_mov_b32 s15, 0xe8f00016178; GCN1-NEXT: s_add_u32 s12, s12, s1116179; GCN1-NEXT: s_addc_u32 s13, s13, 016180; GCN1-NEXT: s_waitcnt lgkmcnt(0)16181; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 316182; GCN1-NEXT: s_add_u32 s0, s0, s416183; GCN1-NEXT: s_addc_u32 s1, s1, s516184; GCN1-NEXT: s_cmp_eq_u32 s1, s816185; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 016186; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]16187; GCN1-NEXT: s_mov_b64 s[4:5], -116188; GCN1-NEXT: s_cbranch_vccnz .LBB121_316189; GCN1-NEXT: ; %bb.1: ; %Flow16190; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]16191; GCN1-NEXT: s_cbranch_vccz .LBB121_416192; GCN1-NEXT: .LBB121_2: ; %atomicrmw.phi16193; GCN1-NEXT: s_endpgm16194; GCN1-NEXT: .LBB121_3: ; %atomicrmw.global16195; GCN1-NEXT: v_mov_b32_e32 v0, s016196; GCN1-NEXT: v_mov_b32_e32 v2, s216197; GCN1-NEXT: v_mov_b32_e32 v1, s116198; GCN1-NEXT: v_mov_b32_e32 v3, s316199; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]16200; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)16201; GCN1-NEXT: buffer_wbinvl1_vol16202; GCN1-NEXT: s_cbranch_execnz .LBB121_216203; GCN1-NEXT: .LBB121_4: ; %atomicrmw.private16204; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 016205; GCN1-NEXT: v_mov_b32_e32 v4, s316206; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec16207; GCN1-NEXT: s_cselect_b32 s0, s0, -116208; GCN1-NEXT: v_mov_b32_e32 v2, s016209; GCN1-NEXT: s_add_i32 s0, s0, 416210; GCN1-NEXT: v_mov_b32_e32 v3, s016211; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen16212; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen16213; GCN1-NEXT: v_mov_b32_e32 v5, s216214; GCN1-NEXT: s_waitcnt vmcnt(0)16215; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]16216; GCN1-NEXT: v_cmp_lt_u64_e64 s[0:1], s[2:3], v[0:1]16217; GCN1-NEXT: v_add_i32_e64 v0, s[2:3], -1, v016218; GCN1-NEXT: s_or_b64 vcc, vcc, s[0:1]16219; GCN1-NEXT: v_addc_u32_e64 v1, s[2:3], -1, v1, s[2:3]16220; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc16221; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc16222; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen16223; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen16224; GCN1-NEXT: s_endpgm16225;16226; GCN2-LABEL: atomic_dec_i64_decr64:16227; GCN2: ; %bb.0: ; %entry16228; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD016229; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD116230; GCN2-NEXT: s_mov_b32 s90, -116231; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3416232; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2416233; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc16234; GCN2-NEXT: s_mov_b32 s91, 0xe8000016235; GCN2-NEXT: s_add_u32 s88, s88, s1116236; GCN2-NEXT: s_addc_u32 s89, s89, 016237; GCN2-NEXT: s_waitcnt lgkmcnt(0)16238; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 316239; GCN2-NEXT: s_add_u32 s0, s0, s416240; GCN2-NEXT: s_addc_u32 s1, s1, s516241; GCN2-NEXT: s_cmp_eq_u32 s1, s816242; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 016243; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]16244; GCN2-NEXT: s_mov_b64 s[4:5], -116245; GCN2-NEXT: s_cbranch_vccnz .LBB121_316246; GCN2-NEXT: ; %bb.1: ; %Flow16247; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]16248; GCN2-NEXT: s_cbranch_vccz .LBB121_416249; GCN2-NEXT: .LBB121_2: ; %atomicrmw.phi16250; GCN2-NEXT: s_endpgm16251; GCN2-NEXT: .LBB121_3: ; %atomicrmw.global16252; GCN2-NEXT: v_mov_b32_e32 v0, s016253; GCN2-NEXT: v_mov_b32_e32 v2, s216254; GCN2-NEXT: v_mov_b32_e32 v1, s116255; GCN2-NEXT: v_mov_b32_e32 v3, s316256; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]16257; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)16258; GCN2-NEXT: buffer_wbinvl1_vol16259; GCN2-NEXT: s_cbranch_execnz .LBB121_216260; GCN2-NEXT: .LBB121_4: ; %atomicrmw.private16261; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 016262; GCN2-NEXT: s_cselect_b32 s0, s0, -116263; GCN2-NEXT: v_mov_b32_e32 v2, s016264; GCN2-NEXT: s_add_i32 s0, s0, 416265; GCN2-NEXT: v_mov_b32_e32 v3, s016266; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen16267; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen16268; GCN2-NEXT: v_mov_b32_e32 v4, s316269; GCN2-NEXT: v_mov_b32_e32 v5, s216270; GCN2-NEXT: s_waitcnt vmcnt(0)16271; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]16272; GCN2-NEXT: v_cmp_lt_u64_e64 s[0:1], s[2:3], v[0:1]16273; GCN2-NEXT: v_add_u32_e64 v0, s[2:3], -1, v016274; GCN2-NEXT: s_or_b64 vcc, vcc, s[0:1]16275; GCN2-NEXT: v_addc_u32_e64 v1, s[2:3], -1, v1, s[2:3]16276; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc16277; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc16278; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen16279; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen16280; GCN2-NEXT: s_endpgm16281;16282; GFX12-LABEL: atomic_dec_i64_decr64:16283; GFX12: ; %bb.0: ; %entry16284; GFX12-NEXT: s_clause 0x116285; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x3416286; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2416287; GFX12-NEXT: s_mov_b64 s[6:7], src_private_base16288; GFX12-NEXT: s_wait_kmcnt 0x016289; GFX12-NEXT: s_lshl_b64 s[4:5], s[8:9], 316290; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)16291; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]16292; GFX12-NEXT: s_cmp_eq_u32 s1, s716293; GFX12-NEXT: s_cselect_b32 s4, -1, 016294; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)16295; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s416296; GFX12-NEXT: s_mov_b32 s4, -116297; GFX12-NEXT: s_cbranch_vccnz .LBB121_316298; GFX12-NEXT: ; %bb.1: ; %Flow16299; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s416300; GFX12-NEXT: s_cbranch_vccz .LBB121_416301; GFX12-NEXT: .LBB121_2: ; %atomicrmw.phi16302; GFX12-NEXT: s_endpgm16303; GFX12-NEXT: .LBB121_3: ; %atomicrmw.global16304; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s116305; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s316306; GFX12-NEXT: flat_atomic_dec_u64 v[0:1], v[2:3] scope:SCOPE_DEV16307; GFX12-NEXT: s_wait_storecnt_dscnt 0x016308; GFX12-NEXT: global_inv scope:SCOPE_DEV16309; GFX12-NEXT: s_cbranch_execnz .LBB121_216310; GFX12-NEXT: .LBB121_4: ; %atomicrmw.private16311; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 016312; GFX12-NEXT: s_cselect_b32 s4, s0, -116313; GFX12-NEXT: scratch_load_b64 v[0:1], off, s416314; GFX12-NEXT: s_wait_loadcnt 0x016315; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]16316; GFX12-NEXT: v_cmp_lt_u64_e64 s0, s[2:3], v[0:1]16317; GFX12-NEXT: v_add_co_u32 v0, s1, v0, -116318; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)16319; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, s116320; GFX12-NEXT: s_or_b32 s0, vcc_lo, s016321; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)16322; GFX12-NEXT: v_cndmask_b32_e64 v1, v1, s3, s016323; GFX12-NEXT: v_cndmask_b32_e64 v0, v0, s2, s016324; GFX12-NEXT: scratch_store_b64 off, v[0:1], s416325; GFX12-NEXT: s_endpgm16326entry:16327 %ptr = getelementptr inbounds i64, ptr %out, i64 %index16328 %tmp0 = atomicrmw volatile udec_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !016329 ret void16330}16331 16332define amdgpu_kernel void @atomic_dec_i64_ret_decr64(ptr %out, ptr %out2, i64 %in, i64 %index) {16333; GCN1-LABEL: atomic_dec_i64_ret_decr64:16334; GCN1: ; %bb.0: ; %entry16335; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD016336; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD116337; GCN1-NEXT: s_mov_b32 s18, -116338; GCN1-NEXT: s_mov_b32 s19, 0xe8f00016339; GCN1-NEXT: s_add_u32 s16, s16, s1116340; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x916341; GCN1-NEXT: s_load_dword s2, s[4:5], 0x4116342; GCN1-NEXT: s_addc_u32 s17, s17, 016343; GCN1-NEXT: s_waitcnt lgkmcnt(0)16344; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 316345; GCN1-NEXT: s_add_u32 s0, s8, s016346; GCN1-NEXT: s_addc_u32 s1, s9, s116347; GCN1-NEXT: s_cmp_eq_u32 s1, s216348; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 016349; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]16350; GCN1-NEXT: s_cbranch_vccz .LBB122_216351; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global16352; GCN1-NEXT: v_mov_b32_e32 v0, s016353; GCN1-NEXT: v_mov_b32_e32 v2, s1216354; GCN1-NEXT: v_mov_b32_e32 v1, s116355; GCN1-NEXT: v_mov_b32_e32 v3, s1316356; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc16357; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)16358; GCN1-NEXT: buffer_wbinvl1_vol16359; GCN1-NEXT: s_cbranch_execz .LBB122_316360; GCN1-NEXT: s_branch .LBB122_416361; GCN1-NEXT: .LBB122_2:16362; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr116363; GCN1-NEXT: .LBB122_3: ; %atomicrmw.private16364; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 016365; GCN1-NEXT: v_mov_b32_e32 v5, s1216366; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec16367; GCN1-NEXT: s_cselect_b32 s0, s0, -116368; GCN1-NEXT: v_mov_b32_e32 v2, s016369; GCN1-NEXT: s_add_i32 s0, s0, 416370; GCN1-NEXT: v_mov_b32_e32 v3, s016371; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen16372; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen16373; GCN1-NEXT: v_mov_b32_e32 v4, s1316374; GCN1-NEXT: s_waitcnt vmcnt(1)16375; GCN1-NEXT: v_add_i32_e64 v6, s[2:3], -1, v016376; GCN1-NEXT: s_waitcnt vmcnt(0)16377; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]16378; GCN1-NEXT: v_cmp_lt_u64_e64 s[0:1], s[12:13], v[0:1]16379; GCN1-NEXT: v_addc_u32_e64 v7, s[2:3], -1, v1, s[2:3]16380; GCN1-NEXT: s_or_b64 vcc, vcc, s[0:1]16381; GCN1-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc16382; GCN1-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc16383; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen16384; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen16385; GCN1-NEXT: .LBB122_4: ; %atomicrmw.end16386; GCN1-NEXT: v_mov_b32_e32 v2, s1016387; GCN1-NEXT: v_mov_b32_e32 v3, s1116388; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]16389; GCN1-NEXT: s_endpgm16390;16391; GCN2-LABEL: atomic_dec_i64_ret_decr64:16392; GCN2: ; %bb.0: ; %entry16393; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD016394; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD116395; GCN2-NEXT: s_mov_b32 s90, -116396; GCN2-NEXT: s_mov_b32 s91, 0xe8000016397; GCN2-NEXT: s_add_u32 s88, s88, s1116398; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2416399; GCN2-NEXT: s_load_dword s2, s[4:5], 0x10416400; GCN2-NEXT: s_addc_u32 s89, s89, 016401; GCN2-NEXT: s_waitcnt lgkmcnt(0)16402; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 316403; GCN2-NEXT: s_add_u32 s0, s8, s016404; GCN2-NEXT: s_addc_u32 s1, s9, s116405; GCN2-NEXT: s_cmp_eq_u32 s1, s216406; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 016407; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]16408; GCN2-NEXT: s_cbranch_vccz .LBB122_216409; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global16410; GCN2-NEXT: v_mov_b32_e32 v0, s016411; GCN2-NEXT: v_mov_b32_e32 v2, s1216412; GCN2-NEXT: v_mov_b32_e32 v1, s116413; GCN2-NEXT: v_mov_b32_e32 v3, s1316414; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc16415; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)16416; GCN2-NEXT: buffer_wbinvl1_vol16417; GCN2-NEXT: s_cbranch_execz .LBB122_316418; GCN2-NEXT: s_branch .LBB122_416419; GCN2-NEXT: .LBB122_2:16420; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr116421; GCN2-NEXT: .LBB122_3: ; %atomicrmw.private16422; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 016423; GCN2-NEXT: s_cselect_b32 s0, s0, -116424; GCN2-NEXT: v_mov_b32_e32 v2, s016425; GCN2-NEXT: s_add_i32 s0, s0, 416426; GCN2-NEXT: v_mov_b32_e32 v3, s016427; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen16428; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen16429; GCN2-NEXT: v_mov_b32_e32 v5, s1216430; GCN2-NEXT: v_mov_b32_e32 v4, s1316431; GCN2-NEXT: s_waitcnt vmcnt(1)16432; GCN2-NEXT: v_add_u32_e64 v6, s[2:3], -1, v016433; GCN2-NEXT: s_waitcnt vmcnt(0)16434; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]16435; GCN2-NEXT: v_cmp_lt_u64_e64 s[0:1], s[12:13], v[0:1]16436; GCN2-NEXT: v_addc_u32_e64 v7, s[2:3], -1, v1, s[2:3]16437; GCN2-NEXT: s_or_b64 vcc, vcc, s[0:1]16438; GCN2-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc16439; GCN2-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc16440; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen16441; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen16442; GCN2-NEXT: .LBB122_4: ; %atomicrmw.end16443; GCN2-NEXT: v_mov_b32_e32 v2, s1016444; GCN2-NEXT: v_mov_b32_e32 v3, s1116445; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]16446; GCN2-NEXT: s_endpgm16447;16448; GFX12-LABEL: atomic_dec_i64_ret_decr64:16449; GFX12: ; %bb.0: ; %entry16450; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x2416451; GFX12-NEXT: s_mov_b64 s[8:9], src_private_base16452; GFX12-NEXT: s_wait_kmcnt 0x016453; GFX12-NEXT: s_lshl_b64 s[6:7], s[6:7], 316454; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)16455; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]16456; GFX12-NEXT: s_cmp_eq_u32 s1, s916457; GFX12-NEXT: s_cselect_b32 s6, -1, 016458; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)16459; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s616460; GFX12-NEXT: s_cbranch_vccz .LBB122_216461; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global16462; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s116463; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s516464; GFX12-NEXT: flat_atomic_dec_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV16465; GFX12-NEXT: s_wait_loadcnt_dscnt 0x016466; GFX12-NEXT: global_inv scope:SCOPE_DEV16467; GFX12-NEXT: s_cbranch_execz .LBB122_316468; GFX12-NEXT: s_branch .LBB122_416469; GFX12-NEXT: .LBB122_2:16470; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr116471; GFX12-NEXT: .LBB122_3: ; %atomicrmw.private16472; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 016473; GFX12-NEXT: s_cselect_b32 s6, s0, -116474; GFX12-NEXT: scratch_load_b64 v[0:1], off, s616475; GFX12-NEXT: s_wait_loadcnt 0x016476; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]16477; GFX12-NEXT: v_cmp_lt_u64_e64 s0, s[4:5], v[0:1]16478; GFX12-NEXT: v_add_co_u32 v2, s1, v0, -116479; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)16480; GFX12-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v1, s116481; GFX12-NEXT: s_or_b32 s0, vcc_lo, s016482; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)16483; GFX12-NEXT: v_cndmask_b32_e64 v3, v3, s5, s016484; GFX12-NEXT: v_cndmask_b32_e64 v2, v2, s4, s016485; GFX12-NEXT: scratch_store_b64 off, v[2:3], s616486; GFX12-NEXT: .LBB122_4: ; %atomicrmw.end16487; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s316488; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]16489; GFX12-NEXT: s_endpgm16490entry:16491 %ptr = getelementptr inbounds i64, ptr %out, i64 %index16492 %tmp0 = atomicrmw volatile udec_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !016493 store i64 %tmp0, ptr %out216494 ret void16495}16496 16497!0 = !{}16498