brintos / llvm-project-archived public Read only

1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -mcpu=bonaire < %s | FileCheck -check-prefix=GFX7 %s3; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=GFX8 %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s5 6define amdgpu_kernel void @atomic_add_i64_offset(ptr %out, i64 %in) {7; GFX7-LABEL: atomic_add_i64_offset:8; GFX7:       ; %bb.0: ; %entry9; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x910; GFX7-NEXT:    s_waitcnt lgkmcnt(0)11; GFX7-NEXT:    s_add_u32 s0, s0, 3212; GFX7-NEXT:    s_addc_u32 s1, s1, 013; GFX7-NEXT:    v_mov_b32_e32 v3, s114; GFX7-NEXT:    v_mov_b32_e32 v0, s215; GFX7-NEXT:    v_mov_b32_e32 v1, s316; GFX7-NEXT:    v_mov_b32_e32 v2, s017; GFX7-NEXT:    flat_atomic_add_x2 v[2:3], v[0:1]18; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)19; GFX7-NEXT:    buffer_wbinvl1_vol20; GFX7-NEXT:    s_endpgm21;22; GFX8-LABEL: atomic_add_i64_offset:23; GFX8:       ; %bb.0: ; %entry24; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2425; GFX8-NEXT:    s_waitcnt lgkmcnt(0)26; GFX8-NEXT:    s_add_u32 s0, s0, 3227; GFX8-NEXT:    s_addc_u32 s1, s1, 028; GFX8-NEXT:    v_mov_b32_e32 v3, s129; GFX8-NEXT:    v_mov_b32_e32 v0, s230; GFX8-NEXT:    v_mov_b32_e32 v1, s331; GFX8-NEXT:    v_mov_b32_e32 v2, s032; GFX8-NEXT:    flat_atomic_add_x2 v[2:3], v[0:1]33; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)34; GFX8-NEXT:    buffer_wbinvl1_vol35; GFX8-NEXT:    s_endpgm36;37; GFX12-LABEL: atomic_add_i64_offset:38; GFX12:       ; %bb.0: ; %entry39; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2440; GFX12-NEXT:    s_wait_kmcnt 0x041; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s142; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s343; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[2:3] offset:32 scope:SCOPE_DEV44; GFX12-NEXT:    s_wait_storecnt_dscnt 0x045; GFX12-NEXT:    global_inv scope:SCOPE_DEV46; GFX12-NEXT:    s_endpgm47entry:48  %gep = getelementptr inbounds i64, ptr %out, i64 449  %tmp0 = atomicrmw volatile add ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !050  ret void51}52 53define amdgpu_kernel void @atomic_add_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {54; GFX7-LABEL: atomic_add_i64_ret_offset:55; GFX7:       ; %bb.0: ; %entry56; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd57; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x958; GFX7-NEXT:    s_waitcnt lgkmcnt(0)59; GFX7-NEXT:    v_mov_b32_e32 v0, s660; GFX7-NEXT:    s_add_u32 s0, s0, 3261; GFX7-NEXT:    s_addc_u32 s1, s1, 062; GFX7-NEXT:    v_mov_b32_e32 v3, s163; GFX7-NEXT:    v_mov_b32_e32 v1, s764; GFX7-NEXT:    v_mov_b32_e32 v2, s065; GFX7-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3], v[0:1] glc66; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)67; GFX7-NEXT:    buffer_wbinvl1_vol68; GFX7-NEXT:    v_mov_b32_e32 v2, s269; GFX7-NEXT:    v_mov_b32_e32 v3, s370; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]71; GFX7-NEXT:    s_endpgm72;73; GFX8-LABEL: atomic_add_i64_ret_offset:74; GFX8:       ; %bb.0: ; %entry75; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3476; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2477; GFX8-NEXT:    s_waitcnt lgkmcnt(0)78; GFX8-NEXT:    v_mov_b32_e32 v0, s679; GFX8-NEXT:    s_add_u32 s0, s0, 3280; GFX8-NEXT:    s_addc_u32 s1, s1, 081; GFX8-NEXT:    v_mov_b32_e32 v3, s182; GFX8-NEXT:    v_mov_b32_e32 v1, s783; GFX8-NEXT:    v_mov_b32_e32 v2, s084; GFX8-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3], v[0:1] glc85; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)86; GFX8-NEXT:    buffer_wbinvl1_vol87; GFX8-NEXT:    v_mov_b32_e32 v2, s288; GFX8-NEXT:    v_mov_b32_e32 v3, s389; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]90; GFX8-NEXT:    s_endpgm91;92; GFX12-LABEL: atomic_add_i64_ret_offset:93; GFX12:       ; %bb.0: ; %entry94; GFX12-NEXT:    s_clause 0x195; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2496; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x3497; GFX12-NEXT:    s_wait_kmcnt 0x098; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s199; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5100; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[0:1], v[2:3] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV101; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0102; GFX12-NEXT:    global_inv scope:SCOPE_DEV103; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3104; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]105; GFX12-NEXT:    s_endpgm106entry:107  %gep = getelementptr inbounds i64, ptr %out, i64 4108  %tmp0 = atomicrmw volatile add ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !0109  store i64 %tmp0, ptr %out2110  ret void111}112 113define amdgpu_kernel void @atomic_add_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {114; GFX7-LABEL: atomic_add_i64_addr64_offset:115; GFX7:       ; %bb.0: ; %entry116; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9117; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd118; GFX7-NEXT:    s_waitcnt lgkmcnt(0)119; GFX7-NEXT:    v_mov_b32_e32 v0, s2120; GFX7-NEXT:    v_mov_b32_e32 v1, s3121; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3122; GFX7-NEXT:    s_add_u32 s0, s0, s2123; GFX7-NEXT:    s_addc_u32 s1, s1, s3124; GFX7-NEXT:    s_add_u32 s0, s0, 32125; GFX7-NEXT:    s_addc_u32 s1, s1, 0126; GFX7-NEXT:    v_mov_b32_e32 v3, s1127; GFX7-NEXT:    v_mov_b32_e32 v2, s0128; GFX7-NEXT:    flat_atomic_add_x2 v[2:3], v[0:1]129; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)130; GFX7-NEXT:    buffer_wbinvl1_vol131; GFX7-NEXT:    s_endpgm132;133; GFX8-LABEL: atomic_add_i64_addr64_offset:134; GFX8:       ; %bb.0: ; %entry135; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24136; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34137; GFX8-NEXT:    s_waitcnt lgkmcnt(0)138; GFX8-NEXT:    v_mov_b32_e32 v0, s2139; GFX8-NEXT:    v_mov_b32_e32 v1, s3140; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3141; GFX8-NEXT:    s_add_u32 s0, s0, s2142; GFX8-NEXT:    s_addc_u32 s1, s1, s3143; GFX8-NEXT:    s_add_u32 s0, s0, 32144; GFX8-NEXT:    s_addc_u32 s1, s1, 0145; GFX8-NEXT:    v_mov_b32_e32 v3, s1146; GFX8-NEXT:    v_mov_b32_e32 v2, s0147; GFX8-NEXT:    flat_atomic_add_x2 v[2:3], v[0:1]148; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)149; GFX8-NEXT:    buffer_wbinvl1_vol150; GFX8-NEXT:    s_endpgm151;152; GFX12-LABEL: atomic_add_i64_addr64_offset:153; GFX12:       ; %bb.0: ; %entry154; GFX12-NEXT:    s_clause 0x1155; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34156; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24157; GFX12-NEXT:    s_wait_kmcnt 0x0158; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3159; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3160; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]161; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)162; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0163; GFX12-NEXT:    flat_atomic_add_u64 v[2:3], v[0:1] offset:32 scope:SCOPE_DEV164; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0165; GFX12-NEXT:    global_inv scope:SCOPE_DEV166; GFX12-NEXT:    s_endpgm167entry:168  %ptr = getelementptr inbounds i64, ptr %out, i64 %index169  %gep = getelementptr inbounds i64, ptr %ptr, i64 4170  %tmp0 = atomicrmw volatile add ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !0171  ret void172}173 174define amdgpu_kernel void @atomic_add_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {175; GFX7-LABEL: atomic_add_i64_ret_addr64_offset:176; GFX7:       ; %bb.0: ; %entry177; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9178; GFX7-NEXT:    s_waitcnt lgkmcnt(0)179; GFX7-NEXT:    v_mov_b32_e32 v0, s4180; GFX7-NEXT:    v_mov_b32_e32 v1, s5181; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3182; GFX7-NEXT:    s_add_u32 s0, s0, s4183; GFX7-NEXT:    s_addc_u32 s1, s1, s5184; GFX7-NEXT:    s_add_u32 s0, s0, 32185; GFX7-NEXT:    s_addc_u32 s1, s1, 0186; GFX7-NEXT:    v_mov_b32_e32 v3, s1187; GFX7-NEXT:    v_mov_b32_e32 v2, s0188; GFX7-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3], v[0:1] glc189; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)190; GFX7-NEXT:    buffer_wbinvl1_vol191; GFX7-NEXT:    v_mov_b32_e32 v2, s2192; GFX7-NEXT:    v_mov_b32_e32 v3, s3193; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]194; GFX7-NEXT:    s_endpgm195;196; GFX8-LABEL: atomic_add_i64_ret_addr64_offset:197; GFX8:       ; %bb.0: ; %entry198; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24199; GFX8-NEXT:    s_waitcnt lgkmcnt(0)200; GFX8-NEXT:    v_mov_b32_e32 v0, s4201; GFX8-NEXT:    v_mov_b32_e32 v1, s5202; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3203; GFX8-NEXT:    s_add_u32 s0, s0, s4204; GFX8-NEXT:    s_addc_u32 s1, s1, s5205; GFX8-NEXT:    s_add_u32 s0, s0, 32206; GFX8-NEXT:    s_addc_u32 s1, s1, 0207; GFX8-NEXT:    v_mov_b32_e32 v3, s1208; GFX8-NEXT:    v_mov_b32_e32 v2, s0209; GFX8-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3], v[0:1] glc210; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)211; GFX8-NEXT:    buffer_wbinvl1_vol212; GFX8-NEXT:    v_mov_b32_e32 v2, s2213; GFX8-NEXT:    v_mov_b32_e32 v3, s3214; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]215; GFX8-NEXT:    s_endpgm216;217; GFX12-LABEL: atomic_add_i64_ret_addr64_offset:218; GFX12:       ; %bb.0: ; %entry219; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24220; GFX12-NEXT:    s_wait_kmcnt 0x0221; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3222; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5223; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]224; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)225; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0226; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[2:3], v[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV227; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0228; GFX12-NEXT:    global_inv scope:SCOPE_DEV229; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3230; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]231; GFX12-NEXT:    s_endpgm232entry:233  %ptr = getelementptr inbounds i64, ptr %out, i64 %index234  %gep = getelementptr inbounds i64, ptr %ptr, i64 4235  %tmp0 = atomicrmw volatile add ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !0236  store i64 %tmp0, ptr %out2237  ret void238}239 240define amdgpu_kernel void @atomic_add_i64(ptr %out, i64 %in) {241; GFX7-LABEL: atomic_add_i64:242; GFX7:       ; %bb.0: ; %entry243; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9244; GFX7-NEXT:    s_waitcnt lgkmcnt(0)245; GFX7-NEXT:    v_mov_b32_e32 v0, s0246; GFX7-NEXT:    v_mov_b32_e32 v1, s1247; GFX7-NEXT:    v_mov_b32_e32 v2, s2248; GFX7-NEXT:    v_mov_b32_e32 v3, s3249; GFX7-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3]250; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)251; GFX7-NEXT:    buffer_wbinvl1_vol252; GFX7-NEXT:    s_endpgm253;254; GFX8-LABEL: atomic_add_i64:255; GFX8:       ; %bb.0: ; %entry256; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24257; GFX8-NEXT:    s_waitcnt lgkmcnt(0)258; GFX8-NEXT:    v_mov_b32_e32 v0, s0259; GFX8-NEXT:    v_mov_b32_e32 v1, s1260; GFX8-NEXT:    v_mov_b32_e32 v2, s2261; GFX8-NEXT:    v_mov_b32_e32 v3, s3262; GFX8-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3]263; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)264; GFX8-NEXT:    buffer_wbinvl1_vol265; GFX8-NEXT:    s_endpgm266;267; GFX12-LABEL: atomic_add_i64:268; GFX12:       ; %bb.0: ; %entry269; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24270; GFX12-NEXT:    s_wait_kmcnt 0x0271; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1272; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3273; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[2:3] scope:SCOPE_DEV274; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0275; GFX12-NEXT:    global_inv scope:SCOPE_DEV276; GFX12-NEXT:    s_endpgm277entry:278  %tmp0 = atomicrmw volatile add ptr %out, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !0279  ret void280}281 282define amdgpu_kernel void @atomic_add_i64_ret(ptr %out, ptr %out2, i64 %in) {283; GFX7-LABEL: atomic_add_i64_ret:284; GFX7:       ; %bb.0: ; %entry285; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9286; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd287; GFX7-NEXT:    s_waitcnt lgkmcnt(0)288; GFX7-NEXT:    v_mov_b32_e32 v0, s0289; GFX7-NEXT:    v_mov_b32_e32 v1, s1290; GFX7-NEXT:    v_mov_b32_e32 v2, s4291; GFX7-NEXT:    v_mov_b32_e32 v3, s5292; GFX7-NEXT:    flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc293; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)294; GFX7-NEXT:    buffer_wbinvl1_vol295; GFX7-NEXT:    v_mov_b32_e32 v2, s2296; GFX7-NEXT:    v_mov_b32_e32 v3, s3297; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]298; GFX7-NEXT:    s_endpgm299;300; GFX8-LABEL: atomic_add_i64_ret:301; GFX8:       ; %bb.0: ; %entry302; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24303; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34304; GFX8-NEXT:    s_waitcnt lgkmcnt(0)305; GFX8-NEXT:    v_mov_b32_e32 v0, s0306; GFX8-NEXT:    v_mov_b32_e32 v1, s1307; GFX8-NEXT:    v_mov_b32_e32 v2, s4308; GFX8-NEXT:    v_mov_b32_e32 v3, s5309; GFX8-NEXT:    flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc310; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)311; GFX8-NEXT:    buffer_wbinvl1_vol312; GFX8-NEXT:    v_mov_b32_e32 v2, s2313; GFX8-NEXT:    v_mov_b32_e32 v3, s3314; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]315; GFX8-NEXT:    s_endpgm316;317; GFX12-LABEL: atomic_add_i64_ret:318; GFX12:       ; %bb.0: ; %entry319; GFX12-NEXT:    s_clause 0x1320; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24321; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34322; GFX12-NEXT:    s_wait_kmcnt 0x0323; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1324; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5325; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV326; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0327; GFX12-NEXT:    global_inv scope:SCOPE_DEV328; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3329; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]330; GFX12-NEXT:    s_endpgm331entry:332  %tmp0 = atomicrmw volatile add ptr %out, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !0333  store i64 %tmp0, ptr %out2334  ret void335}336 337define amdgpu_kernel void @atomic_add_i64_addr64(ptr %out, i64 %in, i64 %index) {338; GFX7-LABEL: atomic_add_i64_addr64:339; GFX7:       ; %bb.0: ; %entry340; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9341; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd342; GFX7-NEXT:    s_waitcnt lgkmcnt(0)343; GFX7-NEXT:    v_mov_b32_e32 v0, s2344; GFX7-NEXT:    v_mov_b32_e32 v1, s3345; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3346; GFX7-NEXT:    s_add_u32 s0, s0, s2347; GFX7-NEXT:    s_addc_u32 s1, s1, s3348; GFX7-NEXT:    v_mov_b32_e32 v3, s1349; GFX7-NEXT:    v_mov_b32_e32 v2, s0350; GFX7-NEXT:    flat_atomic_add_x2 v[2:3], v[0:1]351; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)352; GFX7-NEXT:    buffer_wbinvl1_vol353; GFX7-NEXT:    s_endpgm354;355; GFX8-LABEL: atomic_add_i64_addr64:356; GFX8:       ; %bb.0: ; %entry357; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24358; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34359; GFX8-NEXT:    s_waitcnt lgkmcnt(0)360; GFX8-NEXT:    v_mov_b32_e32 v0, s2361; GFX8-NEXT:    v_mov_b32_e32 v1, s3362; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3363; GFX8-NEXT:    s_add_u32 s0, s0, s2364; GFX8-NEXT:    s_addc_u32 s1, s1, s3365; GFX8-NEXT:    v_mov_b32_e32 v3, s1366; GFX8-NEXT:    v_mov_b32_e32 v2, s0367; GFX8-NEXT:    flat_atomic_add_x2 v[2:3], v[0:1]368; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)369; GFX8-NEXT:    buffer_wbinvl1_vol370; GFX8-NEXT:    s_endpgm371;372; GFX12-LABEL: atomic_add_i64_addr64:373; GFX12:       ; %bb.0: ; %entry374; GFX12-NEXT:    s_clause 0x1375; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34376; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24377; GFX12-NEXT:    s_wait_kmcnt 0x0378; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3379; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3380; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]381; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)382; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0383; GFX12-NEXT:    flat_atomic_add_u64 v[2:3], v[0:1] scope:SCOPE_DEV384; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0385; GFX12-NEXT:    global_inv scope:SCOPE_DEV386; GFX12-NEXT:    s_endpgm387entry:388  %ptr = getelementptr inbounds i64, ptr %out, i64 %index389  %tmp0 = atomicrmw volatile add ptr %ptr, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !0390  ret void391}392 393define amdgpu_kernel void @atomic_add_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {394; GFX7-LABEL: atomic_add_i64_ret_addr64:395; GFX7:       ; %bb.0: ; %entry396; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9397; GFX7-NEXT:    s_waitcnt lgkmcnt(0)398; GFX7-NEXT:    v_mov_b32_e32 v0, s4399; GFX7-NEXT:    v_mov_b32_e32 v1, s5400; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3401; GFX7-NEXT:    s_add_u32 s0, s0, s4402; GFX7-NEXT:    s_addc_u32 s1, s1, s5403; GFX7-NEXT:    v_mov_b32_e32 v3, s1404; GFX7-NEXT:    v_mov_b32_e32 v2, s0405; GFX7-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3], v[0:1] glc406; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)407; GFX7-NEXT:    buffer_wbinvl1_vol408; GFX7-NEXT:    v_mov_b32_e32 v2, s2409; GFX7-NEXT:    v_mov_b32_e32 v3, s3410; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]411; GFX7-NEXT:    s_endpgm412;413; GFX8-LABEL: atomic_add_i64_ret_addr64:414; GFX8:       ; %bb.0: ; %entry415; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24416; GFX8-NEXT:    s_waitcnt lgkmcnt(0)417; GFX8-NEXT:    v_mov_b32_e32 v0, s4418; GFX8-NEXT:    v_mov_b32_e32 v1, s5419; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3420; GFX8-NEXT:    s_add_u32 s0, s0, s4421; GFX8-NEXT:    s_addc_u32 s1, s1, s5422; GFX8-NEXT:    v_mov_b32_e32 v3, s1423; GFX8-NEXT:    v_mov_b32_e32 v2, s0424; GFX8-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3], v[0:1] glc425; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)426; GFX8-NEXT:    buffer_wbinvl1_vol427; GFX8-NEXT:    v_mov_b32_e32 v2, s2428; GFX8-NEXT:    v_mov_b32_e32 v3, s3429; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]430; GFX8-NEXT:    s_endpgm431;432; GFX12-LABEL: atomic_add_i64_ret_addr64:433; GFX12:       ; %bb.0: ; %entry434; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24435; GFX12-NEXT:    s_wait_kmcnt 0x0436; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3437; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5438; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]439; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)440; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0441; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[2:3], v[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV442; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0443; GFX12-NEXT:    global_inv scope:SCOPE_DEV444; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3445; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]446; GFX12-NEXT:    s_endpgm447entry:448  %ptr = getelementptr inbounds i64, ptr %out, i64 %index449  %tmp0 = atomicrmw volatile add ptr %ptr, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !0450  store i64 %tmp0, ptr %out2451  ret void452}453 454define amdgpu_kernel void @atomic_and_i64_offset(ptr %out, i64 %in) {455; GFX7-LABEL: atomic_and_i64_offset:456; GFX7:       ; %bb.0: ; %entry457; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9458; GFX7-NEXT:    s_waitcnt lgkmcnt(0)459; GFX7-NEXT:    s_add_u32 s0, s0, 32460; GFX7-NEXT:    s_addc_u32 s1, s1, 0461; GFX7-NEXT:    v_mov_b32_e32 v5, s1462; GFX7-NEXT:    v_mov_b32_e32 v4, s0463; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]464; GFX7-NEXT:    s_mov_b64 s[0:1], 0465; GFX7-NEXT:  .LBB8_1: ; %atomicrmw.start466; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1467; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)468; GFX7-NEXT:    v_and_b32_e32 v1, s3, v3469; GFX7-NEXT:    v_and_b32_e32 v0, s2, v2470; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc471; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)472; GFX7-NEXT:    buffer_wbinvl1_vol473; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]474; GFX7-NEXT:    v_mov_b32_e32 v3, v1475; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]476; GFX7-NEXT:    v_mov_b32_e32 v2, v0477; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]478; GFX7-NEXT:    s_cbranch_execnz .LBB8_1479; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end480; GFX7-NEXT:    s_endpgm481;482; GFX8-LABEL: atomic_and_i64_offset:483; GFX8:       ; %bb.0: ; %entry484; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24485; GFX8-NEXT:    s_waitcnt lgkmcnt(0)486; GFX8-NEXT:    s_add_u32 s0, s0, 32487; GFX8-NEXT:    s_addc_u32 s1, s1, 0488; GFX8-NEXT:    v_mov_b32_e32 v5, s1489; GFX8-NEXT:    v_mov_b32_e32 v4, s0490; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]491; GFX8-NEXT:    s_mov_b64 s[0:1], 0492; GFX8-NEXT:  .LBB8_1: ; %atomicrmw.start493; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1494; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)495; GFX8-NEXT:    v_and_b32_e32 v1, s3, v3496; GFX8-NEXT:    v_and_b32_e32 v0, s2, v2497; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc498; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)499; GFX8-NEXT:    buffer_wbinvl1_vol500; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]501; GFX8-NEXT:    v_mov_b32_e32 v3, v1502; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]503; GFX8-NEXT:    v_mov_b32_e32 v2, v0504; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]505; GFX8-NEXT:    s_cbranch_execnz .LBB8_1506; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end507; GFX8-NEXT:    s_endpgm508;509; GFX12-LABEL: atomic_and_i64_offset:510; GFX12:       ; %bb.0: ; %entry511; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24512; GFX12-NEXT:    s_wait_kmcnt 0x0513; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1514; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3515; GFX12-NEXT:    flat_atomic_and_b64 v[0:1], v[2:3] offset:32 scope:SCOPE_DEV516; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0517; GFX12-NEXT:    global_inv scope:SCOPE_DEV518; GFX12-NEXT:    s_endpgm519entry:520  %gep = getelementptr inbounds i64, ptr %out, i64 4521  %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !0522  ret void523}524 525define amdgpu_kernel void @atomic_and_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {526; GFX7-LABEL: atomic_and_i64_ret_offset:527; GFX7:       ; %bb.0: ; %entry528; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9529; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd530; GFX7-NEXT:    s_waitcnt lgkmcnt(0)531; GFX7-NEXT:    s_add_u32 s0, s0, 32532; GFX7-NEXT:    s_addc_u32 s1, s1, 0533; GFX7-NEXT:    v_mov_b32_e32 v0, s0534; GFX7-NEXT:    v_mov_b32_e32 v1, s1535; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]536; GFX7-NEXT:    s_mov_b64 s[0:1], 0537; GFX7-NEXT:  .LBB9_1: ; %atomicrmw.start538; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1539; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)540; GFX7-NEXT:    v_mov_b32_e32 v5, v3541; GFX7-NEXT:    v_mov_b32_e32 v4, v2542; GFX7-NEXT:    v_and_b32_e32 v3, s5, v5543; GFX7-NEXT:    v_and_b32_e32 v2, s4, v4544; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc545; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)546; GFX7-NEXT:    buffer_wbinvl1_vol547; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]548; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]549; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]550; GFX7-NEXT:    s_cbranch_execnz .LBB9_1551; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end552; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]553; GFX7-NEXT:    v_mov_b32_e32 v0, s2554; GFX7-NEXT:    v_mov_b32_e32 v1, s3555; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]556; GFX7-NEXT:    s_endpgm557;558; GFX8-LABEL: atomic_and_i64_ret_offset:559; GFX8:       ; %bb.0: ; %entry560; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24561; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34562; GFX8-NEXT:    s_waitcnt lgkmcnt(0)563; GFX8-NEXT:    s_add_u32 s0, s0, 32564; GFX8-NEXT:    s_addc_u32 s1, s1, 0565; GFX8-NEXT:    v_mov_b32_e32 v0, s0566; GFX8-NEXT:    v_mov_b32_e32 v1, s1567; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]568; GFX8-NEXT:    s_mov_b64 s[0:1], 0569; GFX8-NEXT:  .LBB9_1: ; %atomicrmw.start570; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1571; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)572; GFX8-NEXT:    v_mov_b32_e32 v5, v3573; GFX8-NEXT:    v_mov_b32_e32 v4, v2574; GFX8-NEXT:    v_and_b32_e32 v3, s5, v5575; GFX8-NEXT:    v_and_b32_e32 v2, s4, v4576; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc577; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)578; GFX8-NEXT:    buffer_wbinvl1_vol579; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]580; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]581; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]582; GFX8-NEXT:    s_cbranch_execnz .LBB9_1583; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end584; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]585; GFX8-NEXT:    v_mov_b32_e32 v0, s2586; GFX8-NEXT:    v_mov_b32_e32 v1, s3587; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]588; GFX8-NEXT:    s_endpgm589;590; GFX12-LABEL: atomic_and_i64_ret_offset:591; GFX12:       ; %bb.0: ; %entry592; GFX12-NEXT:    s_clause 0x1593; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24594; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34595; GFX12-NEXT:    s_wait_kmcnt 0x0596; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1597; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5598; GFX12-NEXT:    flat_atomic_and_b64 v[0:1], v[0:1], v[2:3] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV599; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0600; GFX12-NEXT:    global_inv scope:SCOPE_DEV601; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3602; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]603; GFX12-NEXT:    s_endpgm604entry:605  %gep = getelementptr inbounds i64, ptr %out, i64 4606  %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !0607  store i64 %tmp0, ptr %out2608  ret void609}610 611define amdgpu_kernel void @atomic_and_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {612; GFX7-LABEL: atomic_and_i64_addr64_offset:613; GFX7:       ; %bb.0: ; %entry614; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd615; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9616; GFX7-NEXT:    s_waitcnt lgkmcnt(0)617; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3618; GFX7-NEXT:    s_add_u32 s0, s0, s4619; GFX7-NEXT:    s_addc_u32 s1, s1, s5620; GFX7-NEXT:    s_add_u32 s0, s0, 32621; GFX7-NEXT:    s_addc_u32 s1, s1, 0622; GFX7-NEXT:    v_mov_b32_e32 v5, s1623; GFX7-NEXT:    v_mov_b32_e32 v4, s0624; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]625; GFX7-NEXT:    s_mov_b64 s[0:1], 0626; GFX7-NEXT:  .LBB10_1: ; %atomicrmw.start627; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1628; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)629; GFX7-NEXT:    v_and_b32_e32 v1, s3, v3630; GFX7-NEXT:    v_and_b32_e32 v0, s2, v2631; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc632; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)633; GFX7-NEXT:    buffer_wbinvl1_vol634; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]635; GFX7-NEXT:    v_mov_b32_e32 v3, v1636; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]637; GFX7-NEXT:    v_mov_b32_e32 v2, v0638; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]639; GFX7-NEXT:    s_cbranch_execnz .LBB10_1640; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end641; GFX7-NEXT:    s_endpgm642;643; GFX8-LABEL: atomic_and_i64_addr64_offset:644; GFX8:       ; %bb.0: ; %entry645; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34646; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24647; GFX8-NEXT:    s_waitcnt lgkmcnt(0)648; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3649; GFX8-NEXT:    s_add_u32 s0, s0, s4650; GFX8-NEXT:    s_addc_u32 s1, s1, s5651; GFX8-NEXT:    s_add_u32 s0, s0, 32652; GFX8-NEXT:    s_addc_u32 s1, s1, 0653; GFX8-NEXT:    v_mov_b32_e32 v5, s1654; GFX8-NEXT:    v_mov_b32_e32 v4, s0655; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]656; GFX8-NEXT:    s_mov_b64 s[0:1], 0657; GFX8-NEXT:  .LBB10_1: ; %atomicrmw.start658; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1659; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)660; GFX8-NEXT:    v_and_b32_e32 v1, s3, v3661; GFX8-NEXT:    v_and_b32_e32 v0, s2, v2662; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc663; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)664; GFX8-NEXT:    buffer_wbinvl1_vol665; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]666; GFX8-NEXT:    v_mov_b32_e32 v3, v1667; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]668; GFX8-NEXT:    v_mov_b32_e32 v2, v0669; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]670; GFX8-NEXT:    s_cbranch_execnz .LBB10_1671; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end672; GFX8-NEXT:    s_endpgm673;674; GFX12-LABEL: atomic_and_i64_addr64_offset:675; GFX12:       ; %bb.0: ; %entry676; GFX12-NEXT:    s_clause 0x1677; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34678; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24679; GFX12-NEXT:    s_wait_kmcnt 0x0680; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3681; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3682; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]683; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)684; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0685; GFX12-NEXT:    flat_atomic_and_b64 v[2:3], v[0:1] offset:32 scope:SCOPE_DEV686; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0687; GFX12-NEXT:    global_inv scope:SCOPE_DEV688; GFX12-NEXT:    s_endpgm689entry:690  %ptr = getelementptr inbounds i64, ptr %out, i64 %index691  %gep = getelementptr inbounds i64, ptr %ptr, i64 4692  %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !0693  ret void694}695 696define amdgpu_kernel void @atomic_and_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {697; GFX7-LABEL: atomic_and_i64_ret_addr64_offset:698; GFX7:       ; %bb.0: ; %entry699; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9700; GFX7-NEXT:    s_waitcnt lgkmcnt(0)701; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3702; GFX7-NEXT:    s_add_u32 s0, s0, s6703; GFX7-NEXT:    s_addc_u32 s1, s1, s7704; GFX7-NEXT:    s_add_u32 s0, s0, 32705; GFX7-NEXT:    s_addc_u32 s1, s1, 0706; GFX7-NEXT:    v_mov_b32_e32 v0, s0707; GFX7-NEXT:    v_mov_b32_e32 v1, s1708; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]709; GFX7-NEXT:    s_mov_b64 s[0:1], 0710; GFX7-NEXT:  .LBB11_1: ; %atomicrmw.start711; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1712; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)713; GFX7-NEXT:    v_mov_b32_e32 v5, v3714; GFX7-NEXT:    v_mov_b32_e32 v4, v2715; GFX7-NEXT:    v_and_b32_e32 v3, s5, v5716; GFX7-NEXT:    v_and_b32_e32 v2, s4, v4717; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc718; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)719; GFX7-NEXT:    buffer_wbinvl1_vol720; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]721; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]722; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]723; GFX7-NEXT:    s_cbranch_execnz .LBB11_1724; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end725; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]726; GFX7-NEXT:    v_mov_b32_e32 v0, s2727; GFX7-NEXT:    v_mov_b32_e32 v1, s3728; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]729; GFX7-NEXT:    s_endpgm730;731; GFX8-LABEL: atomic_and_i64_ret_addr64_offset:732; GFX8:       ; %bb.0: ; %entry733; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24734; GFX8-NEXT:    s_waitcnt lgkmcnt(0)735; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3736; GFX8-NEXT:    s_add_u32 s0, s0, s6737; GFX8-NEXT:    s_addc_u32 s1, s1, s7738; GFX8-NEXT:    s_add_u32 s0, s0, 32739; GFX8-NEXT:    s_addc_u32 s1, s1, 0740; GFX8-NEXT:    v_mov_b32_e32 v0, s0741; GFX8-NEXT:    v_mov_b32_e32 v1, s1742; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]743; GFX8-NEXT:    s_mov_b64 s[0:1], 0744; GFX8-NEXT:  .LBB11_1: ; %atomicrmw.start745; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1746; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)747; GFX8-NEXT:    v_mov_b32_e32 v5, v3748; GFX8-NEXT:    v_mov_b32_e32 v4, v2749; GFX8-NEXT:    v_and_b32_e32 v3, s5, v5750; GFX8-NEXT:    v_and_b32_e32 v2, s4, v4751; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc752; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)753; GFX8-NEXT:    buffer_wbinvl1_vol754; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]755; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]756; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]757; GFX8-NEXT:    s_cbranch_execnz .LBB11_1758; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end759; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]760; GFX8-NEXT:    v_mov_b32_e32 v0, s2761; GFX8-NEXT:    v_mov_b32_e32 v1, s3762; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]763; GFX8-NEXT:    s_endpgm764;765; GFX12-LABEL: atomic_and_i64_ret_addr64_offset:766; GFX12:       ; %bb.0: ; %entry767; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24768; GFX12-NEXT:    s_wait_kmcnt 0x0769; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3770; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5771; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]772; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)773; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0774; GFX12-NEXT:    flat_atomic_and_b64 v[0:1], v[2:3], v[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV775; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0776; GFX12-NEXT:    global_inv scope:SCOPE_DEV777; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3778; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]779; GFX12-NEXT:    s_endpgm780entry:781  %ptr = getelementptr inbounds i64, ptr %out, i64 %index782  %gep = getelementptr inbounds i64, ptr %ptr, i64 4783  %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !0784  store i64 %tmp0, ptr %out2785  ret void786}787 788define amdgpu_kernel void @atomic_and_i64(ptr %out, i64 %in) {789; GFX7-LABEL: atomic_and_i64:790; GFX7:       ; %bb.0: ; %entry791; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9792; GFX7-NEXT:    s_mov_b64 s[4:5], 0793; GFX7-NEXT:    s_waitcnt lgkmcnt(0)794; GFX7-NEXT:    v_mov_b32_e32 v0, s0795; GFX7-NEXT:    v_mov_b32_e32 v1, s1796; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]797; GFX7-NEXT:    v_mov_b32_e32 v5, s1798; GFX7-NEXT:    v_mov_b32_e32 v4, s0799; GFX7-NEXT:  .LBB12_1: ; %atomicrmw.start800; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1801; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)802; GFX7-NEXT:    v_and_b32_e32 v1, s3, v3803; GFX7-NEXT:    v_and_b32_e32 v0, s2, v2804; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc805; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)806; GFX7-NEXT:    buffer_wbinvl1_vol807; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]808; GFX7-NEXT:    v_mov_b32_e32 v3, v1809; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]810; GFX7-NEXT:    v_mov_b32_e32 v2, v0811; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]812; GFX7-NEXT:    s_cbranch_execnz .LBB12_1813; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end814; GFX7-NEXT:    s_endpgm815;816; GFX8-LABEL: atomic_and_i64:817; GFX8:       ; %bb.0: ; %entry818; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24819; GFX8-NEXT:    s_mov_b64 s[4:5], 0820; GFX8-NEXT:    s_waitcnt lgkmcnt(0)821; GFX8-NEXT:    v_mov_b32_e32 v0, s0822; GFX8-NEXT:    v_mov_b32_e32 v1, s1823; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]824; GFX8-NEXT:    v_mov_b32_e32 v5, s1825; GFX8-NEXT:    v_mov_b32_e32 v4, s0826; GFX8-NEXT:  .LBB12_1: ; %atomicrmw.start827; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1828; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)829; GFX8-NEXT:    v_and_b32_e32 v1, s3, v3830; GFX8-NEXT:    v_and_b32_e32 v0, s2, v2831; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc832; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)833; GFX8-NEXT:    buffer_wbinvl1_vol834; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]835; GFX8-NEXT:    v_mov_b32_e32 v3, v1836; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]837; GFX8-NEXT:    v_mov_b32_e32 v2, v0838; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]839; GFX8-NEXT:    s_cbranch_execnz .LBB12_1840; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end841; GFX8-NEXT:    s_endpgm842;843; GFX12-LABEL: atomic_and_i64:844; GFX12:       ; %bb.0: ; %entry845; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24846; GFX12-NEXT:    s_wait_kmcnt 0x0847; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1848; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3849; GFX12-NEXT:    flat_atomic_and_b64 v[0:1], v[2:3] scope:SCOPE_DEV850; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0851; GFX12-NEXT:    global_inv scope:SCOPE_DEV852; GFX12-NEXT:    s_endpgm853entry:854  %tmp0 = atomicrmw volatile and ptr %out, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !0855  ret void856}857 858define amdgpu_kernel void @atomic_and_i64_ret(ptr %out, ptr %out2, i64 %in) {859; GFX7-LABEL: atomic_and_i64_ret:860; GFX7:       ; %bb.0: ; %entry861; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9862; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd863; GFX7-NEXT:    s_mov_b64 s[6:7], 0864; GFX7-NEXT:    s_waitcnt lgkmcnt(0)865; GFX7-NEXT:    v_mov_b32_e32 v0, s0866; GFX7-NEXT:    v_mov_b32_e32 v1, s1867; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]868; GFX7-NEXT:    v_mov_b32_e32 v3, s1869; GFX7-NEXT:    v_mov_b32_e32 v2, s0870; GFX7-NEXT:  .LBB13_1: ; %atomicrmw.start871; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1872; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)873; GFX7-NEXT:    v_mov_b32_e32 v7, v1874; GFX7-NEXT:    v_mov_b32_e32 v6, v0875; GFX7-NEXT:    v_and_b32_e32 v5, s5, v7876; GFX7-NEXT:    v_and_b32_e32 v4, s4, v6877; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc878; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)879; GFX7-NEXT:    buffer_wbinvl1_vol880; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]881; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]882; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]883; GFX7-NEXT:    s_cbranch_execnz .LBB13_1884; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end885; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]886; GFX7-NEXT:    v_mov_b32_e32 v2, s2887; GFX7-NEXT:    v_mov_b32_e32 v3, s3888; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]889; GFX7-NEXT:    s_endpgm890;891; GFX8-LABEL: atomic_and_i64_ret:892; GFX8:       ; %bb.0: ; %entry893; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24894; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34895; GFX8-NEXT:    s_mov_b64 s[6:7], 0896; GFX8-NEXT:    s_waitcnt lgkmcnt(0)897; GFX8-NEXT:    v_mov_b32_e32 v0, s0898; GFX8-NEXT:    v_mov_b32_e32 v1, s1899; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]900; GFX8-NEXT:    v_mov_b32_e32 v3, s1901; GFX8-NEXT:    v_mov_b32_e32 v2, s0902; GFX8-NEXT:  .LBB13_1: ; %atomicrmw.start903; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1904; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)905; GFX8-NEXT:    v_mov_b32_e32 v7, v1906; GFX8-NEXT:    v_mov_b32_e32 v6, v0907; GFX8-NEXT:    v_and_b32_e32 v5, s5, v7908; GFX8-NEXT:    v_and_b32_e32 v4, s4, v6909; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc910; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)911; GFX8-NEXT:    buffer_wbinvl1_vol912; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]913; GFX8-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]914; GFX8-NEXT:    s_andn2_b64 exec, exec, s[6:7]915; GFX8-NEXT:    s_cbranch_execnz .LBB13_1916; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end917; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]918; GFX8-NEXT:    v_mov_b32_e32 v2, s2919; GFX8-NEXT:    v_mov_b32_e32 v3, s3920; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]921; GFX8-NEXT:    s_endpgm922;923; GFX12-LABEL: atomic_and_i64_ret:924; GFX12:       ; %bb.0: ; %entry925; GFX12-NEXT:    s_clause 0x1926; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24927; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34928; GFX12-NEXT:    s_wait_kmcnt 0x0929; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1930; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5931; GFX12-NEXT:    flat_atomic_and_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV932; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0933; GFX12-NEXT:    global_inv scope:SCOPE_DEV934; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3935; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]936; GFX12-NEXT:    s_endpgm937entry:938  %tmp0 = atomicrmw volatile and ptr %out, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !0939  store i64 %tmp0, ptr %out2940  ret void941}942 943define amdgpu_kernel void @atomic_and_i64_addr64(ptr %out, i64 %in, i64 %index) {944; GFX7-LABEL: atomic_and_i64_addr64:945; GFX7:       ; %bb.0: ; %entry946; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd947; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9948; GFX7-NEXT:    s_waitcnt lgkmcnt(0)949; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3950; GFX7-NEXT:    s_add_u32 s0, s0, s4951; GFX7-NEXT:    s_addc_u32 s1, s1, s5952; GFX7-NEXT:    v_mov_b32_e32 v5, s1953; GFX7-NEXT:    v_mov_b32_e32 v4, s0954; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]955; GFX7-NEXT:    s_mov_b64 s[0:1], 0956; GFX7-NEXT:  .LBB14_1: ; %atomicrmw.start957; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1958; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)959; GFX7-NEXT:    v_and_b32_e32 v1, s3, v3960; GFX7-NEXT:    v_and_b32_e32 v0, s2, v2961; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc962; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)963; GFX7-NEXT:    buffer_wbinvl1_vol964; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]965; GFX7-NEXT:    v_mov_b32_e32 v3, v1966; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]967; GFX7-NEXT:    v_mov_b32_e32 v2, v0968; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]969; GFX7-NEXT:    s_cbranch_execnz .LBB14_1970; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end971; GFX7-NEXT:    s_endpgm972;973; GFX8-LABEL: atomic_and_i64_addr64:974; GFX8:       ; %bb.0: ; %entry975; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34976; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24977; GFX8-NEXT:    s_waitcnt lgkmcnt(0)978; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3979; GFX8-NEXT:    s_add_u32 s0, s0, s4980; GFX8-NEXT:    s_addc_u32 s1, s1, s5981; GFX8-NEXT:    v_mov_b32_e32 v5, s1982; GFX8-NEXT:    v_mov_b32_e32 v4, s0983; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]984; GFX8-NEXT:    s_mov_b64 s[0:1], 0985; GFX8-NEXT:  .LBB14_1: ; %atomicrmw.start986; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1987; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)988; GFX8-NEXT:    v_and_b32_e32 v1, s3, v3989; GFX8-NEXT:    v_and_b32_e32 v0, s2, v2990; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc991; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)992; GFX8-NEXT:    buffer_wbinvl1_vol993; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]994; GFX8-NEXT:    v_mov_b32_e32 v3, v1995; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]996; GFX8-NEXT:    v_mov_b32_e32 v2, v0997; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]998; GFX8-NEXT:    s_cbranch_execnz .LBB14_1999; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end1000; GFX8-NEXT:    s_endpgm1001;1002; GFX12-LABEL: atomic_and_i64_addr64:1003; GFX12:       ; %bb.0: ; %entry1004; GFX12-NEXT:    s_clause 0x11005; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x341006; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241007; GFX12-NEXT:    s_wait_kmcnt 0x01008; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 31009; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s31010; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]1011; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)1012; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s01013; GFX12-NEXT:    flat_atomic_and_b64 v[2:3], v[0:1] scope:SCOPE_DEV1014; GFX12-NEXT:    s_wait_storecnt_dscnt 0x01015; GFX12-NEXT:    global_inv scope:SCOPE_DEV1016; GFX12-NEXT:    s_endpgm1017entry:1018  %ptr = getelementptr inbounds i64, ptr %out, i64 %index1019  %tmp0 = atomicrmw volatile and ptr %ptr, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !01020  ret void1021}1022 1023define amdgpu_kernel void @atomic_and_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {1024; GFX7-LABEL: atomic_and_i64_ret_addr64:1025; GFX7:       ; %bb.0: ; %entry1026; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x91027; GFX7-NEXT:    s_waitcnt lgkmcnt(0)1028; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 31029; GFX7-NEXT:    s_add_u32 s0, s0, s61030; GFX7-NEXT:    s_addc_u32 s1, s1, s71031; GFX7-NEXT:    v_mov_b32_e32 v0, s01032; GFX7-NEXT:    v_mov_b32_e32 v1, s11033; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]1034; GFX7-NEXT:    s_mov_b64 s[0:1], 01035; GFX7-NEXT:  .LBB15_1: ; %atomicrmw.start1036; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=11037; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1038; GFX7-NEXT:    v_mov_b32_e32 v5, v31039; GFX7-NEXT:    v_mov_b32_e32 v4, v21040; GFX7-NEXT:    v_and_b32_e32 v3, s5, v51041; GFX7-NEXT:    v_and_b32_e32 v2, s4, v41042; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc1043; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1044; GFX7-NEXT:    buffer_wbinvl1_vol1045; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]1046; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]1047; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]1048; GFX7-NEXT:    s_cbranch_execnz .LBB15_11049; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end1050; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]1051; GFX7-NEXT:    v_mov_b32_e32 v0, s21052; GFX7-NEXT:    v_mov_b32_e32 v1, s31053; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]1054; GFX7-NEXT:    s_endpgm1055;1056; GFX8-LABEL: atomic_and_i64_ret_addr64:1057; GFX8:       ; %bb.0: ; %entry1058; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x241059; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1060; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 31061; GFX8-NEXT:    s_add_u32 s0, s0, s61062; GFX8-NEXT:    s_addc_u32 s1, s1, s71063; GFX8-NEXT:    v_mov_b32_e32 v0, s01064; GFX8-NEXT:    v_mov_b32_e32 v1, s11065; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]1066; GFX8-NEXT:    s_mov_b64 s[0:1], 01067; GFX8-NEXT:  .LBB15_1: ; %atomicrmw.start1068; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=11069; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1070; GFX8-NEXT:    v_mov_b32_e32 v5, v31071; GFX8-NEXT:    v_mov_b32_e32 v4, v21072; GFX8-NEXT:    v_and_b32_e32 v3, s5, v51073; GFX8-NEXT:    v_and_b32_e32 v2, s4, v41074; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc1075; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1076; GFX8-NEXT:    buffer_wbinvl1_vol1077; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]1078; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]1079; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]1080; GFX8-NEXT:    s_cbranch_execnz .LBB15_11081; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end1082; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]1083; GFX8-NEXT:    v_mov_b32_e32 v0, s21084; GFX8-NEXT:    v_mov_b32_e32 v1, s31085; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]1086; GFX8-NEXT:    s_endpgm1087;1088; GFX12-LABEL: atomic_and_i64_ret_addr64:1089; GFX12:       ; %bb.0: ; %entry1090; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x241091; GFX12-NEXT:    s_wait_kmcnt 0x01092; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 31093; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s51094; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]1095; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)1096; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s01097; GFX12-NEXT:    flat_atomic_and_b64 v[0:1], v[2:3], v[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV1098; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x01099; GFX12-NEXT:    global_inv scope:SCOPE_DEV1100; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s31101; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]1102; GFX12-NEXT:    s_endpgm1103entry:1104  %ptr = getelementptr inbounds i64, ptr %out, i64 %index1105  %tmp0 = atomicrmw volatile and ptr %ptr, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !01106  store i64 %tmp0, ptr %out21107  ret void1108}1109 1110define amdgpu_kernel void @atomic_sub_i64_offset(ptr %out, i64 %in) {1111; GFX7-LABEL: atomic_sub_i64_offset:1112; GFX7:       ; %bb.0: ; %entry1113; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x91114; GFX7-NEXT:    s_waitcnt lgkmcnt(0)1115; GFX7-NEXT:    s_add_u32 s0, s0, 321116; GFX7-NEXT:    s_addc_u32 s1, s1, 01117; GFX7-NEXT:    v_mov_b32_e32 v5, s11118; GFX7-NEXT:    v_mov_b32_e32 v4, s01119; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]1120; GFX7-NEXT:    s_mov_b64 s[0:1], 01121; GFX7-NEXT:    v_mov_b32_e32 v6, s31122; GFX7-NEXT:  .LBB16_1: ; %atomicrmw.start1123; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=11124; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1125; GFX7-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v21126; GFX7-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc1127; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc1128; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1129; GFX7-NEXT:    buffer_wbinvl1_vol1130; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]1131; GFX7-NEXT:    v_mov_b32_e32 v3, v11132; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]1133; GFX7-NEXT:    v_mov_b32_e32 v2, v01134; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]1135; GFX7-NEXT:    s_cbranch_execnz .LBB16_11136; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end1137; GFX7-NEXT:    s_endpgm1138;1139; GFX8-LABEL: atomic_sub_i64_offset:1140; GFX8:       ; %bb.0: ; %entry1141; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241142; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1143; GFX8-NEXT:    s_add_u32 s0, s0, 321144; GFX8-NEXT:    s_addc_u32 s1, s1, 01145; GFX8-NEXT:    v_mov_b32_e32 v5, s11146; GFX8-NEXT:    v_mov_b32_e32 v4, s01147; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]1148; GFX8-NEXT:    s_mov_b64 s[0:1], 01149; GFX8-NEXT:    v_mov_b32_e32 v6, s31150; GFX8-NEXT:  .LBB16_1: ; %atomicrmw.start1151; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=11152; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1153; GFX8-NEXT:    v_subrev_u32_e32 v0, vcc, s2, v21154; GFX8-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc1155; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc1156; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1157; GFX8-NEXT:    buffer_wbinvl1_vol1158; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]1159; GFX8-NEXT:    v_mov_b32_e32 v3, v11160; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]1161; GFX8-NEXT:    v_mov_b32_e32 v2, v01162; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]1163; GFX8-NEXT:    s_cbranch_execnz .LBB16_11164; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end1165; GFX8-NEXT:    s_endpgm1166;1167; GFX12-LABEL: atomic_sub_i64_offset:1168; GFX12:       ; %bb.0: ; %entry1169; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241170; GFX12-NEXT:    s_wait_kmcnt 0x01171; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11172; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s31173; GFX12-NEXT:    flat_atomic_sub_u64 v[0:1], v[2:3] offset:32 scope:SCOPE_DEV1174; GFX12-NEXT:    s_wait_storecnt_dscnt 0x01175; GFX12-NEXT:    global_inv scope:SCOPE_DEV1176; GFX12-NEXT:    s_endpgm1177entry:1178  %gep = getelementptr inbounds i64, ptr %out, i64 41179  %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !01180  ret void1181}1182 1183define amdgpu_kernel void @atomic_sub_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {1184; GFX7-LABEL: atomic_sub_i64_ret_offset:1185; GFX7:       ; %bb.0: ; %entry1186; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x91187; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd1188; GFX7-NEXT:    s_waitcnt lgkmcnt(0)1189; GFX7-NEXT:    s_add_u32 s0, s0, 321190; GFX7-NEXT:    s_addc_u32 s1, s1, 01191; GFX7-NEXT:    v_mov_b32_e32 v0, s01192; GFX7-NEXT:    v_mov_b32_e32 v1, s11193; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]1194; GFX7-NEXT:    s_mov_b64 s[0:1], 01195; GFX7-NEXT:    v_mov_b32_e32 v4, s51196; GFX7-NEXT:  .LBB17_1: ; %atomicrmw.start1197; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=11198; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1199; GFX7-NEXT:    v_mov_b32_e32 v8, v31200; GFX7-NEXT:    v_mov_b32_e32 v7, v21201; GFX7-NEXT:    v_subrev_i32_e32 v5, vcc, s4, v71202; GFX7-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc1203; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[5:8] glc1204; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1205; GFX7-NEXT:    buffer_wbinvl1_vol1206; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[7:8]1207; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]1208; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]1209; GFX7-NEXT:    s_cbranch_execnz .LBB17_11210; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end1211; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]1212; GFX7-NEXT:    v_mov_b32_e32 v0, s21213; GFX7-NEXT:    v_mov_b32_e32 v1, s31214; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]1215; GFX7-NEXT:    s_endpgm1216;1217; GFX8-LABEL: atomic_sub_i64_ret_offset:1218; GFX8:       ; %bb.0: ; %entry1219; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241220; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x341221; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1222; GFX8-NEXT:    s_add_u32 s0, s0, 321223; GFX8-NEXT:    s_addc_u32 s1, s1, 01224; GFX8-NEXT:    v_mov_b32_e32 v0, s01225; GFX8-NEXT:    v_mov_b32_e32 v1, s11226; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]1227; GFX8-NEXT:    s_mov_b64 s[0:1], 01228; GFX8-NEXT:    v_mov_b32_e32 v4, s51229; GFX8-NEXT:  .LBB17_1: ; %atomicrmw.start1230; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=11231; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1232; GFX8-NEXT:    v_mov_b32_e32 v8, v31233; GFX8-NEXT:    v_mov_b32_e32 v7, v21234; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, s4, v71235; GFX8-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc1236; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[5:8] glc1237; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1238; GFX8-NEXT:    buffer_wbinvl1_vol1239; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[7:8]1240; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]1241; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]1242; GFX8-NEXT:    s_cbranch_execnz .LBB17_11243; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end1244; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]1245; GFX8-NEXT:    v_mov_b32_e32 v0, s21246; GFX8-NEXT:    v_mov_b32_e32 v1, s31247; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]1248; GFX8-NEXT:    s_endpgm1249;1250; GFX12-LABEL: atomic_sub_i64_ret_offset:1251; GFX12:       ; %bb.0: ; %entry1252; GFX12-NEXT:    s_clause 0x11253; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241254; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x341255; GFX12-NEXT:    s_wait_kmcnt 0x01256; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11257; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s51258; GFX12-NEXT:    flat_atomic_sub_u64 v[0:1], v[0:1], v[2:3] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV1259; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x01260; GFX12-NEXT:    global_inv scope:SCOPE_DEV1261; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s31262; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]1263; GFX12-NEXT:    s_endpgm1264entry:1265  %gep = getelementptr inbounds i64, ptr %out, i64 41266  %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !01267  store i64 %tmp0, ptr %out21268  ret void1269}1270 1271define amdgpu_kernel void @atomic_sub_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {1272; GFX7-LABEL: atomic_sub_i64_addr64_offset:1273; GFX7:       ; %bb.0: ; %entry1274; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd1275; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x91276; GFX7-NEXT:    s_waitcnt lgkmcnt(0)1277; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 31278; GFX7-NEXT:    s_add_u32 s0, s0, s41279; GFX7-NEXT:    s_addc_u32 s1, s1, s51280; GFX7-NEXT:    s_add_u32 s0, s0, 321281; GFX7-NEXT:    s_addc_u32 s1, s1, 01282; GFX7-NEXT:    v_mov_b32_e32 v5, s11283; GFX7-NEXT:    v_mov_b32_e32 v4, s01284; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]1285; GFX7-NEXT:    s_mov_b64 s[0:1], 01286; GFX7-NEXT:    v_mov_b32_e32 v6, s31287; GFX7-NEXT:  .LBB18_1: ; %atomicrmw.start1288; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=11289; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1290; GFX7-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v21291; GFX7-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc1292; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc1293; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1294; GFX7-NEXT:    buffer_wbinvl1_vol1295; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]1296; GFX7-NEXT:    v_mov_b32_e32 v3, v11297; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]1298; GFX7-NEXT:    v_mov_b32_e32 v2, v01299; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]1300; GFX7-NEXT:    s_cbranch_execnz .LBB18_11301; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end1302; GFX7-NEXT:    s_endpgm1303;1304; GFX8-LABEL: atomic_sub_i64_addr64_offset:1305; GFX8:       ; %bb.0: ; %entry1306; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x341307; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241308; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1309; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 31310; GFX8-NEXT:    s_add_u32 s0, s0, s41311; GFX8-NEXT:    s_addc_u32 s1, s1, s51312; GFX8-NEXT:    s_add_u32 s0, s0, 321313; GFX8-NEXT:    s_addc_u32 s1, s1, 01314; GFX8-NEXT:    v_mov_b32_e32 v5, s11315; GFX8-NEXT:    v_mov_b32_e32 v4, s01316; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]1317; GFX8-NEXT:    s_mov_b64 s[0:1], 01318; GFX8-NEXT:    v_mov_b32_e32 v6, s31319; GFX8-NEXT:  .LBB18_1: ; %atomicrmw.start1320; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=11321; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1322; GFX8-NEXT:    v_subrev_u32_e32 v0, vcc, s2, v21323; GFX8-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc1324; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc1325; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1326; GFX8-NEXT:    buffer_wbinvl1_vol1327; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]1328; GFX8-NEXT:    v_mov_b32_e32 v3, v11329; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]1330; GFX8-NEXT:    v_mov_b32_e32 v2, v01331; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]1332; GFX8-NEXT:    s_cbranch_execnz .LBB18_11333; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end1334; GFX8-NEXT:    s_endpgm1335;1336; GFX12-LABEL: atomic_sub_i64_addr64_offset:1337; GFX12:       ; %bb.0: ; %entry1338; GFX12-NEXT:    s_clause 0x11339; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x341340; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241341; GFX12-NEXT:    s_wait_kmcnt 0x01342; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 31343; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s31344; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]1345; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)1346; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s01347; GFX12-NEXT:    flat_atomic_sub_u64 v[2:3], v[0:1] offset:32 scope:SCOPE_DEV1348; GFX12-NEXT:    s_wait_storecnt_dscnt 0x01349; GFX12-NEXT:    global_inv scope:SCOPE_DEV1350; GFX12-NEXT:    s_endpgm1351entry:1352  %ptr = getelementptr inbounds i64, ptr %out, i64 %index1353  %gep = getelementptr inbounds i64, ptr %ptr, i64 41354  %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !01355  ret void1356}1357 1358define amdgpu_kernel void @atomic_sub_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {1359; GFX7-LABEL: atomic_sub_i64_ret_addr64_offset:1360; GFX7:       ; %bb.0: ; %entry1361; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x91362; GFX7-NEXT:    s_waitcnt lgkmcnt(0)1363; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 31364; GFX7-NEXT:    s_add_u32 s0, s0, s61365; GFX7-NEXT:    s_addc_u32 s1, s1, s71366; GFX7-NEXT:    s_add_u32 s0, s0, 321367; GFX7-NEXT:    s_addc_u32 s1, s1, 01368; GFX7-NEXT:    v_mov_b32_e32 v0, s01369; GFX7-NEXT:    v_mov_b32_e32 v1, s11370; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]1371; GFX7-NEXT:    s_mov_b64 s[0:1], 01372; GFX7-NEXT:    v_mov_b32_e32 v4, s51373; GFX7-NEXT:  .LBB19_1: ; %atomicrmw.start1374; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=11375; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1376; GFX7-NEXT:    v_mov_b32_e32 v8, v31377; GFX7-NEXT:    v_mov_b32_e32 v7, v21378; GFX7-NEXT:    v_subrev_i32_e32 v5, vcc, s4, v71379; GFX7-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc1380; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[5:8] glc1381; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1382; GFX7-NEXT:    buffer_wbinvl1_vol1383; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[7:8]1384; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]1385; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]1386; GFX7-NEXT:    s_cbranch_execnz .LBB19_11387; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end1388; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]1389; GFX7-NEXT:    v_mov_b32_e32 v0, s21390; GFX7-NEXT:    v_mov_b32_e32 v1, s31391; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]1392; GFX7-NEXT:    s_endpgm1393;1394; GFX8-LABEL: atomic_sub_i64_ret_addr64_offset:1395; GFX8:       ; %bb.0: ; %entry1396; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x241397; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1398; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 31399; GFX8-NEXT:    s_add_u32 s0, s0, s61400; GFX8-NEXT:    s_addc_u32 s1, s1, s71401; GFX8-NEXT:    s_add_u32 s0, s0, 321402; GFX8-NEXT:    s_addc_u32 s1, s1, 01403; GFX8-NEXT:    v_mov_b32_e32 v0, s01404; GFX8-NEXT:    v_mov_b32_e32 v1, s11405; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]1406; GFX8-NEXT:    s_mov_b64 s[0:1], 01407; GFX8-NEXT:    v_mov_b32_e32 v4, s51408; GFX8-NEXT:  .LBB19_1: ; %atomicrmw.start1409; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=11410; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1411; GFX8-NEXT:    v_mov_b32_e32 v8, v31412; GFX8-NEXT:    v_mov_b32_e32 v7, v21413; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, s4, v71414; GFX8-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc1415; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[5:8] glc1416; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1417; GFX8-NEXT:    buffer_wbinvl1_vol1418; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[7:8]1419; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]1420; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]1421; GFX8-NEXT:    s_cbranch_execnz .LBB19_11422; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end1423; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]1424; GFX8-NEXT:    v_mov_b32_e32 v0, s21425; GFX8-NEXT:    v_mov_b32_e32 v1, s31426; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]1427; GFX8-NEXT:    s_endpgm1428;1429; GFX12-LABEL: atomic_sub_i64_ret_addr64_offset:1430; GFX12:       ; %bb.0: ; %entry1431; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x241432; GFX12-NEXT:    s_wait_kmcnt 0x01433; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 31434; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s51435; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]1436; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)1437; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s01438; GFX12-NEXT:    flat_atomic_sub_u64 v[0:1], v[2:3], v[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV1439; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x01440; GFX12-NEXT:    global_inv scope:SCOPE_DEV1441; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s31442; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]1443; GFX12-NEXT:    s_endpgm1444entry:1445  %ptr = getelementptr inbounds i64, ptr %out, i64 %index1446  %gep = getelementptr inbounds i64, ptr %ptr, i64 41447  %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !01448  store i64 %tmp0, ptr %out21449  ret void1450}1451 1452define amdgpu_kernel void @atomic_sub_i64(ptr %out, i64 %in) {1453; GFX7-LABEL: atomic_sub_i64:1454; GFX7:       ; %bb.0: ; %entry1455; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x91456; GFX7-NEXT:    s_mov_b64 s[4:5], 01457; GFX7-NEXT:    s_waitcnt lgkmcnt(0)1458; GFX7-NEXT:    v_mov_b32_e32 v0, s01459; GFX7-NEXT:    v_mov_b32_e32 v1, s11460; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]1461; GFX7-NEXT:    v_mov_b32_e32 v5, s11462; GFX7-NEXT:    v_mov_b32_e32 v6, s31463; GFX7-NEXT:    v_mov_b32_e32 v4, s01464; GFX7-NEXT:  .LBB20_1: ; %atomicrmw.start1465; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=11466; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1467; GFX7-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v21468; GFX7-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc1469; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc1470; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1471; GFX7-NEXT:    buffer_wbinvl1_vol1472; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]1473; GFX7-NEXT:    v_mov_b32_e32 v3, v11474; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]1475; GFX7-NEXT:    v_mov_b32_e32 v2, v01476; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]1477; GFX7-NEXT:    s_cbranch_execnz .LBB20_11478; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end1479; GFX7-NEXT:    s_endpgm1480;1481; GFX8-LABEL: atomic_sub_i64:1482; GFX8:       ; %bb.0: ; %entry1483; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241484; GFX8-NEXT:    s_mov_b64 s[4:5], 01485; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1486; GFX8-NEXT:    v_mov_b32_e32 v0, s01487; GFX8-NEXT:    v_mov_b32_e32 v1, s11488; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]1489; GFX8-NEXT:    v_mov_b32_e32 v5, s11490; GFX8-NEXT:    v_mov_b32_e32 v6, s31491; GFX8-NEXT:    v_mov_b32_e32 v4, s01492; GFX8-NEXT:  .LBB20_1: ; %atomicrmw.start1493; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=11494; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1495; GFX8-NEXT:    v_subrev_u32_e32 v0, vcc, s2, v21496; GFX8-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc1497; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc1498; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1499; GFX8-NEXT:    buffer_wbinvl1_vol1500; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]1501; GFX8-NEXT:    v_mov_b32_e32 v3, v11502; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]1503; GFX8-NEXT:    v_mov_b32_e32 v2, v01504; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]1505; GFX8-NEXT:    s_cbranch_execnz .LBB20_11506; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end1507; GFX8-NEXT:    s_endpgm1508;1509; GFX12-LABEL: atomic_sub_i64:1510; GFX12:       ; %bb.0: ; %entry1511; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241512; GFX12-NEXT:    s_wait_kmcnt 0x01513; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11514; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s31515; GFX12-NEXT:    flat_atomic_sub_u64 v[0:1], v[2:3] scope:SCOPE_DEV1516; GFX12-NEXT:    s_wait_storecnt_dscnt 0x01517; GFX12-NEXT:    global_inv scope:SCOPE_DEV1518; GFX12-NEXT:    s_endpgm1519entry:1520  %tmp0 = atomicrmw volatile sub ptr %out, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !01521  ret void1522}1523 1524define amdgpu_kernel void @atomic_sub_i64_ret(ptr %out, ptr %out2, i64 %in) {1525; GFX7-LABEL: atomic_sub_i64_ret:1526; GFX7:       ; %bb.0: ; %entry1527; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x91528; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd1529; GFX7-NEXT:    s_mov_b64 s[6:7], 01530; GFX7-NEXT:    s_waitcnt lgkmcnt(0)1531; GFX7-NEXT:    v_mov_b32_e32 v0, s01532; GFX7-NEXT:    v_mov_b32_e32 v1, s11533; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]1534; GFX7-NEXT:    v_mov_b32_e32 v3, s11535; GFX7-NEXT:    v_mov_b32_e32 v4, s51536; GFX7-NEXT:    v_mov_b32_e32 v2, s01537; GFX7-NEXT:  .LBB21_1: ; %atomicrmw.start1538; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=11539; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1540; GFX7-NEXT:    v_mov_b32_e32 v8, v11541; GFX7-NEXT:    v_mov_b32_e32 v7, v01542; GFX7-NEXT:    v_subrev_i32_e32 v5, vcc, s4, v71543; GFX7-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc1544; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc1545; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1546; GFX7-NEXT:    buffer_wbinvl1_vol1547; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]1548; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]1549; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]1550; GFX7-NEXT:    s_cbranch_execnz .LBB21_11551; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end1552; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]1553; GFX7-NEXT:    v_mov_b32_e32 v2, s21554; GFX7-NEXT:    v_mov_b32_e32 v3, s31555; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]1556; GFX7-NEXT:    s_endpgm1557;1558; GFX8-LABEL: atomic_sub_i64_ret:1559; GFX8:       ; %bb.0: ; %entry1560; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241561; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x341562; GFX8-NEXT:    s_mov_b64 s[6:7], 01563; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1564; GFX8-NEXT:    v_mov_b32_e32 v0, s01565; GFX8-NEXT:    v_mov_b32_e32 v1, s11566; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]1567; GFX8-NEXT:    v_mov_b32_e32 v3, s11568; GFX8-NEXT:    v_mov_b32_e32 v4, s51569; GFX8-NEXT:    v_mov_b32_e32 v2, s01570; GFX8-NEXT:  .LBB21_1: ; %atomicrmw.start1571; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=11572; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1573; GFX8-NEXT:    v_mov_b32_e32 v8, v11574; GFX8-NEXT:    v_mov_b32_e32 v7, v01575; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, s4, v71576; GFX8-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc1577; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc1578; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1579; GFX8-NEXT:    buffer_wbinvl1_vol1580; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]1581; GFX8-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]1582; GFX8-NEXT:    s_andn2_b64 exec, exec, s[6:7]1583; GFX8-NEXT:    s_cbranch_execnz .LBB21_11584; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end1585; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]1586; GFX8-NEXT:    v_mov_b32_e32 v2, s21587; GFX8-NEXT:    v_mov_b32_e32 v3, s31588; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]1589; GFX8-NEXT:    s_endpgm1590;1591; GFX12-LABEL: atomic_sub_i64_ret:1592; GFX12:       ; %bb.0: ; %entry1593; GFX12-NEXT:    s_clause 0x11594; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241595; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x341596; GFX12-NEXT:    s_wait_kmcnt 0x01597; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11598; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s51599; GFX12-NEXT:    flat_atomic_sub_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV1600; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x01601; GFX12-NEXT:    global_inv scope:SCOPE_DEV1602; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s31603; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]1604; GFX12-NEXT:    s_endpgm1605entry:1606  %tmp0 = atomicrmw volatile sub ptr %out, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !01607  store i64 %tmp0, ptr %out21608  ret void1609}1610 1611define amdgpu_kernel void @atomic_sub_i64_addr64(ptr %out, i64 %in, i64 %index) {1612; GFX7-LABEL: atomic_sub_i64_addr64:1613; GFX7:       ; %bb.0: ; %entry1614; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd1615; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x91616; GFX7-NEXT:    s_waitcnt lgkmcnt(0)1617; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 31618; GFX7-NEXT:    s_add_u32 s0, s0, s41619; GFX7-NEXT:    s_addc_u32 s1, s1, s51620; GFX7-NEXT:    v_mov_b32_e32 v5, s11621; GFX7-NEXT:    v_mov_b32_e32 v4, s01622; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]1623; GFX7-NEXT:    s_mov_b64 s[0:1], 01624; GFX7-NEXT:    v_mov_b32_e32 v6, s31625; GFX7-NEXT:  .LBB22_1: ; %atomicrmw.start1626; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=11627; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1628; GFX7-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v21629; GFX7-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc1630; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc1631; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1632; GFX7-NEXT:    buffer_wbinvl1_vol1633; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]1634; GFX7-NEXT:    v_mov_b32_e32 v3, v11635; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]1636; GFX7-NEXT:    v_mov_b32_e32 v2, v01637; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]1638; GFX7-NEXT:    s_cbranch_execnz .LBB22_11639; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end1640; GFX7-NEXT:    s_endpgm1641;1642; GFX8-LABEL: atomic_sub_i64_addr64:1643; GFX8:       ; %bb.0: ; %entry1644; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x341645; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241646; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1647; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 31648; GFX8-NEXT:    s_add_u32 s0, s0, s41649; GFX8-NEXT:    s_addc_u32 s1, s1, s51650; GFX8-NEXT:    v_mov_b32_e32 v5, s11651; GFX8-NEXT:    v_mov_b32_e32 v4, s01652; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]1653; GFX8-NEXT:    s_mov_b64 s[0:1], 01654; GFX8-NEXT:    v_mov_b32_e32 v6, s31655; GFX8-NEXT:  .LBB22_1: ; %atomicrmw.start1656; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=11657; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1658; GFX8-NEXT:    v_subrev_u32_e32 v0, vcc, s2, v21659; GFX8-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc1660; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc1661; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1662; GFX8-NEXT:    buffer_wbinvl1_vol1663; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]1664; GFX8-NEXT:    v_mov_b32_e32 v3, v11665; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]1666; GFX8-NEXT:    v_mov_b32_e32 v2, v01667; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]1668; GFX8-NEXT:    s_cbranch_execnz .LBB22_11669; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end1670; GFX8-NEXT:    s_endpgm1671;1672; GFX12-LABEL: atomic_sub_i64_addr64:1673; GFX12:       ; %bb.0: ; %entry1674; GFX12-NEXT:    s_clause 0x11675; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x341676; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241677; GFX12-NEXT:    s_wait_kmcnt 0x01678; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 31679; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s31680; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]1681; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)1682; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s01683; GFX12-NEXT:    flat_atomic_sub_u64 v[2:3], v[0:1] scope:SCOPE_DEV1684; GFX12-NEXT:    s_wait_storecnt_dscnt 0x01685; GFX12-NEXT:    global_inv scope:SCOPE_DEV1686; GFX12-NEXT:    s_endpgm1687entry:1688  %ptr = getelementptr inbounds i64, ptr %out, i64 %index1689  %tmp0 = atomicrmw volatile sub ptr %ptr, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !01690  ret void1691}1692 1693define amdgpu_kernel void @atomic_sub_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {1694; GFX7-LABEL: atomic_sub_i64_ret_addr64:1695; GFX7:       ; %bb.0: ; %entry1696; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x91697; GFX7-NEXT:    s_waitcnt lgkmcnt(0)1698; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 31699; GFX7-NEXT:    s_add_u32 s0, s0, s61700; GFX7-NEXT:    s_addc_u32 s1, s1, s71701; GFX7-NEXT:    v_mov_b32_e32 v0, s01702; GFX7-NEXT:    v_mov_b32_e32 v1, s11703; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]1704; GFX7-NEXT:    s_mov_b64 s[0:1], 01705; GFX7-NEXT:    v_mov_b32_e32 v4, s51706; GFX7-NEXT:  .LBB23_1: ; %atomicrmw.start1707; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=11708; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1709; GFX7-NEXT:    v_mov_b32_e32 v8, v31710; GFX7-NEXT:    v_mov_b32_e32 v7, v21711; GFX7-NEXT:    v_subrev_i32_e32 v5, vcc, s4, v71712; GFX7-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc1713; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[5:8] glc1714; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1715; GFX7-NEXT:    buffer_wbinvl1_vol1716; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[7:8]1717; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]1718; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]1719; GFX7-NEXT:    s_cbranch_execnz .LBB23_11720; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end1721; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]1722; GFX7-NEXT:    v_mov_b32_e32 v0, s21723; GFX7-NEXT:    v_mov_b32_e32 v1, s31724; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]1725; GFX7-NEXT:    s_endpgm1726;1727; GFX8-LABEL: atomic_sub_i64_ret_addr64:1728; GFX8:       ; %bb.0: ; %entry1729; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x241730; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1731; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 31732; GFX8-NEXT:    s_add_u32 s0, s0, s61733; GFX8-NEXT:    s_addc_u32 s1, s1, s71734; GFX8-NEXT:    v_mov_b32_e32 v0, s01735; GFX8-NEXT:    v_mov_b32_e32 v1, s11736; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]1737; GFX8-NEXT:    s_mov_b64 s[0:1], 01738; GFX8-NEXT:    v_mov_b32_e32 v4, s51739; GFX8-NEXT:  .LBB23_1: ; %atomicrmw.start1740; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=11741; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1742; GFX8-NEXT:    v_mov_b32_e32 v8, v31743; GFX8-NEXT:    v_mov_b32_e32 v7, v21744; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, s4, v71745; GFX8-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc1746; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[5:8] glc1747; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1748; GFX8-NEXT:    buffer_wbinvl1_vol1749; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[7:8]1750; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]1751; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]1752; GFX8-NEXT:    s_cbranch_execnz .LBB23_11753; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end1754; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]1755; GFX8-NEXT:    v_mov_b32_e32 v0, s21756; GFX8-NEXT:    v_mov_b32_e32 v1, s31757; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]1758; GFX8-NEXT:    s_endpgm1759;1760; GFX12-LABEL: atomic_sub_i64_ret_addr64:1761; GFX12:       ; %bb.0: ; %entry1762; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x241763; GFX12-NEXT:    s_wait_kmcnt 0x01764; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 31765; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s51766; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]1767; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)1768; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s01769; GFX12-NEXT:    flat_atomic_sub_u64 v[0:1], v[2:3], v[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV1770; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x01771; GFX12-NEXT:    global_inv scope:SCOPE_DEV1772; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s31773; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]1774; GFX12-NEXT:    s_endpgm1775entry:1776  %ptr = getelementptr inbounds i64, ptr %out, i64 %index1777  %tmp0 = atomicrmw volatile sub ptr %ptr, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !01778  store i64 %tmp0, ptr %out21779  ret void1780}1781 1782define amdgpu_kernel void @atomic_max_i64_offset(ptr %out, i64 %in) {1783; GFX7-LABEL: atomic_max_i64_offset:1784; GFX7:       ; %bb.0: ; %entry1785; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x91786; GFX7-NEXT:    s_waitcnt lgkmcnt(0)1787; GFX7-NEXT:    s_add_u32 s0, s0, 321788; GFX7-NEXT:    s_addc_u32 s1, s1, 01789; GFX7-NEXT:    v_mov_b32_e32 v5, s11790; GFX7-NEXT:    v_mov_b32_e32 v4, s01791; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]1792; GFX7-NEXT:    s_mov_b64 s[0:1], 01793; GFX7-NEXT:    v_mov_b32_e32 v6, s31794; GFX7-NEXT:    v_mov_b32_e32 v7, s21795; GFX7-NEXT:  .LBB24_1: ; %atomicrmw.start1796; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=11797; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1798; GFX7-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]1799; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc1800; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc1801; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc1802; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1803; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]1804; GFX7-NEXT:    v_mov_b32_e32 v3, v11805; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]1806; GFX7-NEXT:    v_mov_b32_e32 v2, v01807; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]1808; GFX7-NEXT:    s_cbranch_execnz .LBB24_11809; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end1810; GFX7-NEXT:    s_endpgm1811;1812; GFX8-LABEL: atomic_max_i64_offset:1813; GFX8:       ; %bb.0: ; %entry1814; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241815; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1816; GFX8-NEXT:    s_add_u32 s0, s0, 321817; GFX8-NEXT:    s_addc_u32 s1, s1, 01818; GFX8-NEXT:    v_mov_b32_e32 v5, s11819; GFX8-NEXT:    v_mov_b32_e32 v4, s01820; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]1821; GFX8-NEXT:    s_mov_b64 s[0:1], 01822; GFX8-NEXT:    v_mov_b32_e32 v6, s31823; GFX8-NEXT:    v_mov_b32_e32 v7, s21824; GFX8-NEXT:  .LBB24_1: ; %atomicrmw.start1825; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=11826; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1827; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]1828; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc1829; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc1830; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc1831; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1832; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]1833; GFX8-NEXT:    v_mov_b32_e32 v3, v11834; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]1835; GFX8-NEXT:    v_mov_b32_e32 v2, v01836; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]1837; GFX8-NEXT:    s_cbranch_execnz .LBB24_11838; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end1839; GFX8-NEXT:    s_endpgm1840;1841; GFX12-LABEL: atomic_max_i64_offset:1842; GFX12:       ; %bb.0: ; %entry1843; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241844; GFX12-NEXT:    s_wait_kmcnt 0x01845; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11846; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s31847; GFX12-NEXT:    flat_atomic_max_i64 v[0:1], v[2:3] offset:32 scope:SCOPE_SE1848; GFX12-NEXT:    s_wait_storecnt_dscnt 0x01849; GFX12-NEXT:    global_inv scope:SCOPE_SE1850; GFX12-NEXT:    s_endpgm1851entry:1852  %gep = getelementptr inbounds i64, ptr %out, i64 41853  %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !01854  ret void1855}1856 1857define amdgpu_kernel void @atomic_max_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {1858; GFX7-LABEL: atomic_max_i64_ret_offset:1859; GFX7:       ; %bb.0: ; %entry1860; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x91861; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd1862; GFX7-NEXT:    s_waitcnt lgkmcnt(0)1863; GFX7-NEXT:    s_add_u32 s0, s0, 321864; GFX7-NEXT:    s_addc_u32 s1, s1, 01865; GFX7-NEXT:    v_mov_b32_e32 v0, s01866; GFX7-NEXT:    v_mov_b32_e32 v1, s11867; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]1868; GFX7-NEXT:    s_mov_b64 s[0:1], 01869; GFX7-NEXT:    v_mov_b32_e32 v4, s51870; GFX7-NEXT:    v_mov_b32_e32 v5, s41871; GFX7-NEXT:  .LBB25_1: ; %atomicrmw.start1872; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=11873; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1874; GFX7-NEXT:    v_mov_b32_e32 v9, v31875; GFX7-NEXT:    v_mov_b32_e32 v8, v21876; GFX7-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]1877; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc1878; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc1879; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc1880; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1881; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]1882; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]1883; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]1884; GFX7-NEXT:    s_cbranch_execnz .LBB25_11885; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end1886; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]1887; GFX7-NEXT:    v_mov_b32_e32 v0, s21888; GFX7-NEXT:    v_mov_b32_e32 v1, s31889; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]1890; GFX7-NEXT:    s_endpgm1891;1892; GFX8-LABEL: atomic_max_i64_ret_offset:1893; GFX8:       ; %bb.0: ; %entry1894; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241895; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x341896; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1897; GFX8-NEXT:    s_add_u32 s0, s0, 321898; GFX8-NEXT:    s_addc_u32 s1, s1, 01899; GFX8-NEXT:    v_mov_b32_e32 v0, s01900; GFX8-NEXT:    v_mov_b32_e32 v1, s11901; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]1902; GFX8-NEXT:    s_mov_b64 s[0:1], 01903; GFX8-NEXT:    v_mov_b32_e32 v4, s51904; GFX8-NEXT:    v_mov_b32_e32 v5, s41905; GFX8-NEXT:  .LBB25_1: ; %atomicrmw.start1906; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=11907; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1908; GFX8-NEXT:    v_mov_b32_e32 v9, v31909; GFX8-NEXT:    v_mov_b32_e32 v8, v21910; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]1911; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc1912; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc1913; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc1914; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1915; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]1916; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]1917; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]1918; GFX8-NEXT:    s_cbranch_execnz .LBB25_11919; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end1920; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]1921; GFX8-NEXT:    v_mov_b32_e32 v0, s21922; GFX8-NEXT:    v_mov_b32_e32 v1, s31923; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]1924; GFX8-NEXT:    s_endpgm1925;1926; GFX12-LABEL: atomic_max_i64_ret_offset:1927; GFX12:       ; %bb.0: ; %entry1928; GFX12-NEXT:    s_clause 0x11929; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241930; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x341931; GFX12-NEXT:    s_wait_kmcnt 0x01932; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11933; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s51934; GFX12-NEXT:    flat_atomic_max_i64 v[0:1], v[0:1], v[2:3] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_SE1935; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x01936; GFX12-NEXT:    global_inv scope:SCOPE_SE1937; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s31938; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]1939; GFX12-NEXT:    s_endpgm1940entry:1941  %gep = getelementptr inbounds i64, ptr %out, i64 41942  %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !01943  store i64 %tmp0, ptr %out21944  ret void1945}1946 1947define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {1948; GFX7-LABEL: atomic_max_i64_addr64_offset:1949; GFX7:       ; %bb.0: ; %entry1950; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd1951; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x91952; GFX7-NEXT:    s_waitcnt lgkmcnt(0)1953; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 31954; GFX7-NEXT:    s_add_u32 s0, s0, s41955; GFX7-NEXT:    s_addc_u32 s1, s1, s51956; GFX7-NEXT:    s_add_u32 s0, s0, 321957; GFX7-NEXT:    s_addc_u32 s1, s1, 01958; GFX7-NEXT:    v_mov_b32_e32 v5, s11959; GFX7-NEXT:    v_mov_b32_e32 v4, s01960; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]1961; GFX7-NEXT:    s_mov_b64 s[0:1], 01962; GFX7-NEXT:    v_mov_b32_e32 v6, s31963; GFX7-NEXT:    v_mov_b32_e32 v7, s21964; GFX7-NEXT:  .LBB26_1: ; %atomicrmw.start1965; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=11966; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1967; GFX7-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]1968; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc1969; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc1970; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc1971; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1972; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]1973; GFX7-NEXT:    v_mov_b32_e32 v3, v11974; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]1975; GFX7-NEXT:    v_mov_b32_e32 v2, v01976; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]1977; GFX7-NEXT:    s_cbranch_execnz .LBB26_11978; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end1979; GFX7-NEXT:    s_endpgm1980;1981; GFX8-LABEL: atomic_max_i64_addr64_offset:1982; GFX8:       ; %bb.0: ; %entry1983; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x341984; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241985; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1986; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 31987; GFX8-NEXT:    s_add_u32 s0, s0, s41988; GFX8-NEXT:    s_addc_u32 s1, s1, s51989; GFX8-NEXT:    s_add_u32 s0, s0, 321990; GFX8-NEXT:    s_addc_u32 s1, s1, 01991; GFX8-NEXT:    v_mov_b32_e32 v5, s11992; GFX8-NEXT:    v_mov_b32_e32 v4, s01993; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]1994; GFX8-NEXT:    s_mov_b64 s[0:1], 01995; GFX8-NEXT:    v_mov_b32_e32 v6, s31996; GFX8-NEXT:    v_mov_b32_e32 v7, s21997; GFX8-NEXT:  .LBB26_1: ; %atomicrmw.start1998; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=11999; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2000; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]2001; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc2002; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc2003; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc2004; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2005; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]2006; GFX8-NEXT:    v_mov_b32_e32 v3, v12007; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]2008; GFX8-NEXT:    v_mov_b32_e32 v2, v02009; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]2010; GFX8-NEXT:    s_cbranch_execnz .LBB26_12011; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end2012; GFX8-NEXT:    s_endpgm2013;2014; GFX12-LABEL: atomic_max_i64_addr64_offset:2015; GFX12:       ; %bb.0: ; %entry2016; GFX12-NEXT:    s_clause 0x12017; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x342018; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x242019; GFX12-NEXT:    s_wait_kmcnt 0x02020; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 32021; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s32022; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]2023; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)2024; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s02025; GFX12-NEXT:    flat_atomic_max_i64 v[2:3], v[0:1] offset:32 scope:SCOPE_SE2026; GFX12-NEXT:    s_wait_storecnt_dscnt 0x02027; GFX12-NEXT:    global_inv scope:SCOPE_SE2028; GFX12-NEXT:    s_endpgm2029entry:2030  %ptr = getelementptr inbounds i64, ptr %out, i64 %index2031  %gep = getelementptr inbounds i64, ptr %ptr, i64 42032  %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !02033  ret void2034}2035 2036define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {2037; GFX7-LABEL: atomic_max_i64_ret_addr64_offset:2038; GFX7:       ; %bb.0: ; %entry2039; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x92040; GFX7-NEXT:    s_waitcnt lgkmcnt(0)2041; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 32042; GFX7-NEXT:    s_add_u32 s0, s0, s62043; GFX7-NEXT:    s_addc_u32 s1, s1, s72044; GFX7-NEXT:    s_add_u32 s0, s0, 322045; GFX7-NEXT:    s_addc_u32 s1, s1, 02046; GFX7-NEXT:    v_mov_b32_e32 v0, s02047; GFX7-NEXT:    v_mov_b32_e32 v1, s12048; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]2049; GFX7-NEXT:    s_mov_b64 s[0:1], 02050; GFX7-NEXT:    v_mov_b32_e32 v4, s52051; GFX7-NEXT:    v_mov_b32_e32 v5, s42052; GFX7-NEXT:  .LBB27_1: ; %atomicrmw.start2053; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=12054; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2055; GFX7-NEXT:    v_mov_b32_e32 v9, v32056; GFX7-NEXT:    v_mov_b32_e32 v8, v22057; GFX7-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]2058; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc2059; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc2060; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc2061; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2062; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]2063; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]2064; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]2065; GFX7-NEXT:    s_cbranch_execnz .LBB27_12066; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end2067; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]2068; GFX7-NEXT:    v_mov_b32_e32 v0, s22069; GFX7-NEXT:    v_mov_b32_e32 v1, s32070; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]2071; GFX7-NEXT:    s_endpgm2072;2073; GFX8-LABEL: atomic_max_i64_ret_addr64_offset:2074; GFX8:       ; %bb.0: ; %entry2075; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x242076; GFX8-NEXT:    s_waitcnt lgkmcnt(0)2077; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 32078; GFX8-NEXT:    s_add_u32 s0, s0, s62079; GFX8-NEXT:    s_addc_u32 s1, s1, s72080; GFX8-NEXT:    s_add_u32 s0, s0, 322081; GFX8-NEXT:    s_addc_u32 s1, s1, 02082; GFX8-NEXT:    v_mov_b32_e32 v0, s02083; GFX8-NEXT:    v_mov_b32_e32 v1, s12084; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]2085; GFX8-NEXT:    s_mov_b64 s[0:1], 02086; GFX8-NEXT:    v_mov_b32_e32 v4, s52087; GFX8-NEXT:    v_mov_b32_e32 v5, s42088; GFX8-NEXT:  .LBB27_1: ; %atomicrmw.start2089; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=12090; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2091; GFX8-NEXT:    v_mov_b32_e32 v9, v32092; GFX8-NEXT:    v_mov_b32_e32 v8, v22093; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]2094; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc2095; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc2096; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc2097; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2098; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]2099; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]2100; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]2101; GFX8-NEXT:    s_cbranch_execnz .LBB27_12102; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end2103; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]2104; GFX8-NEXT:    v_mov_b32_e32 v0, s22105; GFX8-NEXT:    v_mov_b32_e32 v1, s32106; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]2107; GFX8-NEXT:    s_endpgm2108;2109; GFX12-LABEL: atomic_max_i64_ret_addr64_offset:2110; GFX12:       ; %bb.0: ; %entry2111; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x242112; GFX12-NEXT:    s_wait_kmcnt 0x02113; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 32114; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s52115; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]2116; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)2117; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s02118; GFX12-NEXT:    flat_atomic_max_i64 v[0:1], v[2:3], v[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_SE2119; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x02120; GFX12-NEXT:    global_inv scope:SCOPE_SE2121; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s32122; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]2123; GFX12-NEXT:    s_endpgm2124entry:2125  %ptr = getelementptr inbounds i64, ptr %out, i64 %index2126  %gep = getelementptr inbounds i64, ptr %ptr, i64 42127  %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !02128  store i64 %tmp0, ptr %out22129  ret void2130}2131 2132define amdgpu_kernel void @atomic_max_i64(ptr %out, i64 %in) {2133; GFX7-LABEL: atomic_max_i64:2134; GFX7:       ; %bb.0: ; %entry2135; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x92136; GFX7-NEXT:    s_mov_b64 s[4:5], 02137; GFX7-NEXT:    s_waitcnt lgkmcnt(0)2138; GFX7-NEXT:    v_mov_b32_e32 v0, s02139; GFX7-NEXT:    v_mov_b32_e32 v1, s12140; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]2141; GFX7-NEXT:    v_mov_b32_e32 v5, s12142; GFX7-NEXT:    v_mov_b32_e32 v6, s32143; GFX7-NEXT:    v_mov_b32_e32 v7, s22144; GFX7-NEXT:    v_mov_b32_e32 v4, s02145; GFX7-NEXT:  .LBB28_1: ; %atomicrmw.start2146; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=12147; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2148; GFX7-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]2149; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc2150; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc2151; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc2152; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2153; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]2154; GFX7-NEXT:    v_mov_b32_e32 v3, v12155; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]2156; GFX7-NEXT:    v_mov_b32_e32 v2, v02157; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]2158; GFX7-NEXT:    s_cbranch_execnz .LBB28_12159; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end2160; GFX7-NEXT:    s_endpgm2161;2162; GFX8-LABEL: atomic_max_i64:2163; GFX8:       ; %bb.0: ; %entry2164; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242165; GFX8-NEXT:    s_mov_b64 s[4:5], 02166; GFX8-NEXT:    s_waitcnt lgkmcnt(0)2167; GFX8-NEXT:    v_mov_b32_e32 v0, s02168; GFX8-NEXT:    v_mov_b32_e32 v1, s12169; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]2170; GFX8-NEXT:    v_mov_b32_e32 v5, s12171; GFX8-NEXT:    v_mov_b32_e32 v6, s32172; GFX8-NEXT:    v_mov_b32_e32 v7, s22173; GFX8-NEXT:    v_mov_b32_e32 v4, s02174; GFX8-NEXT:  .LBB28_1: ; %atomicrmw.start2175; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=12176; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2177; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]2178; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc2179; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc2180; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc2181; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2182; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]2183; GFX8-NEXT:    v_mov_b32_e32 v3, v12184; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]2185; GFX8-NEXT:    v_mov_b32_e32 v2, v02186; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]2187; GFX8-NEXT:    s_cbranch_execnz .LBB28_12188; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end2189; GFX8-NEXT:    s_endpgm2190;2191; GFX12-LABEL: atomic_max_i64:2192; GFX12:       ; %bb.0: ; %entry2193; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x242194; GFX12-NEXT:    s_wait_kmcnt 0x02195; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12196; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s32197; GFX12-NEXT:    flat_atomic_max_i64 v[0:1], v[2:3] scope:SCOPE_SE2198; GFX12-NEXT:    s_wait_storecnt_dscnt 0x02199; GFX12-NEXT:    global_inv scope:SCOPE_SE2200; GFX12-NEXT:    s_endpgm2201entry:2202  %tmp0 = atomicrmw volatile max ptr %out, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !02203  ret void2204}2205 2206define amdgpu_kernel void @atomic_max_i64_ret(ptr %out, ptr %out2, i64 %in) {2207; GFX7-LABEL: atomic_max_i64_ret:2208; GFX7:       ; %bb.0: ; %entry2209; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x92210; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd2211; GFX7-NEXT:    s_mov_b64 s[6:7], 02212; GFX7-NEXT:    s_waitcnt lgkmcnt(0)2213; GFX7-NEXT:    v_mov_b32_e32 v0, s02214; GFX7-NEXT:    v_mov_b32_e32 v1, s12215; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]2216; GFX7-NEXT:    v_mov_b32_e32 v3, s12217; GFX7-NEXT:    v_mov_b32_e32 v4, s52218; GFX7-NEXT:    v_mov_b32_e32 v5, s42219; GFX7-NEXT:    v_mov_b32_e32 v2, s02220; GFX7-NEXT:  .LBB29_1: ; %atomicrmw.start2221; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=12222; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2223; GFX7-NEXT:    v_mov_b32_e32 v9, v12224; GFX7-NEXT:    v_mov_b32_e32 v8, v02225; GFX7-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]2226; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc2227; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc2228; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc2229; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2230; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]2231; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]2232; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]2233; GFX7-NEXT:    s_cbranch_execnz .LBB29_12234; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end2235; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]2236; GFX7-NEXT:    v_mov_b32_e32 v2, s22237; GFX7-NEXT:    v_mov_b32_e32 v3, s32238; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]2239; GFX7-NEXT:    s_endpgm2240;2241; GFX8-LABEL: atomic_max_i64_ret:2242; GFX8:       ; %bb.0: ; %entry2243; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242244; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x342245; GFX8-NEXT:    s_mov_b64 s[6:7], 02246; GFX8-NEXT:    s_waitcnt lgkmcnt(0)2247; GFX8-NEXT:    v_mov_b32_e32 v0, s02248; GFX8-NEXT:    v_mov_b32_e32 v1, s12249; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]2250; GFX8-NEXT:    v_mov_b32_e32 v3, s12251; GFX8-NEXT:    v_mov_b32_e32 v4, s52252; GFX8-NEXT:    v_mov_b32_e32 v5, s42253; GFX8-NEXT:    v_mov_b32_e32 v2, s02254; GFX8-NEXT:  .LBB29_1: ; %atomicrmw.start2255; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=12256; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2257; GFX8-NEXT:    v_mov_b32_e32 v9, v12258; GFX8-NEXT:    v_mov_b32_e32 v8, v02259; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]2260; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc2261; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc2262; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc2263; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2264; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]2265; GFX8-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]2266; GFX8-NEXT:    s_andn2_b64 exec, exec, s[6:7]2267; GFX8-NEXT:    s_cbranch_execnz .LBB29_12268; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end2269; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]2270; GFX8-NEXT:    v_mov_b32_e32 v2, s22271; GFX8-NEXT:    v_mov_b32_e32 v3, s32272; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]2273; GFX8-NEXT:    s_endpgm2274;2275; GFX12-LABEL: atomic_max_i64_ret:2276; GFX12:       ; %bb.0: ; %entry2277; GFX12-NEXT:    s_clause 0x12278; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x242279; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x342280; GFX12-NEXT:    s_wait_kmcnt 0x02281; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12282; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s52283; GFX12-NEXT:    flat_atomic_max_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE2284; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x02285; GFX12-NEXT:    global_inv scope:SCOPE_SE2286; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s32287; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]2288; GFX12-NEXT:    s_endpgm2289entry:2290  %tmp0 = atomicrmw volatile max ptr %out, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !02291  store i64 %tmp0, ptr %out22292  ret void2293}2294 2295define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index) {2296; GFX7-LABEL: atomic_max_i64_addr64:2297; GFX7:       ; %bb.0: ; %entry2298; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd2299; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x92300; GFX7-NEXT:    s_waitcnt lgkmcnt(0)2301; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 32302; GFX7-NEXT:    s_add_u32 s0, s0, s42303; GFX7-NEXT:    s_addc_u32 s1, s1, s52304; GFX7-NEXT:    v_mov_b32_e32 v5, s12305; GFX7-NEXT:    v_mov_b32_e32 v4, s02306; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]2307; GFX7-NEXT:    s_mov_b64 s[0:1], 02308; GFX7-NEXT:    v_mov_b32_e32 v6, s32309; GFX7-NEXT:    v_mov_b32_e32 v7, s22310; GFX7-NEXT:  .LBB30_1: ; %atomicrmw.start2311; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=12312; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2313; GFX7-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]2314; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc2315; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc2316; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc2317; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2318; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]2319; GFX7-NEXT:    v_mov_b32_e32 v3, v12320; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]2321; GFX7-NEXT:    v_mov_b32_e32 v2, v02322; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]2323; GFX7-NEXT:    s_cbranch_execnz .LBB30_12324; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end2325; GFX7-NEXT:    s_endpgm2326;2327; GFX8-LABEL: atomic_max_i64_addr64:2328; GFX8:       ; %bb.0: ; %entry2329; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x342330; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242331; GFX8-NEXT:    s_waitcnt lgkmcnt(0)2332; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 32333; GFX8-NEXT:    s_add_u32 s0, s0, s42334; GFX8-NEXT:    s_addc_u32 s1, s1, s52335; GFX8-NEXT:    v_mov_b32_e32 v5, s12336; GFX8-NEXT:    v_mov_b32_e32 v4, s02337; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]2338; GFX8-NEXT:    s_mov_b64 s[0:1], 02339; GFX8-NEXT:    v_mov_b32_e32 v6, s32340; GFX8-NEXT:    v_mov_b32_e32 v7, s22341; GFX8-NEXT:  .LBB30_1: ; %atomicrmw.start2342; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=12343; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2344; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]2345; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc2346; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc2347; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc2348; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2349; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]2350; GFX8-NEXT:    v_mov_b32_e32 v3, v12351; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]2352; GFX8-NEXT:    v_mov_b32_e32 v2, v02353; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]2354; GFX8-NEXT:    s_cbranch_execnz .LBB30_12355; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end2356; GFX8-NEXT:    s_endpgm2357;2358; GFX12-LABEL: atomic_max_i64_addr64:2359; GFX12:       ; %bb.0: ; %entry2360; GFX12-NEXT:    s_clause 0x12361; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x342362; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x242363; GFX12-NEXT:    s_wait_kmcnt 0x02364; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 32365; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s32366; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]2367; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)2368; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s02369; GFX12-NEXT:    flat_atomic_max_i64 v[2:3], v[0:1] scope:SCOPE_SE2370; GFX12-NEXT:    s_wait_storecnt_dscnt 0x02371; GFX12-NEXT:    global_inv scope:SCOPE_SE2372; GFX12-NEXT:    s_endpgm2373entry:2374  %ptr = getelementptr inbounds i64, ptr %out, i64 %index2375  %tmp0 = atomicrmw volatile max ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !02376  ret void2377}2378 2379define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {2380; GFX7-LABEL: atomic_max_i64_ret_addr64:2381; GFX7:       ; %bb.0: ; %entry2382; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x92383; GFX7-NEXT:    s_waitcnt lgkmcnt(0)2384; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 32385; GFX7-NEXT:    s_add_u32 s0, s0, s62386; GFX7-NEXT:    s_addc_u32 s1, s1, s72387; GFX7-NEXT:    v_mov_b32_e32 v0, s02388; GFX7-NEXT:    v_mov_b32_e32 v1, s12389; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]2390; GFX7-NEXT:    s_mov_b64 s[0:1], 02391; GFX7-NEXT:    v_mov_b32_e32 v4, s52392; GFX7-NEXT:    v_mov_b32_e32 v5, s42393; GFX7-NEXT:  .LBB31_1: ; %atomicrmw.start2394; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=12395; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2396; GFX7-NEXT:    v_mov_b32_e32 v9, v32397; GFX7-NEXT:    v_mov_b32_e32 v8, v22398; GFX7-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]2399; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc2400; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc2401; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc2402; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2403; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]2404; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]2405; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]2406; GFX7-NEXT:    s_cbranch_execnz .LBB31_12407; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end2408; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]2409; GFX7-NEXT:    v_mov_b32_e32 v0, s22410; GFX7-NEXT:    v_mov_b32_e32 v1, s32411; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]2412; GFX7-NEXT:    s_endpgm2413;2414; GFX8-LABEL: atomic_max_i64_ret_addr64:2415; GFX8:       ; %bb.0: ; %entry2416; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x242417; GFX8-NEXT:    s_waitcnt lgkmcnt(0)2418; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 32419; GFX8-NEXT:    s_add_u32 s0, s0, s62420; GFX8-NEXT:    s_addc_u32 s1, s1, s72421; GFX8-NEXT:    v_mov_b32_e32 v0, s02422; GFX8-NEXT:    v_mov_b32_e32 v1, s12423; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]2424; GFX8-NEXT:    s_mov_b64 s[0:1], 02425; GFX8-NEXT:    v_mov_b32_e32 v4, s52426; GFX8-NEXT:    v_mov_b32_e32 v5, s42427; GFX8-NEXT:  .LBB31_1: ; %atomicrmw.start2428; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=12429; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2430; GFX8-NEXT:    v_mov_b32_e32 v9, v32431; GFX8-NEXT:    v_mov_b32_e32 v8, v22432; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]2433; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc2434; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc2435; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc2436; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2437; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]2438; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]2439; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]2440; GFX8-NEXT:    s_cbranch_execnz .LBB31_12441; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end2442; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]2443; GFX8-NEXT:    v_mov_b32_e32 v0, s22444; GFX8-NEXT:    v_mov_b32_e32 v1, s32445; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]2446; GFX8-NEXT:    s_endpgm2447;2448; GFX12-LABEL: atomic_max_i64_ret_addr64:2449; GFX12:       ; %bb.0: ; %entry2450; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x242451; GFX12-NEXT:    s_wait_kmcnt 0x02452; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 32453; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s52454; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]2455; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)2456; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s02457; GFX12-NEXT:    flat_atomic_max_i64 v[0:1], v[2:3], v[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SE2458; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x02459; GFX12-NEXT:    global_inv scope:SCOPE_SE2460; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s32461; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]2462; GFX12-NEXT:    s_endpgm2463entry:2464  %ptr = getelementptr inbounds i64, ptr %out, i64 %index2465  %tmp0 = atomicrmw volatile max ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !02466  store i64 %tmp0, ptr %out22467  ret void2468}2469 2470define amdgpu_kernel void @atomic_umax_i64_offset(ptr %out, i64 %in) {2471; GFX7-LABEL: atomic_umax_i64_offset:2472; GFX7:       ; %bb.0: ; %entry2473; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x92474; GFX7-NEXT:    s_waitcnt lgkmcnt(0)2475; GFX7-NEXT:    s_add_u32 s0, s0, 322476; GFX7-NEXT:    s_addc_u32 s1, s1, 02477; GFX7-NEXT:    v_mov_b32_e32 v5, s12478; GFX7-NEXT:    v_mov_b32_e32 v4, s02479; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]2480; GFX7-NEXT:    s_mov_b64 s[0:1], 02481; GFX7-NEXT:    v_mov_b32_e32 v6, s32482; GFX7-NEXT:    v_mov_b32_e32 v7, s22483; GFX7-NEXT:  .LBB32_1: ; %atomicrmw.start2484; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=12485; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2486; GFX7-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]2487; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc2488; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc2489; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc2490; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2491; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]2492; GFX7-NEXT:    v_mov_b32_e32 v3, v12493; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]2494; GFX7-NEXT:    v_mov_b32_e32 v2, v02495; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]2496; GFX7-NEXT:    s_cbranch_execnz .LBB32_12497; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end2498; GFX7-NEXT:    s_endpgm2499;2500; GFX8-LABEL: atomic_umax_i64_offset:2501; GFX8:       ; %bb.0: ; %entry2502; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242503; GFX8-NEXT:    s_waitcnt lgkmcnt(0)2504; GFX8-NEXT:    s_add_u32 s0, s0, 322505; GFX8-NEXT:    s_addc_u32 s1, s1, 02506; GFX8-NEXT:    v_mov_b32_e32 v5, s12507; GFX8-NEXT:    v_mov_b32_e32 v4, s02508; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]2509; GFX8-NEXT:    s_mov_b64 s[0:1], 02510; GFX8-NEXT:    v_mov_b32_e32 v6, s32511; GFX8-NEXT:    v_mov_b32_e32 v7, s22512; GFX8-NEXT:  .LBB32_1: ; %atomicrmw.start2513; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=12514; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2515; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]2516; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc2517; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc2518; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc2519; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2520; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]2521; GFX8-NEXT:    v_mov_b32_e32 v3, v12522; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]2523; GFX8-NEXT:    v_mov_b32_e32 v2, v02524; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]2525; GFX8-NEXT:    s_cbranch_execnz .LBB32_12526; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end2527; GFX8-NEXT:    s_endpgm2528;2529; GFX12-LABEL: atomic_umax_i64_offset:2530; GFX12:       ; %bb.0: ; %entry2531; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x242532; GFX12-NEXT:    s_wait_kmcnt 0x02533; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12534; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s32535; GFX12-NEXT:    flat_atomic_max_u64 v[0:1], v[2:3] offset:32 scope:SCOPE_SE2536; GFX12-NEXT:    s_wait_storecnt_dscnt 0x02537; GFX12-NEXT:    global_inv scope:SCOPE_SE2538; GFX12-NEXT:    s_endpgm2539entry:2540  %gep = getelementptr inbounds i64, ptr %out, i64 42541  %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !02542  ret void2543}2544 2545define amdgpu_kernel void @atomic_umax_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {2546; GFX7-LABEL: atomic_umax_i64_ret_offset:2547; GFX7:       ; %bb.0: ; %entry2548; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x92549; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd2550; GFX7-NEXT:    s_waitcnt lgkmcnt(0)2551; GFX7-NEXT:    s_add_u32 s0, s0, 322552; GFX7-NEXT:    s_addc_u32 s1, s1, 02553; GFX7-NEXT:    v_mov_b32_e32 v0, s02554; GFX7-NEXT:    v_mov_b32_e32 v1, s12555; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]2556; GFX7-NEXT:    s_mov_b64 s[0:1], 02557; GFX7-NEXT:    v_mov_b32_e32 v4, s52558; GFX7-NEXT:    v_mov_b32_e32 v5, s42559; GFX7-NEXT:  .LBB33_1: ; %atomicrmw.start2560; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=12561; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2562; GFX7-NEXT:    v_mov_b32_e32 v9, v32563; GFX7-NEXT:    v_mov_b32_e32 v8, v22564; GFX7-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]2565; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc2566; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc2567; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc2568; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2569; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]2570; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]2571; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]2572; GFX7-NEXT:    s_cbranch_execnz .LBB33_12573; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end2574; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]2575; GFX7-NEXT:    v_mov_b32_e32 v0, s22576; GFX7-NEXT:    v_mov_b32_e32 v1, s32577; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]2578; GFX7-NEXT:    s_endpgm2579;2580; GFX8-LABEL: atomic_umax_i64_ret_offset:2581; GFX8:       ; %bb.0: ; %entry2582; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242583; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x342584; GFX8-NEXT:    s_waitcnt lgkmcnt(0)2585; GFX8-NEXT:    s_add_u32 s0, s0, 322586; GFX8-NEXT:    s_addc_u32 s1, s1, 02587; GFX8-NEXT:    v_mov_b32_e32 v0, s02588; GFX8-NEXT:    v_mov_b32_e32 v1, s12589; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]2590; GFX8-NEXT:    s_mov_b64 s[0:1], 02591; GFX8-NEXT:    v_mov_b32_e32 v4, s52592; GFX8-NEXT:    v_mov_b32_e32 v5, s42593; GFX8-NEXT:  .LBB33_1: ; %atomicrmw.start2594; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=12595; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2596; GFX8-NEXT:    v_mov_b32_e32 v9, v32597; GFX8-NEXT:    v_mov_b32_e32 v8, v22598; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]2599; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc2600; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc2601; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc2602; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2603; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]2604; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]2605; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]2606; GFX8-NEXT:    s_cbranch_execnz .LBB33_12607; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end2608; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]2609; GFX8-NEXT:    v_mov_b32_e32 v0, s22610; GFX8-NEXT:    v_mov_b32_e32 v1, s32611; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]2612; GFX8-NEXT:    s_endpgm2613;2614; GFX12-LABEL: atomic_umax_i64_ret_offset:2615; GFX12:       ; %bb.0: ; %entry2616; GFX12-NEXT:    s_clause 0x12617; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x242618; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x342619; GFX12-NEXT:    s_wait_kmcnt 0x02620; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12621; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s52622; GFX12-NEXT:    flat_atomic_max_u64 v[0:1], v[0:1], v[2:3] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_SE2623; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x02624; GFX12-NEXT:    global_inv scope:SCOPE_SE2625; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s32626; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]2627; GFX12-NEXT:    s_endpgm2628entry:2629  %gep = getelementptr inbounds i64, ptr %out, i64 42630  %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !02631  store i64 %tmp0, ptr %out22632  ret void2633}2634 2635define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {2636; GFX7-LABEL: atomic_umax_i64_addr64_offset:2637; GFX7:       ; %bb.0: ; %entry2638; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd2639; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x92640; GFX7-NEXT:    s_waitcnt lgkmcnt(0)2641; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 32642; GFX7-NEXT:    s_add_u32 s0, s0, s42643; GFX7-NEXT:    s_addc_u32 s1, s1, s52644; GFX7-NEXT:    s_add_u32 s0, s0, 322645; GFX7-NEXT:    s_addc_u32 s1, s1, 02646; GFX7-NEXT:    v_mov_b32_e32 v5, s12647; GFX7-NEXT:    v_mov_b32_e32 v4, s02648; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]2649; GFX7-NEXT:    s_mov_b64 s[0:1], 02650; GFX7-NEXT:    v_mov_b32_e32 v6, s32651; GFX7-NEXT:    v_mov_b32_e32 v7, s22652; GFX7-NEXT:  .LBB34_1: ; %atomicrmw.start2653; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=12654; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2655; GFX7-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]2656; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc2657; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc2658; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc2659; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2660; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]2661; GFX7-NEXT:    v_mov_b32_e32 v3, v12662; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]2663; GFX7-NEXT:    v_mov_b32_e32 v2, v02664; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]2665; GFX7-NEXT:    s_cbranch_execnz .LBB34_12666; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end2667; GFX7-NEXT:    s_endpgm2668;2669; GFX8-LABEL: atomic_umax_i64_addr64_offset:2670; GFX8:       ; %bb.0: ; %entry2671; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x342672; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242673; GFX8-NEXT:    s_waitcnt lgkmcnt(0)2674; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 32675; GFX8-NEXT:    s_add_u32 s0, s0, s42676; GFX8-NEXT:    s_addc_u32 s1, s1, s52677; GFX8-NEXT:    s_add_u32 s0, s0, 322678; GFX8-NEXT:    s_addc_u32 s1, s1, 02679; GFX8-NEXT:    v_mov_b32_e32 v5, s12680; GFX8-NEXT:    v_mov_b32_e32 v4, s02681; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]2682; GFX8-NEXT:    s_mov_b64 s[0:1], 02683; GFX8-NEXT:    v_mov_b32_e32 v6, s32684; GFX8-NEXT:    v_mov_b32_e32 v7, s22685; GFX8-NEXT:  .LBB34_1: ; %atomicrmw.start2686; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=12687; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2688; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]2689; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc2690; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc2691; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc2692; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2693; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]2694; GFX8-NEXT:    v_mov_b32_e32 v3, v12695; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]2696; GFX8-NEXT:    v_mov_b32_e32 v2, v02697; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]2698; GFX8-NEXT:    s_cbranch_execnz .LBB34_12699; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end2700; GFX8-NEXT:    s_endpgm2701;2702; GFX12-LABEL: atomic_umax_i64_addr64_offset:2703; GFX12:       ; %bb.0: ; %entry2704; GFX12-NEXT:    s_clause 0x12705; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x342706; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x242707; GFX12-NEXT:    s_wait_kmcnt 0x02708; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 32709; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s32710; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]2711; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)2712; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s02713; GFX12-NEXT:    flat_atomic_max_u64 v[2:3], v[0:1] offset:32 scope:SCOPE_SE2714; GFX12-NEXT:    s_wait_storecnt_dscnt 0x02715; GFX12-NEXT:    global_inv scope:SCOPE_SE2716; GFX12-NEXT:    s_endpgm2717entry:2718  %ptr = getelementptr inbounds i64, ptr %out, i64 %index2719  %gep = getelementptr inbounds i64, ptr %ptr, i64 42720  %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !02721  ret void2722}2723 2724define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {2725; GFX7-LABEL: atomic_umax_i64_ret_addr64_offset:2726; GFX7:       ; %bb.0: ; %entry2727; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x92728; GFX7-NEXT:    s_waitcnt lgkmcnt(0)2729; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 32730; GFX7-NEXT:    s_add_u32 s0, s0, s62731; GFX7-NEXT:    s_addc_u32 s1, s1, s72732; GFX7-NEXT:    s_add_u32 s0, s0, 322733; GFX7-NEXT:    s_addc_u32 s1, s1, 02734; GFX7-NEXT:    v_mov_b32_e32 v0, s02735; GFX7-NEXT:    v_mov_b32_e32 v1, s12736; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]2737; GFX7-NEXT:    s_mov_b64 s[0:1], 02738; GFX7-NEXT:    v_mov_b32_e32 v4, s52739; GFX7-NEXT:    v_mov_b32_e32 v5, s42740; GFX7-NEXT:  .LBB35_1: ; %atomicrmw.start2741; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=12742; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2743; GFX7-NEXT:    v_mov_b32_e32 v9, v32744; GFX7-NEXT:    v_mov_b32_e32 v8, v22745; GFX7-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]2746; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc2747; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc2748; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc2749; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2750; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]2751; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]2752; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]2753; GFX7-NEXT:    s_cbranch_execnz .LBB35_12754; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end2755; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]2756; GFX7-NEXT:    v_mov_b32_e32 v0, s22757; GFX7-NEXT:    v_mov_b32_e32 v1, s32758; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]2759; GFX7-NEXT:    s_endpgm2760;2761; GFX8-LABEL: atomic_umax_i64_ret_addr64_offset:2762; GFX8:       ; %bb.0: ; %entry2763; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x242764; GFX8-NEXT:    s_waitcnt lgkmcnt(0)2765; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 32766; GFX8-NEXT:    s_add_u32 s0, s0, s62767; GFX8-NEXT:    s_addc_u32 s1, s1, s72768; GFX8-NEXT:    s_add_u32 s0, s0, 322769; GFX8-NEXT:    s_addc_u32 s1, s1, 02770; GFX8-NEXT:    v_mov_b32_e32 v0, s02771; GFX8-NEXT:    v_mov_b32_e32 v1, s12772; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]2773; GFX8-NEXT:    s_mov_b64 s[0:1], 02774; GFX8-NEXT:    v_mov_b32_e32 v4, s52775; GFX8-NEXT:    v_mov_b32_e32 v5, s42776; GFX8-NEXT:  .LBB35_1: ; %atomicrmw.start2777; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=12778; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2779; GFX8-NEXT:    v_mov_b32_e32 v9, v32780; GFX8-NEXT:    v_mov_b32_e32 v8, v22781; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]2782; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc2783; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc2784; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc2785; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2786; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]2787; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]2788; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]2789; GFX8-NEXT:    s_cbranch_execnz .LBB35_12790; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end2791; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]2792; GFX8-NEXT:    v_mov_b32_e32 v0, s22793; GFX8-NEXT:    v_mov_b32_e32 v1, s32794; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]2795; GFX8-NEXT:    s_endpgm2796;2797; GFX12-LABEL: atomic_umax_i64_ret_addr64_offset:2798; GFX12:       ; %bb.0: ; %entry2799; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x242800; GFX12-NEXT:    s_wait_kmcnt 0x02801; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 32802; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s52803; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]2804; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)2805; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s02806; GFX12-NEXT:    flat_atomic_max_u64 v[0:1], v[2:3], v[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_SE2807; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x02808; GFX12-NEXT:    global_inv scope:SCOPE_SE2809; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s32810; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]2811; GFX12-NEXT:    s_endpgm2812entry:2813  %ptr = getelementptr inbounds i64, ptr %out, i64 %index2814  %gep = getelementptr inbounds i64, ptr %ptr, i64 42815  %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !02816  store i64 %tmp0, ptr %out22817  ret void2818}2819 2820define amdgpu_kernel void @atomic_umax_i64(ptr %out, i64 %in) {2821; GFX7-LABEL: atomic_umax_i64:2822; GFX7:       ; %bb.0: ; %entry2823; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x92824; GFX7-NEXT:    s_mov_b64 s[4:5], 02825; GFX7-NEXT:    s_waitcnt lgkmcnt(0)2826; GFX7-NEXT:    v_mov_b32_e32 v0, s02827; GFX7-NEXT:    v_mov_b32_e32 v1, s12828; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]2829; GFX7-NEXT:    v_mov_b32_e32 v5, s12830; GFX7-NEXT:    v_mov_b32_e32 v6, s32831; GFX7-NEXT:    v_mov_b32_e32 v7, s22832; GFX7-NEXT:    v_mov_b32_e32 v4, s02833; GFX7-NEXT:  .LBB36_1: ; %atomicrmw.start2834; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=12835; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2836; GFX7-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]2837; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc2838; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc2839; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc2840; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2841; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]2842; GFX7-NEXT:    v_mov_b32_e32 v3, v12843; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]2844; GFX7-NEXT:    v_mov_b32_e32 v2, v02845; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]2846; GFX7-NEXT:    s_cbranch_execnz .LBB36_12847; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end2848; GFX7-NEXT:    s_endpgm2849;2850; GFX8-LABEL: atomic_umax_i64:2851; GFX8:       ; %bb.0: ; %entry2852; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242853; GFX8-NEXT:    s_mov_b64 s[4:5], 02854; GFX8-NEXT:    s_waitcnt lgkmcnt(0)2855; GFX8-NEXT:    v_mov_b32_e32 v0, s02856; GFX8-NEXT:    v_mov_b32_e32 v1, s12857; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]2858; GFX8-NEXT:    v_mov_b32_e32 v5, s12859; GFX8-NEXT:    v_mov_b32_e32 v6, s32860; GFX8-NEXT:    v_mov_b32_e32 v7, s22861; GFX8-NEXT:    v_mov_b32_e32 v4, s02862; GFX8-NEXT:  .LBB36_1: ; %atomicrmw.start2863; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=12864; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2865; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]2866; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc2867; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc2868; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc2869; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2870; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]2871; GFX8-NEXT:    v_mov_b32_e32 v3, v12872; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]2873; GFX8-NEXT:    v_mov_b32_e32 v2, v02874; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]2875; GFX8-NEXT:    s_cbranch_execnz .LBB36_12876; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end2877; GFX8-NEXT:    s_endpgm2878;2879; GFX12-LABEL: atomic_umax_i64:2880; GFX12:       ; %bb.0: ; %entry2881; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x242882; GFX12-NEXT:    s_wait_kmcnt 0x02883; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12884; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s32885; GFX12-NEXT:    flat_atomic_max_u64 v[0:1], v[2:3] scope:SCOPE_SE2886; GFX12-NEXT:    s_wait_storecnt_dscnt 0x02887; GFX12-NEXT:    global_inv scope:SCOPE_SE2888; GFX12-NEXT:    s_endpgm2889entry:2890  %tmp0 = atomicrmw volatile umax ptr %out, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !02891  ret void2892}2893 2894define amdgpu_kernel void @atomic_umax_i64_ret(ptr %out, ptr %out2, i64 %in) {2895; GFX7-LABEL: atomic_umax_i64_ret:2896; GFX7:       ; %bb.0: ; %entry2897; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x92898; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd2899; GFX7-NEXT:    s_mov_b64 s[6:7], 02900; GFX7-NEXT:    s_waitcnt lgkmcnt(0)2901; GFX7-NEXT:    v_mov_b32_e32 v0, s02902; GFX7-NEXT:    v_mov_b32_e32 v1, s12903; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]2904; GFX7-NEXT:    v_mov_b32_e32 v3, s12905; GFX7-NEXT:    v_mov_b32_e32 v4, s52906; GFX7-NEXT:    v_mov_b32_e32 v5, s42907; GFX7-NEXT:    v_mov_b32_e32 v2, s02908; GFX7-NEXT:  .LBB37_1: ; %atomicrmw.start2909; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=12910; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2911; GFX7-NEXT:    v_mov_b32_e32 v9, v12912; GFX7-NEXT:    v_mov_b32_e32 v8, v02913; GFX7-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]2914; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc2915; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc2916; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc2917; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2918; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]2919; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]2920; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]2921; GFX7-NEXT:    s_cbranch_execnz .LBB37_12922; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end2923; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]2924; GFX7-NEXT:    v_mov_b32_e32 v2, s22925; GFX7-NEXT:    v_mov_b32_e32 v3, s32926; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]2927; GFX7-NEXT:    s_endpgm2928;2929; GFX8-LABEL: atomic_umax_i64_ret:2930; GFX8:       ; %bb.0: ; %entry2931; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242932; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x342933; GFX8-NEXT:    s_mov_b64 s[6:7], 02934; GFX8-NEXT:    s_waitcnt lgkmcnt(0)2935; GFX8-NEXT:    v_mov_b32_e32 v0, s02936; GFX8-NEXT:    v_mov_b32_e32 v1, s12937; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]2938; GFX8-NEXT:    v_mov_b32_e32 v3, s12939; GFX8-NEXT:    v_mov_b32_e32 v4, s52940; GFX8-NEXT:    v_mov_b32_e32 v5, s42941; GFX8-NEXT:    v_mov_b32_e32 v2, s02942; GFX8-NEXT:  .LBB37_1: ; %atomicrmw.start2943; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=12944; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2945; GFX8-NEXT:    v_mov_b32_e32 v9, v12946; GFX8-NEXT:    v_mov_b32_e32 v8, v02947; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]2948; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc2949; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc2950; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc2951; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2952; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]2953; GFX8-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]2954; GFX8-NEXT:    s_andn2_b64 exec, exec, s[6:7]2955; GFX8-NEXT:    s_cbranch_execnz .LBB37_12956; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end2957; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]2958; GFX8-NEXT:    v_mov_b32_e32 v2, s22959; GFX8-NEXT:    v_mov_b32_e32 v3, s32960; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]2961; GFX8-NEXT:    s_endpgm2962;2963; GFX12-LABEL: atomic_umax_i64_ret:2964; GFX12:       ; %bb.0: ; %entry2965; GFX12-NEXT:    s_clause 0x12966; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x242967; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x342968; GFX12-NEXT:    s_wait_kmcnt 0x02969; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12970; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s52971; GFX12-NEXT:    flat_atomic_max_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE2972; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x02973; GFX12-NEXT:    global_inv scope:SCOPE_SE2974; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s32975; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]2976; GFX12-NEXT:    s_endpgm2977entry:2978  %tmp0 = atomicrmw volatile umax ptr %out, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !02979  store i64 %tmp0, ptr %out22980  ret void2981}2982 2983define amdgpu_kernel void @atomic_umax_i64_addr64(ptr %out, i64 %in, i64 %index) {2984; GFX7-LABEL: atomic_umax_i64_addr64:2985; GFX7:       ; %bb.0: ; %entry2986; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd2987; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x92988; GFX7-NEXT:    s_waitcnt lgkmcnt(0)2989; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 32990; GFX7-NEXT:    s_add_u32 s0, s0, s42991; GFX7-NEXT:    s_addc_u32 s1, s1, s52992; GFX7-NEXT:    v_mov_b32_e32 v5, s12993; GFX7-NEXT:    v_mov_b32_e32 v4, s02994; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]2995; GFX7-NEXT:    s_mov_b64 s[0:1], 02996; GFX7-NEXT:    v_mov_b32_e32 v6, s32997; GFX7-NEXT:    v_mov_b32_e32 v7, s22998; GFX7-NEXT:  .LBB38_1: ; %atomicrmw.start2999; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=13000; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3001; GFX7-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]3002; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc3003; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc3004; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc3005; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3006; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3007; GFX7-NEXT:    v_mov_b32_e32 v3, v13008; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]3009; GFX7-NEXT:    v_mov_b32_e32 v2, v03010; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]3011; GFX7-NEXT:    s_cbranch_execnz .LBB38_13012; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end3013; GFX7-NEXT:    s_endpgm3014;3015; GFX8-LABEL: atomic_umax_i64_addr64:3016; GFX8:       ; %bb.0: ; %entry3017; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x343018; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243019; GFX8-NEXT:    s_waitcnt lgkmcnt(0)3020; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 33021; GFX8-NEXT:    s_add_u32 s0, s0, s43022; GFX8-NEXT:    s_addc_u32 s1, s1, s53023; GFX8-NEXT:    v_mov_b32_e32 v5, s13024; GFX8-NEXT:    v_mov_b32_e32 v4, s03025; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]3026; GFX8-NEXT:    s_mov_b64 s[0:1], 03027; GFX8-NEXT:    v_mov_b32_e32 v6, s33028; GFX8-NEXT:    v_mov_b32_e32 v7, s23029; GFX8-NEXT:  .LBB38_1: ; %atomicrmw.start3030; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=13031; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3032; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]3033; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc3034; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc3035; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc3036; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3037; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3038; GFX8-NEXT:    v_mov_b32_e32 v3, v13039; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]3040; GFX8-NEXT:    v_mov_b32_e32 v2, v03041; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]3042; GFX8-NEXT:    s_cbranch_execnz .LBB38_13043; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end3044; GFX8-NEXT:    s_endpgm3045;3046; GFX12-LABEL: atomic_umax_i64_addr64:3047; GFX12:       ; %bb.0: ; %entry3048; GFX12-NEXT:    s_clause 0x13049; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x343050; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x243051; GFX12-NEXT:    s_wait_kmcnt 0x03052; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 33053; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s33054; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]3055; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)3056; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s03057; GFX12-NEXT:    flat_atomic_max_u64 v[2:3], v[0:1] scope:SCOPE_SE3058; GFX12-NEXT:    s_wait_storecnt_dscnt 0x03059; GFX12-NEXT:    global_inv scope:SCOPE_SE3060; GFX12-NEXT:    s_endpgm3061entry:3062  %ptr = getelementptr inbounds i64, ptr %out, i64 %index3063  %tmp0 = atomicrmw volatile umax ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !03064  ret void3065}3066 3067define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {3068; GFX7-LABEL: atomic_umax_i64_ret_addr64:3069; GFX7:       ; %bb.0: ; %entry3070; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x93071; GFX7-NEXT:    s_waitcnt lgkmcnt(0)3072; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 33073; GFX7-NEXT:    s_add_u32 s0, s0, s63074; GFX7-NEXT:    s_addc_u32 s1, s1, s73075; GFX7-NEXT:    v_mov_b32_e32 v0, s03076; GFX7-NEXT:    v_mov_b32_e32 v1, s13077; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]3078; GFX7-NEXT:    s_mov_b64 s[0:1], 03079; GFX7-NEXT:    v_mov_b32_e32 v4, s53080; GFX7-NEXT:    v_mov_b32_e32 v5, s43081; GFX7-NEXT:  .LBB39_1: ; %atomicrmw.start3082; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=13083; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3084; GFX7-NEXT:    v_mov_b32_e32 v9, v33085; GFX7-NEXT:    v_mov_b32_e32 v8, v23086; GFX7-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]3087; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc3088; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc3089; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc3090; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3091; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]3092; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]3093; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]3094; GFX7-NEXT:    s_cbranch_execnz .LBB39_13095; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end3096; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]3097; GFX7-NEXT:    v_mov_b32_e32 v0, s23098; GFX7-NEXT:    v_mov_b32_e32 v1, s33099; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]3100; GFX7-NEXT:    s_endpgm3101;3102; GFX8-LABEL: atomic_umax_i64_ret_addr64:3103; GFX8:       ; %bb.0: ; %entry3104; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x243105; GFX8-NEXT:    s_waitcnt lgkmcnt(0)3106; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 33107; GFX8-NEXT:    s_add_u32 s0, s0, s63108; GFX8-NEXT:    s_addc_u32 s1, s1, s73109; GFX8-NEXT:    v_mov_b32_e32 v0, s03110; GFX8-NEXT:    v_mov_b32_e32 v1, s13111; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]3112; GFX8-NEXT:    s_mov_b64 s[0:1], 03113; GFX8-NEXT:    v_mov_b32_e32 v4, s53114; GFX8-NEXT:    v_mov_b32_e32 v5, s43115; GFX8-NEXT:  .LBB39_1: ; %atomicrmw.start3116; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=13117; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3118; GFX8-NEXT:    v_mov_b32_e32 v9, v33119; GFX8-NEXT:    v_mov_b32_e32 v8, v23120; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]3121; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc3122; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc3123; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc3124; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3125; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]3126; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]3127; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]3128; GFX8-NEXT:    s_cbranch_execnz .LBB39_13129; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end3130; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]3131; GFX8-NEXT:    v_mov_b32_e32 v0, s23132; GFX8-NEXT:    v_mov_b32_e32 v1, s33133; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]3134; GFX8-NEXT:    s_endpgm3135;3136; GFX12-LABEL: atomic_umax_i64_ret_addr64:3137; GFX12:       ; %bb.0: ; %entry3138; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x243139; GFX12-NEXT:    s_wait_kmcnt 0x03140; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 33141; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s53142; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]3143; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)3144; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s03145; GFX12-NEXT:    flat_atomic_max_u64 v[0:1], v[2:3], v[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SE3146; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x03147; GFX12-NEXT:    global_inv scope:SCOPE_SE3148; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s33149; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]3150; GFX12-NEXT:    s_endpgm3151entry:3152  %ptr = getelementptr inbounds i64, ptr %out, i64 %index3153  %tmp0 = atomicrmw volatile umax ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !03154  store i64 %tmp0, ptr %out23155  ret void3156}3157 3158define amdgpu_kernel void @atomic_min_i64_offset(ptr %out, i64 %in) {3159; GFX7-LABEL: atomic_min_i64_offset:3160; GFX7:       ; %bb.0: ; %entry3161; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x93162; GFX7-NEXT:    s_waitcnt lgkmcnt(0)3163; GFX7-NEXT:    s_add_u32 s0, s0, 323164; GFX7-NEXT:    s_addc_u32 s1, s1, 03165; GFX7-NEXT:    v_mov_b32_e32 v5, s13166; GFX7-NEXT:    v_mov_b32_e32 v4, s03167; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]3168; GFX7-NEXT:    s_mov_b64 s[0:1], 03169; GFX7-NEXT:    v_mov_b32_e32 v6, s33170; GFX7-NEXT:    v_mov_b32_e32 v7, s23171; GFX7-NEXT:  .LBB40_1: ; %atomicrmw.start3172; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=13173; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3174; GFX7-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]3175; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc3176; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc3177; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc3178; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3179; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3180; GFX7-NEXT:    v_mov_b32_e32 v3, v13181; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]3182; GFX7-NEXT:    v_mov_b32_e32 v2, v03183; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]3184; GFX7-NEXT:    s_cbranch_execnz .LBB40_13185; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end3186; GFX7-NEXT:    s_endpgm3187;3188; GFX8-LABEL: atomic_min_i64_offset:3189; GFX8:       ; %bb.0: ; %entry3190; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243191; GFX8-NEXT:    s_waitcnt lgkmcnt(0)3192; GFX8-NEXT:    s_add_u32 s0, s0, 323193; GFX8-NEXT:    s_addc_u32 s1, s1, 03194; GFX8-NEXT:    v_mov_b32_e32 v5, s13195; GFX8-NEXT:    v_mov_b32_e32 v4, s03196; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]3197; GFX8-NEXT:    s_mov_b64 s[0:1], 03198; GFX8-NEXT:    v_mov_b32_e32 v6, s33199; GFX8-NEXT:    v_mov_b32_e32 v7, s23200; GFX8-NEXT:  .LBB40_1: ; %atomicrmw.start3201; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=13202; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3203; GFX8-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]3204; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc3205; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc3206; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc3207; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3208; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3209; GFX8-NEXT:    v_mov_b32_e32 v3, v13210; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]3211; GFX8-NEXT:    v_mov_b32_e32 v2, v03212; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]3213; GFX8-NEXT:    s_cbranch_execnz .LBB40_13214; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end3215; GFX8-NEXT:    s_endpgm3216;3217; GFX12-LABEL: atomic_min_i64_offset:3218; GFX12:       ; %bb.0: ; %entry3219; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x243220; GFX12-NEXT:    s_wait_kmcnt 0x03221; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13222; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s33223; GFX12-NEXT:    flat_atomic_min_i64 v[0:1], v[2:3] offset:32 scope:SCOPE_SE3224; GFX12-NEXT:    s_wait_storecnt_dscnt 0x03225; GFX12-NEXT:    global_inv scope:SCOPE_SE3226; GFX12-NEXT:    s_endpgm3227entry:3228  %gep = getelementptr inbounds i64, ptr %out, i64 43229  %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !03230  ret void3231}3232 3233define amdgpu_kernel void @atomic_min_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {3234; GFX7-LABEL: atomic_min_i64_ret_offset:3235; GFX7:       ; %bb.0: ; %entry3236; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x93237; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd3238; GFX7-NEXT:    s_waitcnt lgkmcnt(0)3239; GFX7-NEXT:    s_add_u32 s0, s0, 323240; GFX7-NEXT:    s_addc_u32 s1, s1, 03241; GFX7-NEXT:    v_mov_b32_e32 v0, s03242; GFX7-NEXT:    v_mov_b32_e32 v1, s13243; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]3244; GFX7-NEXT:    s_mov_b64 s[0:1], 03245; GFX7-NEXT:    v_mov_b32_e32 v4, s53246; GFX7-NEXT:    v_mov_b32_e32 v5, s43247; GFX7-NEXT:  .LBB41_1: ; %atomicrmw.start3248; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=13249; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3250; GFX7-NEXT:    v_mov_b32_e32 v9, v33251; GFX7-NEXT:    v_mov_b32_e32 v8, v23252; GFX7-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]3253; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc3254; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc3255; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc3256; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3257; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]3258; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]3259; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]3260; GFX7-NEXT:    s_cbranch_execnz .LBB41_13261; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end3262; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]3263; GFX7-NEXT:    v_mov_b32_e32 v0, s23264; GFX7-NEXT:    v_mov_b32_e32 v1, s33265; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]3266; GFX7-NEXT:    s_endpgm3267;3268; GFX8-LABEL: atomic_min_i64_ret_offset:3269; GFX8:       ; %bb.0: ; %entry3270; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243271; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x343272; GFX8-NEXT:    s_waitcnt lgkmcnt(0)3273; GFX8-NEXT:    s_add_u32 s0, s0, 323274; GFX8-NEXT:    s_addc_u32 s1, s1, 03275; GFX8-NEXT:    v_mov_b32_e32 v0, s03276; GFX8-NEXT:    v_mov_b32_e32 v1, s13277; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]3278; GFX8-NEXT:    s_mov_b64 s[0:1], 03279; GFX8-NEXT:    v_mov_b32_e32 v4, s53280; GFX8-NEXT:    v_mov_b32_e32 v5, s43281; GFX8-NEXT:  .LBB41_1: ; %atomicrmw.start3282; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=13283; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3284; GFX8-NEXT:    v_mov_b32_e32 v9, v33285; GFX8-NEXT:    v_mov_b32_e32 v8, v23286; GFX8-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]3287; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc3288; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc3289; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc3290; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3291; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]3292; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]3293; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]3294; GFX8-NEXT:    s_cbranch_execnz .LBB41_13295; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end3296; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]3297; GFX8-NEXT:    v_mov_b32_e32 v0, s23298; GFX8-NEXT:    v_mov_b32_e32 v1, s33299; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]3300; GFX8-NEXT:    s_endpgm3301;3302; GFX12-LABEL: atomic_min_i64_ret_offset:3303; GFX12:       ; %bb.0: ; %entry3304; GFX12-NEXT:    s_clause 0x13305; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x243306; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x343307; GFX12-NEXT:    s_wait_kmcnt 0x03308; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13309; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s53310; GFX12-NEXT:    flat_atomic_min_i64 v[0:1], v[0:1], v[2:3] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_SE3311; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x03312; GFX12-NEXT:    global_inv scope:SCOPE_SE3313; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s33314; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]3315; GFX12-NEXT:    s_endpgm3316entry:3317  %gep = getelementptr inbounds i64, ptr %out, i64 43318  %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !03319  store i64 %tmp0, ptr %out23320  ret void3321}3322 3323define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {3324; GFX7-LABEL: atomic_min_i64_addr64_offset:3325; GFX7:       ; %bb.0: ; %entry3326; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd3327; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x93328; GFX7-NEXT:    s_waitcnt lgkmcnt(0)3329; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 33330; GFX7-NEXT:    s_add_u32 s0, s0, s43331; GFX7-NEXT:    s_addc_u32 s1, s1, s53332; GFX7-NEXT:    s_add_u32 s0, s0, 323333; GFX7-NEXT:    s_addc_u32 s1, s1, 03334; GFX7-NEXT:    v_mov_b32_e32 v5, s13335; GFX7-NEXT:    v_mov_b32_e32 v4, s03336; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]3337; GFX7-NEXT:    s_mov_b64 s[0:1], 03338; GFX7-NEXT:    v_mov_b32_e32 v6, s33339; GFX7-NEXT:    v_mov_b32_e32 v7, s23340; GFX7-NEXT:  .LBB42_1: ; %atomicrmw.start3341; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=13342; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3343; GFX7-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]3344; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc3345; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc3346; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc3347; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3348; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3349; GFX7-NEXT:    v_mov_b32_e32 v3, v13350; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]3351; GFX7-NEXT:    v_mov_b32_e32 v2, v03352; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]3353; GFX7-NEXT:    s_cbranch_execnz .LBB42_13354; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end3355; GFX7-NEXT:    s_endpgm3356;3357; GFX8-LABEL: atomic_min_i64_addr64_offset:3358; GFX8:       ; %bb.0: ; %entry3359; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x343360; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243361; GFX8-NEXT:    s_waitcnt lgkmcnt(0)3362; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 33363; GFX8-NEXT:    s_add_u32 s0, s0, s43364; GFX8-NEXT:    s_addc_u32 s1, s1, s53365; GFX8-NEXT:    s_add_u32 s0, s0, 323366; GFX8-NEXT:    s_addc_u32 s1, s1, 03367; GFX8-NEXT:    v_mov_b32_e32 v5, s13368; GFX8-NEXT:    v_mov_b32_e32 v4, s03369; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]3370; GFX8-NEXT:    s_mov_b64 s[0:1], 03371; GFX8-NEXT:    v_mov_b32_e32 v6, s33372; GFX8-NEXT:    v_mov_b32_e32 v7, s23373; GFX8-NEXT:  .LBB42_1: ; %atomicrmw.start3374; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=13375; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3376; GFX8-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]3377; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc3378; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc3379; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc3380; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3381; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3382; GFX8-NEXT:    v_mov_b32_e32 v3, v13383; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]3384; GFX8-NEXT:    v_mov_b32_e32 v2, v03385; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]3386; GFX8-NEXT:    s_cbranch_execnz .LBB42_13387; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end3388; GFX8-NEXT:    s_endpgm3389;3390; GFX12-LABEL: atomic_min_i64_addr64_offset:3391; GFX12:       ; %bb.0: ; %entry3392; GFX12-NEXT:    s_clause 0x13393; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x343394; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x243395; GFX12-NEXT:    s_wait_kmcnt 0x03396; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 33397; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s33398; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]3399; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)3400; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s03401; GFX12-NEXT:    flat_atomic_min_i64 v[2:3], v[0:1] offset:32 scope:SCOPE_SE3402; GFX12-NEXT:    s_wait_storecnt_dscnt 0x03403; GFX12-NEXT:    global_inv scope:SCOPE_SE3404; GFX12-NEXT:    s_endpgm3405entry:3406  %ptr = getelementptr inbounds i64, ptr %out, i64 %index3407  %gep = getelementptr inbounds i64, ptr %ptr, i64 43408  %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !03409  ret void3410}3411 3412define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {3413; GFX7-LABEL: atomic_min_i64_ret_addr64_offset:3414; GFX7:       ; %bb.0: ; %entry3415; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x93416; GFX7-NEXT:    s_waitcnt lgkmcnt(0)3417; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 33418; GFX7-NEXT:    s_add_u32 s0, s0, s63419; GFX7-NEXT:    s_addc_u32 s1, s1, s73420; GFX7-NEXT:    s_add_u32 s0, s0, 323421; GFX7-NEXT:    s_addc_u32 s1, s1, 03422; GFX7-NEXT:    v_mov_b32_e32 v0, s03423; GFX7-NEXT:    v_mov_b32_e32 v1, s13424; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]3425; GFX7-NEXT:    s_mov_b64 s[0:1], 03426; GFX7-NEXT:    v_mov_b32_e32 v4, s53427; GFX7-NEXT:    v_mov_b32_e32 v5, s43428; GFX7-NEXT:  .LBB43_1: ; %atomicrmw.start3429; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=13430; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3431; GFX7-NEXT:    v_mov_b32_e32 v9, v33432; GFX7-NEXT:    v_mov_b32_e32 v8, v23433; GFX7-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]3434; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc3435; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc3436; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc3437; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3438; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]3439; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]3440; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]3441; GFX7-NEXT:    s_cbranch_execnz .LBB43_13442; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end3443; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]3444; GFX7-NEXT:    v_mov_b32_e32 v0, s23445; GFX7-NEXT:    v_mov_b32_e32 v1, s33446; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]3447; GFX7-NEXT:    s_endpgm3448;3449; GFX8-LABEL: atomic_min_i64_ret_addr64_offset:3450; GFX8:       ; %bb.0: ; %entry3451; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x243452; GFX8-NEXT:    s_waitcnt lgkmcnt(0)3453; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 33454; GFX8-NEXT:    s_add_u32 s0, s0, s63455; GFX8-NEXT:    s_addc_u32 s1, s1, s73456; GFX8-NEXT:    s_add_u32 s0, s0, 323457; GFX8-NEXT:    s_addc_u32 s1, s1, 03458; GFX8-NEXT:    v_mov_b32_e32 v0, s03459; GFX8-NEXT:    v_mov_b32_e32 v1, s13460; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]3461; GFX8-NEXT:    s_mov_b64 s[0:1], 03462; GFX8-NEXT:    v_mov_b32_e32 v4, s53463; GFX8-NEXT:    v_mov_b32_e32 v5, s43464; GFX8-NEXT:  .LBB43_1: ; %atomicrmw.start3465; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=13466; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3467; GFX8-NEXT:    v_mov_b32_e32 v9, v33468; GFX8-NEXT:    v_mov_b32_e32 v8, v23469; GFX8-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]3470; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc3471; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc3472; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc3473; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3474; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]3475; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]3476; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]3477; GFX8-NEXT:    s_cbranch_execnz .LBB43_13478; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end3479; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]3480; GFX8-NEXT:    v_mov_b32_e32 v0, s23481; GFX8-NEXT:    v_mov_b32_e32 v1, s33482; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]3483; GFX8-NEXT:    s_endpgm3484;3485; GFX12-LABEL: atomic_min_i64_ret_addr64_offset:3486; GFX12:       ; %bb.0: ; %entry3487; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x243488; GFX12-NEXT:    s_wait_kmcnt 0x03489; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 33490; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s53491; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]3492; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)3493; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s03494; GFX12-NEXT:    flat_atomic_min_i64 v[0:1], v[2:3], v[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_SE3495; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x03496; GFX12-NEXT:    global_inv scope:SCOPE_SE3497; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s33498; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]3499; GFX12-NEXT:    s_endpgm3500entry:3501  %ptr = getelementptr inbounds i64, ptr %out, i64 %index3502  %gep = getelementptr inbounds i64, ptr %ptr, i64 43503  %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !03504  store i64 %tmp0, ptr %out23505  ret void3506}3507 3508define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {3509; GFX7-LABEL: atomic_min_i64:3510; GFX7:       ; %bb.0: ; %entry3511; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x93512; GFX7-NEXT:    s_mov_b64 s[4:5], 03513; GFX7-NEXT:    s_waitcnt lgkmcnt(0)3514; GFX7-NEXT:    v_mov_b32_e32 v0, s03515; GFX7-NEXT:    v_mov_b32_e32 v1, s13516; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]3517; GFX7-NEXT:    v_mov_b32_e32 v5, s13518; GFX7-NEXT:    v_mov_b32_e32 v6, s33519; GFX7-NEXT:    v_mov_b32_e32 v7, s23520; GFX7-NEXT:    v_mov_b32_e32 v4, s03521; GFX7-NEXT:  .LBB44_1: ; %atomicrmw.start3522; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=13523; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3524; GFX7-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]3525; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc3526; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc3527; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc3528; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3529; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3530; GFX7-NEXT:    v_mov_b32_e32 v3, v13531; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]3532; GFX7-NEXT:    v_mov_b32_e32 v2, v03533; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]3534; GFX7-NEXT:    s_cbranch_execnz .LBB44_13535; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end3536; GFX7-NEXT:    s_endpgm3537;3538; GFX8-LABEL: atomic_min_i64:3539; GFX8:       ; %bb.0: ; %entry3540; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243541; GFX8-NEXT:    s_mov_b64 s[4:5], 03542; GFX8-NEXT:    s_waitcnt lgkmcnt(0)3543; GFX8-NEXT:    v_mov_b32_e32 v0, s03544; GFX8-NEXT:    v_mov_b32_e32 v1, s13545; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]3546; GFX8-NEXT:    v_mov_b32_e32 v5, s13547; GFX8-NEXT:    v_mov_b32_e32 v6, s33548; GFX8-NEXT:    v_mov_b32_e32 v7, s23549; GFX8-NEXT:    v_mov_b32_e32 v4, s03550; GFX8-NEXT:  .LBB44_1: ; %atomicrmw.start3551; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=13552; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3553; GFX8-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]3554; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc3555; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc3556; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc3557; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3558; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3559; GFX8-NEXT:    v_mov_b32_e32 v3, v13560; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]3561; GFX8-NEXT:    v_mov_b32_e32 v2, v03562; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]3563; GFX8-NEXT:    s_cbranch_execnz .LBB44_13564; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end3565; GFX8-NEXT:    s_endpgm3566;3567; GFX12-LABEL: atomic_min_i64:3568; GFX12:       ; %bb.0: ; %entry3569; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x243570; GFX12-NEXT:    s_wait_kmcnt 0x03571; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13572; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s33573; GFX12-NEXT:    flat_atomic_min_i64 v[0:1], v[2:3] scope:SCOPE_SE3574; GFX12-NEXT:    s_wait_storecnt_dscnt 0x03575; GFX12-NEXT:    global_inv scope:SCOPE_SE3576; GFX12-NEXT:    s_endpgm3577entry:3578  %tmp0 = atomicrmw volatile min ptr %out, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !03579  ret void3580}3581 3582define amdgpu_kernel void @atomic_min_i64_ret(ptr %out, ptr %out2, i64 %in) {3583; GFX7-LABEL: atomic_min_i64_ret:3584; GFX7:       ; %bb.0: ; %entry3585; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x93586; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd3587; GFX7-NEXT:    s_mov_b64 s[6:7], 03588; GFX7-NEXT:    s_waitcnt lgkmcnt(0)3589; GFX7-NEXT:    v_mov_b32_e32 v0, s03590; GFX7-NEXT:    v_mov_b32_e32 v1, s13591; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]3592; GFX7-NEXT:    v_mov_b32_e32 v3, s13593; GFX7-NEXT:    v_mov_b32_e32 v4, s53594; GFX7-NEXT:    v_mov_b32_e32 v5, s43595; GFX7-NEXT:    v_mov_b32_e32 v2, s03596; GFX7-NEXT:  .LBB45_1: ; %atomicrmw.start3597; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=13598; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3599; GFX7-NEXT:    v_mov_b32_e32 v9, v13600; GFX7-NEXT:    v_mov_b32_e32 v8, v03601; GFX7-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]3602; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc3603; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc3604; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc3605; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3606; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]3607; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]3608; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]3609; GFX7-NEXT:    s_cbranch_execnz .LBB45_13610; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end3611; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]3612; GFX7-NEXT:    v_mov_b32_e32 v2, s23613; GFX7-NEXT:    v_mov_b32_e32 v3, s33614; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]3615; GFX7-NEXT:    s_endpgm3616;3617; GFX8-LABEL: atomic_min_i64_ret:3618; GFX8:       ; %bb.0: ; %entry3619; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243620; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x343621; GFX8-NEXT:    s_mov_b64 s[6:7], 03622; GFX8-NEXT:    s_waitcnt lgkmcnt(0)3623; GFX8-NEXT:    v_mov_b32_e32 v0, s03624; GFX8-NEXT:    v_mov_b32_e32 v1, s13625; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]3626; GFX8-NEXT:    v_mov_b32_e32 v3, s13627; GFX8-NEXT:    v_mov_b32_e32 v4, s53628; GFX8-NEXT:    v_mov_b32_e32 v5, s43629; GFX8-NEXT:    v_mov_b32_e32 v2, s03630; GFX8-NEXT:  .LBB45_1: ; %atomicrmw.start3631; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=13632; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3633; GFX8-NEXT:    v_mov_b32_e32 v9, v13634; GFX8-NEXT:    v_mov_b32_e32 v8, v03635; GFX8-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]3636; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc3637; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc3638; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc3639; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3640; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]3641; GFX8-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]3642; GFX8-NEXT:    s_andn2_b64 exec, exec, s[6:7]3643; GFX8-NEXT:    s_cbranch_execnz .LBB45_13644; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end3645; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]3646; GFX8-NEXT:    v_mov_b32_e32 v2, s23647; GFX8-NEXT:    v_mov_b32_e32 v3, s33648; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]3649; GFX8-NEXT:    s_endpgm3650;3651; GFX12-LABEL: atomic_min_i64_ret:3652; GFX12:       ; %bb.0: ; %entry3653; GFX12-NEXT:    s_clause 0x13654; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x243655; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x343656; GFX12-NEXT:    s_wait_kmcnt 0x03657; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13658; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s53659; GFX12-NEXT:    flat_atomic_min_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE3660; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x03661; GFX12-NEXT:    global_inv scope:SCOPE_SE3662; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s33663; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]3664; GFX12-NEXT:    s_endpgm3665entry:3666  %tmp0 = atomicrmw volatile min ptr %out, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !03667  store i64 %tmp0, ptr %out23668  ret void3669}3670 3671define amdgpu_kernel void @atomic_min_i64_addr64(ptr %out, i64 %in, i64 %index) {3672; GFX7-LABEL: atomic_min_i64_addr64:3673; GFX7:       ; %bb.0: ; %entry3674; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd3675; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x93676; GFX7-NEXT:    s_waitcnt lgkmcnt(0)3677; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 33678; GFX7-NEXT:    s_add_u32 s0, s0, s43679; GFX7-NEXT:    s_addc_u32 s1, s1, s53680; GFX7-NEXT:    v_mov_b32_e32 v5, s13681; GFX7-NEXT:    v_mov_b32_e32 v4, s03682; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]3683; GFX7-NEXT:    s_mov_b64 s[0:1], 03684; GFX7-NEXT:    v_mov_b32_e32 v6, s33685; GFX7-NEXT:    v_mov_b32_e32 v7, s23686; GFX7-NEXT:  .LBB46_1: ; %atomicrmw.start3687; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=13688; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3689; GFX7-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]3690; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc3691; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc3692; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc3693; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3694; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3695; GFX7-NEXT:    v_mov_b32_e32 v3, v13696; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]3697; GFX7-NEXT:    v_mov_b32_e32 v2, v03698; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]3699; GFX7-NEXT:    s_cbranch_execnz .LBB46_13700; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end3701; GFX7-NEXT:    s_endpgm3702;3703; GFX8-LABEL: atomic_min_i64_addr64:3704; GFX8:       ; %bb.0: ; %entry3705; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x343706; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243707; GFX8-NEXT:    s_waitcnt lgkmcnt(0)3708; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 33709; GFX8-NEXT:    s_add_u32 s0, s0, s43710; GFX8-NEXT:    s_addc_u32 s1, s1, s53711; GFX8-NEXT:    v_mov_b32_e32 v5, s13712; GFX8-NEXT:    v_mov_b32_e32 v4, s03713; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]3714; GFX8-NEXT:    s_mov_b64 s[0:1], 03715; GFX8-NEXT:    v_mov_b32_e32 v6, s33716; GFX8-NEXT:    v_mov_b32_e32 v7, s23717; GFX8-NEXT:  .LBB46_1: ; %atomicrmw.start3718; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=13719; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3720; GFX8-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]3721; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc3722; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc3723; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc3724; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3725; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3726; GFX8-NEXT:    v_mov_b32_e32 v3, v13727; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]3728; GFX8-NEXT:    v_mov_b32_e32 v2, v03729; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]3730; GFX8-NEXT:    s_cbranch_execnz .LBB46_13731; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end3732; GFX8-NEXT:    s_endpgm3733;3734; GFX12-LABEL: atomic_min_i64_addr64:3735; GFX12:       ; %bb.0: ; %entry3736; GFX12-NEXT:    s_clause 0x13737; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x343738; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x243739; GFX12-NEXT:    s_wait_kmcnt 0x03740; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 33741; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s33742; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]3743; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)3744; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s03745; GFX12-NEXT:    flat_atomic_min_i64 v[2:3], v[0:1] scope:SCOPE_SE3746; GFX12-NEXT:    s_wait_storecnt_dscnt 0x03747; GFX12-NEXT:    global_inv scope:SCOPE_SE3748; GFX12-NEXT:    s_endpgm3749entry:3750  %ptr = getelementptr inbounds i64, ptr %out, i64 %index3751  %tmp0 = atomicrmw volatile min ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !03752  ret void3753}3754 3755define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {3756; GFX7-LABEL: atomic_min_i64_ret_addr64:3757; GFX7:       ; %bb.0: ; %entry3758; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x93759; GFX7-NEXT:    s_waitcnt lgkmcnt(0)3760; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 33761; GFX7-NEXT:    s_add_u32 s0, s0, s63762; GFX7-NEXT:    s_addc_u32 s1, s1, s73763; GFX7-NEXT:    v_mov_b32_e32 v0, s03764; GFX7-NEXT:    v_mov_b32_e32 v1, s13765; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]3766; GFX7-NEXT:    s_mov_b64 s[0:1], 03767; GFX7-NEXT:    v_mov_b32_e32 v4, s53768; GFX7-NEXT:    v_mov_b32_e32 v5, s43769; GFX7-NEXT:  .LBB47_1: ; %atomicrmw.start3770; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=13771; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3772; GFX7-NEXT:    v_mov_b32_e32 v9, v33773; GFX7-NEXT:    v_mov_b32_e32 v8, v23774; GFX7-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]3775; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc3776; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc3777; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc3778; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3779; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]3780; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]3781; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]3782; GFX7-NEXT:    s_cbranch_execnz .LBB47_13783; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end3784; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]3785; GFX7-NEXT:    v_mov_b32_e32 v0, s23786; GFX7-NEXT:    v_mov_b32_e32 v1, s33787; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]3788; GFX7-NEXT:    s_endpgm3789;3790; GFX8-LABEL: atomic_min_i64_ret_addr64:3791; GFX8:       ; %bb.0: ; %entry3792; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x243793; GFX8-NEXT:    s_waitcnt lgkmcnt(0)3794; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 33795; GFX8-NEXT:    s_add_u32 s0, s0, s63796; GFX8-NEXT:    s_addc_u32 s1, s1, s73797; GFX8-NEXT:    v_mov_b32_e32 v0, s03798; GFX8-NEXT:    v_mov_b32_e32 v1, s13799; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]3800; GFX8-NEXT:    s_mov_b64 s[0:1], 03801; GFX8-NEXT:    v_mov_b32_e32 v4, s53802; GFX8-NEXT:    v_mov_b32_e32 v5, s43803; GFX8-NEXT:  .LBB47_1: ; %atomicrmw.start3804; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=13805; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3806; GFX8-NEXT:    v_mov_b32_e32 v9, v33807; GFX8-NEXT:    v_mov_b32_e32 v8, v23808; GFX8-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]3809; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc3810; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc3811; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc3812; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3813; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]3814; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]3815; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]3816; GFX8-NEXT:    s_cbranch_execnz .LBB47_13817; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end3818; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]3819; GFX8-NEXT:    v_mov_b32_e32 v0, s23820; GFX8-NEXT:    v_mov_b32_e32 v1, s33821; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]3822; GFX8-NEXT:    s_endpgm3823;3824; GFX12-LABEL: atomic_min_i64_ret_addr64:3825; GFX12:       ; %bb.0: ; %entry3826; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x243827; GFX12-NEXT:    s_wait_kmcnt 0x03828; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 33829; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s53830; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]3831; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)3832; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s03833; GFX12-NEXT:    flat_atomic_min_i64 v[0:1], v[2:3], v[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SE3834; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x03835; GFX12-NEXT:    global_inv scope:SCOPE_SE3836; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s33837; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]3838; GFX12-NEXT:    s_endpgm3839entry:3840  %ptr = getelementptr inbounds i64, ptr %out, i64 %index3841  %tmp0 = atomicrmw volatile min ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !03842  store i64 %tmp0, ptr %out23843  ret void3844}3845 3846define amdgpu_kernel void @atomic_umin_i64_offset(ptr %out, i64 %in) {3847; GFX7-LABEL: atomic_umin_i64_offset:3848; GFX7:       ; %bb.0: ; %entry3849; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x93850; GFX7-NEXT:    s_waitcnt lgkmcnt(0)3851; GFX7-NEXT:    s_add_u32 s0, s0, 323852; GFX7-NEXT:    s_addc_u32 s1, s1, 03853; GFX7-NEXT:    v_mov_b32_e32 v5, s13854; GFX7-NEXT:    v_mov_b32_e32 v4, s03855; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]3856; GFX7-NEXT:    s_mov_b64 s[0:1], 03857; GFX7-NEXT:    v_mov_b32_e32 v6, s33858; GFX7-NEXT:    v_mov_b32_e32 v7, s23859; GFX7-NEXT:  .LBB48_1: ; %atomicrmw.start3860; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=13861; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3862; GFX7-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]3863; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc3864; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc3865; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc3866; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3867; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3868; GFX7-NEXT:    v_mov_b32_e32 v3, v13869; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]3870; GFX7-NEXT:    v_mov_b32_e32 v2, v03871; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]3872; GFX7-NEXT:    s_cbranch_execnz .LBB48_13873; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end3874; GFX7-NEXT:    s_endpgm3875;3876; GFX8-LABEL: atomic_umin_i64_offset:3877; GFX8:       ; %bb.0: ; %entry3878; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243879; GFX8-NEXT:    s_waitcnt lgkmcnt(0)3880; GFX8-NEXT:    s_add_u32 s0, s0, 323881; GFX8-NEXT:    s_addc_u32 s1, s1, 03882; GFX8-NEXT:    v_mov_b32_e32 v5, s13883; GFX8-NEXT:    v_mov_b32_e32 v4, s03884; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]3885; GFX8-NEXT:    s_mov_b64 s[0:1], 03886; GFX8-NEXT:    v_mov_b32_e32 v6, s33887; GFX8-NEXT:    v_mov_b32_e32 v7, s23888; GFX8-NEXT:  .LBB48_1: ; %atomicrmw.start3889; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=13890; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3891; GFX8-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]3892; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc3893; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc3894; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc3895; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3896; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3897; GFX8-NEXT:    v_mov_b32_e32 v3, v13898; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]3899; GFX8-NEXT:    v_mov_b32_e32 v2, v03900; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]3901; GFX8-NEXT:    s_cbranch_execnz .LBB48_13902; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end3903; GFX8-NEXT:    s_endpgm3904;3905; GFX12-LABEL: atomic_umin_i64_offset:3906; GFX12:       ; %bb.0: ; %entry3907; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x243908; GFX12-NEXT:    s_wait_kmcnt 0x03909; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13910; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s33911; GFX12-NEXT:    flat_atomic_min_u64 v[0:1], v[2:3] offset:32 scope:SCOPE_SE3912; GFX12-NEXT:    s_wait_storecnt_dscnt 0x03913; GFX12-NEXT:    global_inv scope:SCOPE_SE3914; GFX12-NEXT:    s_endpgm3915entry:3916  %gep = getelementptr inbounds i64, ptr %out, i64 43917  %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !03918  ret void3919}3920 3921define amdgpu_kernel void @atomic_umin_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {3922; GFX7-LABEL: atomic_umin_i64_ret_offset:3923; GFX7:       ; %bb.0: ; %entry3924; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x93925; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd3926; GFX7-NEXT:    s_waitcnt lgkmcnt(0)3927; GFX7-NEXT:    s_add_u32 s0, s0, 323928; GFX7-NEXT:    s_addc_u32 s1, s1, 03929; GFX7-NEXT:    v_mov_b32_e32 v0, s03930; GFX7-NEXT:    v_mov_b32_e32 v1, s13931; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]3932; GFX7-NEXT:    s_mov_b64 s[0:1], 03933; GFX7-NEXT:    v_mov_b32_e32 v4, s53934; GFX7-NEXT:    v_mov_b32_e32 v5, s43935; GFX7-NEXT:  .LBB49_1: ; %atomicrmw.start3936; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=13937; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3938; GFX7-NEXT:    v_mov_b32_e32 v9, v33939; GFX7-NEXT:    v_mov_b32_e32 v8, v23940; GFX7-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]3941; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc3942; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc3943; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc3944; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3945; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]3946; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]3947; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]3948; GFX7-NEXT:    s_cbranch_execnz .LBB49_13949; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end3950; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]3951; GFX7-NEXT:    v_mov_b32_e32 v0, s23952; GFX7-NEXT:    v_mov_b32_e32 v1, s33953; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]3954; GFX7-NEXT:    s_endpgm3955;3956; GFX8-LABEL: atomic_umin_i64_ret_offset:3957; GFX8:       ; %bb.0: ; %entry3958; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243959; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x343960; GFX8-NEXT:    s_waitcnt lgkmcnt(0)3961; GFX8-NEXT:    s_add_u32 s0, s0, 323962; GFX8-NEXT:    s_addc_u32 s1, s1, 03963; GFX8-NEXT:    v_mov_b32_e32 v0, s03964; GFX8-NEXT:    v_mov_b32_e32 v1, s13965; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]3966; GFX8-NEXT:    s_mov_b64 s[0:1], 03967; GFX8-NEXT:    v_mov_b32_e32 v4, s53968; GFX8-NEXT:    v_mov_b32_e32 v5, s43969; GFX8-NEXT:  .LBB49_1: ; %atomicrmw.start3970; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=13971; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3972; GFX8-NEXT:    v_mov_b32_e32 v9, v33973; GFX8-NEXT:    v_mov_b32_e32 v8, v23974; GFX8-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]3975; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc3976; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc3977; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc3978; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3979; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]3980; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]3981; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]3982; GFX8-NEXT:    s_cbranch_execnz .LBB49_13983; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end3984; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]3985; GFX8-NEXT:    v_mov_b32_e32 v0, s23986; GFX8-NEXT:    v_mov_b32_e32 v1, s33987; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]3988; GFX8-NEXT:    s_endpgm3989;3990; GFX12-LABEL: atomic_umin_i64_ret_offset:3991; GFX12:       ; %bb.0: ; %entry3992; GFX12-NEXT:    s_clause 0x13993; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x243994; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x343995; GFX12-NEXT:    s_wait_kmcnt 0x03996; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13997; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s53998; GFX12-NEXT:    flat_atomic_min_u64 v[0:1], v[0:1], v[2:3] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_SE3999; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x04000; GFX12-NEXT:    global_inv scope:SCOPE_SE4001; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s34002; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]4003; GFX12-NEXT:    s_endpgm4004entry:4005  %gep = getelementptr inbounds i64, ptr %out, i64 44006  %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !04007  store i64 %tmp0, ptr %out24008  ret void4009}4010 4011define amdgpu_kernel void @atomic_umin_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {4012; GFX7-LABEL: atomic_umin_i64_addr64_offset:4013; GFX7:       ; %bb.0: ; %entry4014; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd4015; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x94016; GFX7-NEXT:    s_waitcnt lgkmcnt(0)4017; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 34018; GFX7-NEXT:    s_add_u32 s0, s0, s44019; GFX7-NEXT:    s_addc_u32 s1, s1, s54020; GFX7-NEXT:    s_add_u32 s0, s0, 324021; GFX7-NEXT:    s_addc_u32 s1, s1, 04022; GFX7-NEXT:    v_mov_b32_e32 v5, s14023; GFX7-NEXT:    v_mov_b32_e32 v4, s04024; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]4025; GFX7-NEXT:    s_mov_b64 s[0:1], 04026; GFX7-NEXT:    v_mov_b32_e32 v6, s34027; GFX7-NEXT:    v_mov_b32_e32 v7, s24028; GFX7-NEXT:  .LBB50_1: ; %atomicrmw.start4029; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=14030; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4031; GFX7-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]4032; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc4033; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc4034; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4035; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4036; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4037; GFX7-NEXT:    v_mov_b32_e32 v3, v14038; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]4039; GFX7-NEXT:    v_mov_b32_e32 v2, v04040; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]4041; GFX7-NEXT:    s_cbranch_execnz .LBB50_14042; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end4043; GFX7-NEXT:    s_endpgm4044;4045; GFX8-LABEL: atomic_umin_i64_addr64_offset:4046; GFX8:       ; %bb.0: ; %entry4047; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x344048; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244049; GFX8-NEXT:    s_waitcnt lgkmcnt(0)4050; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 34051; GFX8-NEXT:    s_add_u32 s0, s0, s44052; GFX8-NEXT:    s_addc_u32 s1, s1, s54053; GFX8-NEXT:    s_add_u32 s0, s0, 324054; GFX8-NEXT:    s_addc_u32 s1, s1, 04055; GFX8-NEXT:    v_mov_b32_e32 v5, s14056; GFX8-NEXT:    v_mov_b32_e32 v4, s04057; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]4058; GFX8-NEXT:    s_mov_b64 s[0:1], 04059; GFX8-NEXT:    v_mov_b32_e32 v6, s34060; GFX8-NEXT:    v_mov_b32_e32 v7, s24061; GFX8-NEXT:  .LBB50_1: ; %atomicrmw.start4062; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=14063; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4064; GFX8-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]4065; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc4066; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc4067; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4068; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4069; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4070; GFX8-NEXT:    v_mov_b32_e32 v3, v14071; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]4072; GFX8-NEXT:    v_mov_b32_e32 v2, v04073; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]4074; GFX8-NEXT:    s_cbranch_execnz .LBB50_14075; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end4076; GFX8-NEXT:    s_endpgm4077;4078; GFX12-LABEL: atomic_umin_i64_addr64_offset:4079; GFX12:       ; %bb.0: ; %entry4080; GFX12-NEXT:    s_clause 0x14081; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x344082; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x244083; GFX12-NEXT:    s_wait_kmcnt 0x04084; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 34085; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s34086; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]4087; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)4088; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s04089; GFX12-NEXT:    flat_atomic_min_u64 v[2:3], v[0:1] offset:32 scope:SCOPE_SE4090; GFX12-NEXT:    s_wait_storecnt_dscnt 0x04091; GFX12-NEXT:    global_inv scope:SCOPE_SE4092; GFX12-NEXT:    s_endpgm4093entry:4094  %ptr = getelementptr inbounds i64, ptr %out, i64 %index4095  %gep = getelementptr inbounds i64, ptr %ptr, i64 44096  %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !04097  ret void4098}4099 4100define amdgpu_kernel void @atomic_umin_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {4101; GFX7-LABEL: atomic_umin_i64_ret_addr64_offset:4102; GFX7:       ; %bb.0: ; %entry4103; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x94104; GFX7-NEXT:    s_waitcnt lgkmcnt(0)4105; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 34106; GFX7-NEXT:    s_add_u32 s0, s0, s64107; GFX7-NEXT:    s_addc_u32 s1, s1, s74108; GFX7-NEXT:    s_add_u32 s0, s0, 324109; GFX7-NEXT:    s_addc_u32 s1, s1, 04110; GFX7-NEXT:    v_mov_b32_e32 v0, s04111; GFX7-NEXT:    v_mov_b32_e32 v1, s14112; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]4113; GFX7-NEXT:    s_mov_b64 s[0:1], 04114; GFX7-NEXT:    v_mov_b32_e32 v4, s54115; GFX7-NEXT:    v_mov_b32_e32 v5, s44116; GFX7-NEXT:  .LBB51_1: ; %atomicrmw.start4117; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=14118; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4119; GFX7-NEXT:    v_mov_b32_e32 v9, v34120; GFX7-NEXT:    v_mov_b32_e32 v8, v24121; GFX7-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]4122; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc4123; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc4124; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc4125; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4126; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]4127; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]4128; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]4129; GFX7-NEXT:    s_cbranch_execnz .LBB51_14130; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end4131; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]4132; GFX7-NEXT:    v_mov_b32_e32 v0, s24133; GFX7-NEXT:    v_mov_b32_e32 v1, s34134; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]4135; GFX7-NEXT:    s_endpgm4136;4137; GFX8-LABEL: atomic_umin_i64_ret_addr64_offset:4138; GFX8:       ; %bb.0: ; %entry4139; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x244140; GFX8-NEXT:    s_waitcnt lgkmcnt(0)4141; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 34142; GFX8-NEXT:    s_add_u32 s0, s0, s64143; GFX8-NEXT:    s_addc_u32 s1, s1, s74144; GFX8-NEXT:    s_add_u32 s0, s0, 324145; GFX8-NEXT:    s_addc_u32 s1, s1, 04146; GFX8-NEXT:    v_mov_b32_e32 v0, s04147; GFX8-NEXT:    v_mov_b32_e32 v1, s14148; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]4149; GFX8-NEXT:    s_mov_b64 s[0:1], 04150; GFX8-NEXT:    v_mov_b32_e32 v4, s54151; GFX8-NEXT:    v_mov_b32_e32 v5, s44152; GFX8-NEXT:  .LBB51_1: ; %atomicrmw.start4153; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=14154; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4155; GFX8-NEXT:    v_mov_b32_e32 v9, v34156; GFX8-NEXT:    v_mov_b32_e32 v8, v24157; GFX8-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]4158; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc4159; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc4160; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc4161; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4162; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]4163; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]4164; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]4165; GFX8-NEXT:    s_cbranch_execnz .LBB51_14166; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end4167; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]4168; GFX8-NEXT:    v_mov_b32_e32 v0, s24169; GFX8-NEXT:    v_mov_b32_e32 v1, s34170; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]4171; GFX8-NEXT:    s_endpgm4172;4173; GFX12-LABEL: atomic_umin_i64_ret_addr64_offset:4174; GFX12:       ; %bb.0: ; %entry4175; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x244176; GFX12-NEXT:    s_wait_kmcnt 0x04177; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 34178; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s54179; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]4180; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)4181; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s04182; GFX12-NEXT:    flat_atomic_min_u64 v[0:1], v[2:3], v[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_SE4183; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x04184; GFX12-NEXT:    global_inv scope:SCOPE_SE4185; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s34186; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]4187; GFX12-NEXT:    s_endpgm4188entry:4189  %ptr = getelementptr inbounds i64, ptr %out, i64 %index4190  %gep = getelementptr inbounds i64, ptr %ptr, i64 44191  %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !04192  store i64 %tmp0, ptr %out24193  ret void4194}4195 4196define amdgpu_kernel void @atomic_umin_i64(ptr %out, i64 %in) {4197; GFX7-LABEL: atomic_umin_i64:4198; GFX7:       ; %bb.0: ; %entry4199; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x94200; GFX7-NEXT:    s_mov_b64 s[4:5], 04201; GFX7-NEXT:    s_waitcnt lgkmcnt(0)4202; GFX7-NEXT:    v_mov_b32_e32 v0, s04203; GFX7-NEXT:    v_mov_b32_e32 v1, s14204; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]4205; GFX7-NEXT:    v_mov_b32_e32 v5, s14206; GFX7-NEXT:    v_mov_b32_e32 v6, s34207; GFX7-NEXT:    v_mov_b32_e32 v7, s24208; GFX7-NEXT:    v_mov_b32_e32 v4, s04209; GFX7-NEXT:  .LBB52_1: ; %atomicrmw.start4210; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=14211; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4212; GFX7-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]4213; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc4214; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc4215; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4216; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4217; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4218; GFX7-NEXT:    v_mov_b32_e32 v3, v14219; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]4220; GFX7-NEXT:    v_mov_b32_e32 v2, v04221; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]4222; GFX7-NEXT:    s_cbranch_execnz .LBB52_14223; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end4224; GFX7-NEXT:    s_endpgm4225;4226; GFX8-LABEL: atomic_umin_i64:4227; GFX8:       ; %bb.0: ; %entry4228; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244229; GFX8-NEXT:    s_mov_b64 s[4:5], 04230; GFX8-NEXT:    s_waitcnt lgkmcnt(0)4231; GFX8-NEXT:    v_mov_b32_e32 v0, s04232; GFX8-NEXT:    v_mov_b32_e32 v1, s14233; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]4234; GFX8-NEXT:    v_mov_b32_e32 v5, s14235; GFX8-NEXT:    v_mov_b32_e32 v6, s34236; GFX8-NEXT:    v_mov_b32_e32 v7, s24237; GFX8-NEXT:    v_mov_b32_e32 v4, s04238; GFX8-NEXT:  .LBB52_1: ; %atomicrmw.start4239; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=14240; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4241; GFX8-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]4242; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc4243; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc4244; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4245; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4246; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4247; GFX8-NEXT:    v_mov_b32_e32 v3, v14248; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]4249; GFX8-NEXT:    v_mov_b32_e32 v2, v04250; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]4251; GFX8-NEXT:    s_cbranch_execnz .LBB52_14252; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end4253; GFX8-NEXT:    s_endpgm4254;4255; GFX12-LABEL: atomic_umin_i64:4256; GFX12:       ; %bb.0: ; %entry4257; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x244258; GFX12-NEXT:    s_wait_kmcnt 0x04259; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14260; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s34261; GFX12-NEXT:    flat_atomic_min_u64 v[0:1], v[2:3] scope:SCOPE_SE4262; GFX12-NEXT:    s_wait_storecnt_dscnt 0x04263; GFX12-NEXT:    global_inv scope:SCOPE_SE4264; GFX12-NEXT:    s_endpgm4265entry:4266  %tmp0 = atomicrmw volatile umin ptr %out, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !04267  ret void4268}4269 4270define amdgpu_kernel void @atomic_umin_i64_ret(ptr %out, ptr %out2, i64 %in) {4271; GFX7-LABEL: atomic_umin_i64_ret:4272; GFX7:       ; %bb.0: ; %entry4273; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x94274; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd4275; GFX7-NEXT:    s_mov_b64 s[6:7], 04276; GFX7-NEXT:    s_waitcnt lgkmcnt(0)4277; GFX7-NEXT:    v_mov_b32_e32 v0, s04278; GFX7-NEXT:    v_mov_b32_e32 v1, s14279; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]4280; GFX7-NEXT:    v_mov_b32_e32 v3, s14281; GFX7-NEXT:    v_mov_b32_e32 v4, s54282; GFX7-NEXT:    v_mov_b32_e32 v5, s44283; GFX7-NEXT:    v_mov_b32_e32 v2, s04284; GFX7-NEXT:  .LBB53_1: ; %atomicrmw.start4285; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=14286; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4287; GFX7-NEXT:    v_mov_b32_e32 v9, v14288; GFX7-NEXT:    v_mov_b32_e32 v8, v04289; GFX7-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]4290; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc4291; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc4292; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc4293; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4294; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]4295; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]4296; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]4297; GFX7-NEXT:    s_cbranch_execnz .LBB53_14298; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end4299; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]4300; GFX7-NEXT:    v_mov_b32_e32 v2, s24301; GFX7-NEXT:    v_mov_b32_e32 v3, s34302; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]4303; GFX7-NEXT:    s_endpgm4304;4305; GFX8-LABEL: atomic_umin_i64_ret:4306; GFX8:       ; %bb.0: ; %entry4307; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244308; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x344309; GFX8-NEXT:    s_mov_b64 s[6:7], 04310; GFX8-NEXT:    s_waitcnt lgkmcnt(0)4311; GFX8-NEXT:    v_mov_b32_e32 v0, s04312; GFX8-NEXT:    v_mov_b32_e32 v1, s14313; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]4314; GFX8-NEXT:    v_mov_b32_e32 v3, s14315; GFX8-NEXT:    v_mov_b32_e32 v4, s54316; GFX8-NEXT:    v_mov_b32_e32 v5, s44317; GFX8-NEXT:    v_mov_b32_e32 v2, s04318; GFX8-NEXT:  .LBB53_1: ; %atomicrmw.start4319; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=14320; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4321; GFX8-NEXT:    v_mov_b32_e32 v9, v14322; GFX8-NEXT:    v_mov_b32_e32 v8, v04323; GFX8-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]4324; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc4325; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc4326; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc4327; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4328; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]4329; GFX8-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]4330; GFX8-NEXT:    s_andn2_b64 exec, exec, s[6:7]4331; GFX8-NEXT:    s_cbranch_execnz .LBB53_14332; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end4333; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]4334; GFX8-NEXT:    v_mov_b32_e32 v2, s24335; GFX8-NEXT:    v_mov_b32_e32 v3, s34336; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]4337; GFX8-NEXT:    s_endpgm4338;4339; GFX12-LABEL: atomic_umin_i64_ret:4340; GFX12:       ; %bb.0: ; %entry4341; GFX12-NEXT:    s_clause 0x14342; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x244343; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x344344; GFX12-NEXT:    s_wait_kmcnt 0x04345; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14346; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s54347; GFX12-NEXT:    flat_atomic_min_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE4348; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x04349; GFX12-NEXT:    global_inv scope:SCOPE_SE4350; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s34351; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]4352; GFX12-NEXT:    s_endpgm4353entry:4354  %tmp0 = atomicrmw volatile umin ptr %out, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !04355  store i64 %tmp0, ptr %out24356  ret void4357}4358 4359define amdgpu_kernel void @atomic_umin_i64_addr64(ptr %out, i64 %in, i64 %index) {4360; GFX7-LABEL: atomic_umin_i64_addr64:4361; GFX7:       ; %bb.0: ; %entry4362; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd4363; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x94364; GFX7-NEXT:    s_waitcnt lgkmcnt(0)4365; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 34366; GFX7-NEXT:    s_add_u32 s0, s0, s44367; GFX7-NEXT:    s_addc_u32 s1, s1, s54368; GFX7-NEXT:    v_mov_b32_e32 v5, s14369; GFX7-NEXT:    v_mov_b32_e32 v4, s04370; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]4371; GFX7-NEXT:    s_mov_b64 s[0:1], 04372; GFX7-NEXT:    v_mov_b32_e32 v6, s34373; GFX7-NEXT:    v_mov_b32_e32 v7, s24374; GFX7-NEXT:  .LBB54_1: ; %atomicrmw.start4375; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=14376; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4377; GFX7-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]4378; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc4379; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc4380; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4381; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4382; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4383; GFX7-NEXT:    v_mov_b32_e32 v3, v14384; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]4385; GFX7-NEXT:    v_mov_b32_e32 v2, v04386; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]4387; GFX7-NEXT:    s_cbranch_execnz .LBB54_14388; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end4389; GFX7-NEXT:    s_endpgm4390;4391; GFX8-LABEL: atomic_umin_i64_addr64:4392; GFX8:       ; %bb.0: ; %entry4393; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x344394; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244395; GFX8-NEXT:    s_waitcnt lgkmcnt(0)4396; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 34397; GFX8-NEXT:    s_add_u32 s0, s0, s44398; GFX8-NEXT:    s_addc_u32 s1, s1, s54399; GFX8-NEXT:    v_mov_b32_e32 v5, s14400; GFX8-NEXT:    v_mov_b32_e32 v4, s04401; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]4402; GFX8-NEXT:    s_mov_b64 s[0:1], 04403; GFX8-NEXT:    v_mov_b32_e32 v6, s34404; GFX8-NEXT:    v_mov_b32_e32 v7, s24405; GFX8-NEXT:  .LBB54_1: ; %atomicrmw.start4406; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=14407; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4408; GFX8-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]4409; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc4410; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc4411; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4412; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4413; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4414; GFX8-NEXT:    v_mov_b32_e32 v3, v14415; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]4416; GFX8-NEXT:    v_mov_b32_e32 v2, v04417; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]4418; GFX8-NEXT:    s_cbranch_execnz .LBB54_14419; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end4420; GFX8-NEXT:    s_endpgm4421;4422; GFX12-LABEL: atomic_umin_i64_addr64:4423; GFX12:       ; %bb.0: ; %entry4424; GFX12-NEXT:    s_clause 0x14425; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x344426; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x244427; GFX12-NEXT:    s_wait_kmcnt 0x04428; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 34429; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s34430; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]4431; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)4432; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s04433; GFX12-NEXT:    flat_atomic_min_u64 v[2:3], v[0:1] scope:SCOPE_SE4434; GFX12-NEXT:    s_wait_storecnt_dscnt 0x04435; GFX12-NEXT:    global_inv scope:SCOPE_SE4436; GFX12-NEXT:    s_endpgm4437entry:4438  %ptr = getelementptr inbounds i64, ptr %out, i64 %index4439  %tmp0 = atomicrmw volatile umin ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !04440  ret void4441}4442 4443define amdgpu_kernel void @atomic_umin_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {4444; GFX7-LABEL: atomic_umin_i64_ret_addr64:4445; GFX7:       ; %bb.0: ; %entry4446; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x94447; GFX7-NEXT:    s_waitcnt lgkmcnt(0)4448; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 34449; GFX7-NEXT:    s_add_u32 s0, s0, s64450; GFX7-NEXT:    s_addc_u32 s1, s1, s74451; GFX7-NEXT:    v_mov_b32_e32 v0, s04452; GFX7-NEXT:    v_mov_b32_e32 v1, s14453; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]4454; GFX7-NEXT:    s_mov_b64 s[0:1], 04455; GFX7-NEXT:    v_mov_b32_e32 v4, s54456; GFX7-NEXT:    v_mov_b32_e32 v5, s44457; GFX7-NEXT:  .LBB55_1: ; %atomicrmw.start4458; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=14459; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4460; GFX7-NEXT:    v_mov_b32_e32 v9, v34461; GFX7-NEXT:    v_mov_b32_e32 v8, v24462; GFX7-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]4463; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc4464; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc4465; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc4466; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4467; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]4468; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]4469; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]4470; GFX7-NEXT:    s_cbranch_execnz .LBB55_14471; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end4472; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]4473; GFX7-NEXT:    v_mov_b32_e32 v0, s24474; GFX7-NEXT:    v_mov_b32_e32 v1, s34475; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]4476; GFX7-NEXT:    s_endpgm4477;4478; GFX8-LABEL: atomic_umin_i64_ret_addr64:4479; GFX8:       ; %bb.0: ; %entry4480; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x244481; GFX8-NEXT:    s_waitcnt lgkmcnt(0)4482; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 34483; GFX8-NEXT:    s_add_u32 s0, s0, s64484; GFX8-NEXT:    s_addc_u32 s1, s1, s74485; GFX8-NEXT:    v_mov_b32_e32 v0, s04486; GFX8-NEXT:    v_mov_b32_e32 v1, s14487; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]4488; GFX8-NEXT:    s_mov_b64 s[0:1], 04489; GFX8-NEXT:    v_mov_b32_e32 v4, s54490; GFX8-NEXT:    v_mov_b32_e32 v5, s44491; GFX8-NEXT:  .LBB55_1: ; %atomicrmw.start4492; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=14493; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4494; GFX8-NEXT:    v_mov_b32_e32 v9, v34495; GFX8-NEXT:    v_mov_b32_e32 v8, v24496; GFX8-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]4497; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc4498; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc4499; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc4500; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4501; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]4502; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]4503; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]4504; GFX8-NEXT:    s_cbranch_execnz .LBB55_14505; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end4506; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]4507; GFX8-NEXT:    v_mov_b32_e32 v0, s24508; GFX8-NEXT:    v_mov_b32_e32 v1, s34509; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]4510; GFX8-NEXT:    s_endpgm4511;4512; GFX12-LABEL: atomic_umin_i64_ret_addr64:4513; GFX12:       ; %bb.0: ; %entry4514; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x244515; GFX12-NEXT:    s_wait_kmcnt 0x04516; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 34517; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s54518; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]4519; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)4520; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s04521; GFX12-NEXT:    flat_atomic_min_u64 v[0:1], v[2:3], v[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SE4522; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x04523; GFX12-NEXT:    global_inv scope:SCOPE_SE4524; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s34525; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]4526; GFX12-NEXT:    s_endpgm4527entry:4528  %ptr = getelementptr inbounds i64, ptr %out, i64 %index4529  %tmp0 = atomicrmw volatile umin ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !noalias.addrspace !04530  store i64 %tmp0, ptr %out24531  ret void4532}4533 4534define amdgpu_kernel void @atomic_or_i64_offset(ptr %out, i64 %in) {4535; GFX7-LABEL: atomic_or_i64_offset:4536; GFX7:       ; %bb.0: ; %entry4537; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x94538; GFX7-NEXT:    s_waitcnt lgkmcnt(0)4539; GFX7-NEXT:    s_add_u32 s0, s0, 324540; GFX7-NEXT:    s_addc_u32 s1, s1, 04541; GFX7-NEXT:    v_mov_b32_e32 v5, s14542; GFX7-NEXT:    v_mov_b32_e32 v4, s04543; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]4544; GFX7-NEXT:    s_mov_b64 s[0:1], 04545; GFX7-NEXT:  .LBB56_1: ; %atomicrmw.start4546; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=14547; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4548; GFX7-NEXT:    v_or_b32_e32 v1, s3, v34549; GFX7-NEXT:    v_or_b32_e32 v0, s2, v24550; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4551; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4552; GFX7-NEXT:    buffer_wbinvl1_vol4553; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4554; GFX7-NEXT:    v_mov_b32_e32 v3, v14555; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]4556; GFX7-NEXT:    v_mov_b32_e32 v2, v04557; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]4558; GFX7-NEXT:    s_cbranch_execnz .LBB56_14559; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end4560; GFX7-NEXT:    s_endpgm4561;4562; GFX8-LABEL: atomic_or_i64_offset:4563; GFX8:       ; %bb.0: ; %entry4564; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244565; GFX8-NEXT:    s_waitcnt lgkmcnt(0)4566; GFX8-NEXT:    s_add_u32 s0, s0, 324567; GFX8-NEXT:    s_addc_u32 s1, s1, 04568; GFX8-NEXT:    v_mov_b32_e32 v5, s14569; GFX8-NEXT:    v_mov_b32_e32 v4, s04570; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]4571; GFX8-NEXT:    s_mov_b64 s[0:1], 04572; GFX8-NEXT:  .LBB56_1: ; %atomicrmw.start4573; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=14574; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4575; GFX8-NEXT:    v_or_b32_e32 v1, s3, v34576; GFX8-NEXT:    v_or_b32_e32 v0, s2, v24577; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4578; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4579; GFX8-NEXT:    buffer_wbinvl1_vol4580; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4581; GFX8-NEXT:    v_mov_b32_e32 v3, v14582; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]4583; GFX8-NEXT:    v_mov_b32_e32 v2, v04584; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]4585; GFX8-NEXT:    s_cbranch_execnz .LBB56_14586; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end4587; GFX8-NEXT:    s_endpgm4588;4589; GFX12-LABEL: atomic_or_i64_offset:4590; GFX12:       ; %bb.0: ; %entry4591; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x244592; GFX12-NEXT:    s_wait_kmcnt 0x04593; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14594; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s34595; GFX12-NEXT:    flat_atomic_or_b64 v[0:1], v[2:3] offset:32 scope:SCOPE_DEV4596; GFX12-NEXT:    s_wait_storecnt_dscnt 0x04597; GFX12-NEXT:    global_inv scope:SCOPE_DEV4598; GFX12-NEXT:    s_endpgm4599entry:4600  %gep = getelementptr inbounds i64, ptr %out, i64 44601  %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !04602  ret void4603}4604 4605define amdgpu_kernel void @atomic_or_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {4606; GFX7-LABEL: atomic_or_i64_ret_offset:4607; GFX7:       ; %bb.0: ; %entry4608; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x94609; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd4610; GFX7-NEXT:    s_waitcnt lgkmcnt(0)4611; GFX7-NEXT:    s_add_u32 s0, s0, 324612; GFX7-NEXT:    s_addc_u32 s1, s1, 04613; GFX7-NEXT:    v_mov_b32_e32 v0, s04614; GFX7-NEXT:    v_mov_b32_e32 v1, s14615; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]4616; GFX7-NEXT:    s_mov_b64 s[0:1], 04617; GFX7-NEXT:  .LBB57_1: ; %atomicrmw.start4618; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=14619; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4620; GFX7-NEXT:    v_mov_b32_e32 v5, v34621; GFX7-NEXT:    v_mov_b32_e32 v4, v24622; GFX7-NEXT:    v_or_b32_e32 v3, s5, v54623; GFX7-NEXT:    v_or_b32_e32 v2, s4, v44624; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc4625; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4626; GFX7-NEXT:    buffer_wbinvl1_vol4627; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]4628; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]4629; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]4630; GFX7-NEXT:    s_cbranch_execnz .LBB57_14631; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end4632; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]4633; GFX7-NEXT:    v_mov_b32_e32 v0, s24634; GFX7-NEXT:    v_mov_b32_e32 v1, s34635; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]4636; GFX7-NEXT:    s_endpgm4637;4638; GFX8-LABEL: atomic_or_i64_ret_offset:4639; GFX8:       ; %bb.0: ; %entry4640; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244641; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x344642; GFX8-NEXT:    s_waitcnt lgkmcnt(0)4643; GFX8-NEXT:    s_add_u32 s0, s0, 324644; GFX8-NEXT:    s_addc_u32 s1, s1, 04645; GFX8-NEXT:    v_mov_b32_e32 v0, s04646; GFX8-NEXT:    v_mov_b32_e32 v1, s14647; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]4648; GFX8-NEXT:    s_mov_b64 s[0:1], 04649; GFX8-NEXT:  .LBB57_1: ; %atomicrmw.start4650; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=14651; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4652; GFX8-NEXT:    v_mov_b32_e32 v5, v34653; GFX8-NEXT:    v_mov_b32_e32 v4, v24654; GFX8-NEXT:    v_or_b32_e32 v3, s5, v54655; GFX8-NEXT:    v_or_b32_e32 v2, s4, v44656; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc4657; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4658; GFX8-NEXT:    buffer_wbinvl1_vol4659; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]4660; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]4661; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]4662; GFX8-NEXT:    s_cbranch_execnz .LBB57_14663; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end4664; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]4665; GFX8-NEXT:    v_mov_b32_e32 v0, s24666; GFX8-NEXT:    v_mov_b32_e32 v1, s34667; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]4668; GFX8-NEXT:    s_endpgm4669;4670; GFX12-LABEL: atomic_or_i64_ret_offset:4671; GFX12:       ; %bb.0: ; %entry4672; GFX12-NEXT:    s_clause 0x14673; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x244674; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x344675; GFX12-NEXT:    s_wait_kmcnt 0x04676; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14677; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s54678; GFX12-NEXT:    flat_atomic_or_b64 v[0:1], v[0:1], v[2:3] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV4679; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x04680; GFX12-NEXT:    global_inv scope:SCOPE_DEV4681; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s34682; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]4683; GFX12-NEXT:    s_endpgm4684entry:4685  %gep = getelementptr inbounds i64, ptr %out, i64 44686  %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !04687  store i64 %tmp0, ptr %out24688  ret void4689}4690 4691define amdgpu_kernel void @atomic_or_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {4692; GFX7-LABEL: atomic_or_i64_addr64_offset:4693; GFX7:       ; %bb.0: ; %entry4694; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd4695; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x94696; GFX7-NEXT:    s_waitcnt lgkmcnt(0)4697; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 34698; GFX7-NEXT:    s_add_u32 s0, s0, s44699; GFX7-NEXT:    s_addc_u32 s1, s1, s54700; GFX7-NEXT:    s_add_u32 s0, s0, 324701; GFX7-NEXT:    s_addc_u32 s1, s1, 04702; GFX7-NEXT:    v_mov_b32_e32 v5, s14703; GFX7-NEXT:    v_mov_b32_e32 v4, s04704; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]4705; GFX7-NEXT:    s_mov_b64 s[0:1], 04706; GFX7-NEXT:  .LBB58_1: ; %atomicrmw.start4707; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=14708; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4709; GFX7-NEXT:    v_or_b32_e32 v1, s3, v34710; GFX7-NEXT:    v_or_b32_e32 v0, s2, v24711; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4712; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4713; GFX7-NEXT:    buffer_wbinvl1_vol4714; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4715; GFX7-NEXT:    v_mov_b32_e32 v3, v14716; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]4717; GFX7-NEXT:    v_mov_b32_e32 v2, v04718; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]4719; GFX7-NEXT:    s_cbranch_execnz .LBB58_14720; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end4721; GFX7-NEXT:    s_endpgm4722;4723; GFX8-LABEL: atomic_or_i64_addr64_offset:4724; GFX8:       ; %bb.0: ; %entry4725; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x344726; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244727; GFX8-NEXT:    s_waitcnt lgkmcnt(0)4728; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 34729; GFX8-NEXT:    s_add_u32 s0, s0, s44730; GFX8-NEXT:    s_addc_u32 s1, s1, s54731; GFX8-NEXT:    s_add_u32 s0, s0, 324732; GFX8-NEXT:    s_addc_u32 s1, s1, 04733; GFX8-NEXT:    v_mov_b32_e32 v5, s14734; GFX8-NEXT:    v_mov_b32_e32 v4, s04735; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]4736; GFX8-NEXT:    s_mov_b64 s[0:1], 04737; GFX8-NEXT:  .LBB58_1: ; %atomicrmw.start4738; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=14739; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4740; GFX8-NEXT:    v_or_b32_e32 v1, s3, v34741; GFX8-NEXT:    v_or_b32_e32 v0, s2, v24742; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4743; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4744; GFX8-NEXT:    buffer_wbinvl1_vol4745; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4746; GFX8-NEXT:    v_mov_b32_e32 v3, v14747; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]4748; GFX8-NEXT:    v_mov_b32_e32 v2, v04749; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]4750; GFX8-NEXT:    s_cbranch_execnz .LBB58_14751; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end4752; GFX8-NEXT:    s_endpgm4753;4754; GFX12-LABEL: atomic_or_i64_addr64_offset:4755; GFX12:       ; %bb.0: ; %entry4756; GFX12-NEXT:    s_clause 0x14757; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x344758; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x244759; GFX12-NEXT:    s_wait_kmcnt 0x04760; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 34761; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s34762; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]4763; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)4764; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s04765; GFX12-NEXT:    flat_atomic_or_b64 v[2:3], v[0:1] offset:32 scope:SCOPE_DEV4766; GFX12-NEXT:    s_wait_storecnt_dscnt 0x04767; GFX12-NEXT:    global_inv scope:SCOPE_DEV4768; GFX12-NEXT:    s_endpgm4769entry:4770  %ptr = getelementptr inbounds i64, ptr %out, i64 %index4771  %gep = getelementptr inbounds i64, ptr %ptr, i64 44772  %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !04773  ret void4774}4775 4776define amdgpu_kernel void @atomic_or_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {4777; GFX7-LABEL: atomic_or_i64_ret_addr64_offset:4778; GFX7:       ; %bb.0: ; %entry4779; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x94780; GFX7-NEXT:    s_waitcnt lgkmcnt(0)4781; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 34782; GFX7-NEXT:    s_add_u32 s0, s0, s64783; GFX7-NEXT:    s_addc_u32 s1, s1, s74784; GFX7-NEXT:    s_add_u32 s0, s0, 324785; GFX7-NEXT:    s_addc_u32 s1, s1, 04786; GFX7-NEXT:    v_mov_b32_e32 v0, s04787; GFX7-NEXT:    v_mov_b32_e32 v1, s14788; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]4789; GFX7-NEXT:    s_mov_b64 s[0:1], 04790; GFX7-NEXT:  .LBB59_1: ; %atomicrmw.start4791; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=14792; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4793; GFX7-NEXT:    v_mov_b32_e32 v5, v34794; GFX7-NEXT:    v_mov_b32_e32 v4, v24795; GFX7-NEXT:    v_or_b32_e32 v3, s5, v54796; GFX7-NEXT:    v_or_b32_e32 v2, s4, v44797; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc4798; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4799; GFX7-NEXT:    buffer_wbinvl1_vol4800; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]4801; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]4802; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]4803; GFX7-NEXT:    s_cbranch_execnz .LBB59_14804; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end4805; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]4806; GFX7-NEXT:    v_mov_b32_e32 v0, s24807; GFX7-NEXT:    v_mov_b32_e32 v1, s34808; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]4809; GFX7-NEXT:    s_endpgm4810;4811; GFX8-LABEL: atomic_or_i64_ret_addr64_offset:4812; GFX8:       ; %bb.0: ; %entry4813; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x244814; GFX8-NEXT:    s_waitcnt lgkmcnt(0)4815; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 34816; GFX8-NEXT:    s_add_u32 s0, s0, s64817; GFX8-NEXT:    s_addc_u32 s1, s1, s74818; GFX8-NEXT:    s_add_u32 s0, s0, 324819; GFX8-NEXT:    s_addc_u32 s1, s1, 04820; GFX8-NEXT:    v_mov_b32_e32 v0, s04821; GFX8-NEXT:    v_mov_b32_e32 v1, s14822; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]4823; GFX8-NEXT:    s_mov_b64 s[0:1], 04824; GFX8-NEXT:  .LBB59_1: ; %atomicrmw.start4825; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=14826; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4827; GFX8-NEXT:    v_mov_b32_e32 v5, v34828; GFX8-NEXT:    v_mov_b32_e32 v4, v24829; GFX8-NEXT:    v_or_b32_e32 v3, s5, v54830; GFX8-NEXT:    v_or_b32_e32 v2, s4, v44831; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc4832; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4833; GFX8-NEXT:    buffer_wbinvl1_vol4834; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]4835; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]4836; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]4837; GFX8-NEXT:    s_cbranch_execnz .LBB59_14838; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end4839; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]4840; GFX8-NEXT:    v_mov_b32_e32 v0, s24841; GFX8-NEXT:    v_mov_b32_e32 v1, s34842; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]4843; GFX8-NEXT:    s_endpgm4844;4845; GFX12-LABEL: atomic_or_i64_ret_addr64_offset:4846; GFX12:       ; %bb.0: ; %entry4847; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x244848; GFX12-NEXT:    s_wait_kmcnt 0x04849; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 34850; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s54851; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]4852; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)4853; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s04854; GFX12-NEXT:    flat_atomic_or_b64 v[0:1], v[2:3], v[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV4855; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x04856; GFX12-NEXT:    global_inv scope:SCOPE_DEV4857; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s34858; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]4859; GFX12-NEXT:    s_endpgm4860entry:4861  %ptr = getelementptr inbounds i64, ptr %out, i64 %index4862  %gep = getelementptr inbounds i64, ptr %ptr, i64 44863  %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !04864  store i64 %tmp0, ptr %out24865  ret void4866}4867 4868define amdgpu_kernel void @atomic_or_i64(ptr %out, i64 %in) {4869; GFX7-LABEL: atomic_or_i64:4870; GFX7:       ; %bb.0: ; %entry4871; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x94872; GFX7-NEXT:    s_mov_b64 s[4:5], 04873; GFX7-NEXT:    s_waitcnt lgkmcnt(0)4874; GFX7-NEXT:    v_mov_b32_e32 v0, s04875; GFX7-NEXT:    v_mov_b32_e32 v1, s14876; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]4877; GFX7-NEXT:    v_mov_b32_e32 v5, s14878; GFX7-NEXT:    v_mov_b32_e32 v4, s04879; GFX7-NEXT:  .LBB60_1: ; %atomicrmw.start4880; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=14881; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4882; GFX7-NEXT:    v_or_b32_e32 v1, s3, v34883; GFX7-NEXT:    v_or_b32_e32 v0, s2, v24884; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4885; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4886; GFX7-NEXT:    buffer_wbinvl1_vol4887; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4888; GFX7-NEXT:    v_mov_b32_e32 v3, v14889; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]4890; GFX7-NEXT:    v_mov_b32_e32 v2, v04891; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]4892; GFX7-NEXT:    s_cbranch_execnz .LBB60_14893; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end4894; GFX7-NEXT:    s_endpgm4895;4896; GFX8-LABEL: atomic_or_i64:4897; GFX8:       ; %bb.0: ; %entry4898; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244899; GFX8-NEXT:    s_mov_b64 s[4:5], 04900; GFX8-NEXT:    s_waitcnt lgkmcnt(0)4901; GFX8-NEXT:    v_mov_b32_e32 v0, s04902; GFX8-NEXT:    v_mov_b32_e32 v1, s14903; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]4904; GFX8-NEXT:    v_mov_b32_e32 v5, s14905; GFX8-NEXT:    v_mov_b32_e32 v4, s04906; GFX8-NEXT:  .LBB60_1: ; %atomicrmw.start4907; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=14908; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4909; GFX8-NEXT:    v_or_b32_e32 v1, s3, v34910; GFX8-NEXT:    v_or_b32_e32 v0, s2, v24911; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4912; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4913; GFX8-NEXT:    buffer_wbinvl1_vol4914; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4915; GFX8-NEXT:    v_mov_b32_e32 v3, v14916; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]4917; GFX8-NEXT:    v_mov_b32_e32 v2, v04918; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]4919; GFX8-NEXT:    s_cbranch_execnz .LBB60_14920; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end4921; GFX8-NEXT:    s_endpgm4922;4923; GFX12-LABEL: atomic_or_i64:4924; GFX12:       ; %bb.0: ; %entry4925; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x244926; GFX12-NEXT:    s_wait_kmcnt 0x04927; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14928; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s34929; GFX12-NEXT:    flat_atomic_or_b64 v[0:1], v[2:3] scope:SCOPE_DEV4930; GFX12-NEXT:    s_wait_storecnt_dscnt 0x04931; GFX12-NEXT:    global_inv scope:SCOPE_DEV4932; GFX12-NEXT:    s_endpgm4933entry:4934  %tmp0 = atomicrmw volatile or ptr %out, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !04935  ret void4936}4937 4938define amdgpu_kernel void @atomic_or_i64_ret(ptr %out, ptr %out2, i64 %in) {4939; GFX7-LABEL: atomic_or_i64_ret:4940; GFX7:       ; %bb.0: ; %entry4941; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x94942; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd4943; GFX7-NEXT:    s_mov_b64 s[6:7], 04944; GFX7-NEXT:    s_waitcnt lgkmcnt(0)4945; GFX7-NEXT:    v_mov_b32_e32 v0, s04946; GFX7-NEXT:    v_mov_b32_e32 v1, s14947; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]4948; GFX7-NEXT:    v_mov_b32_e32 v3, s14949; GFX7-NEXT:    v_mov_b32_e32 v2, s04950; GFX7-NEXT:  .LBB61_1: ; %atomicrmw.start4951; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=14952; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4953; GFX7-NEXT:    v_mov_b32_e32 v7, v14954; GFX7-NEXT:    v_mov_b32_e32 v6, v04955; GFX7-NEXT:    v_or_b32_e32 v5, s5, v74956; GFX7-NEXT:    v_or_b32_e32 v4, s4, v64957; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc4958; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4959; GFX7-NEXT:    buffer_wbinvl1_vol4960; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]4961; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]4962; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]4963; GFX7-NEXT:    s_cbranch_execnz .LBB61_14964; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end4965; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]4966; GFX7-NEXT:    v_mov_b32_e32 v2, s24967; GFX7-NEXT:    v_mov_b32_e32 v3, s34968; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]4969; GFX7-NEXT:    s_endpgm4970;4971; GFX8-LABEL: atomic_or_i64_ret:4972; GFX8:       ; %bb.0: ; %entry4973; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244974; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x344975; GFX8-NEXT:    s_mov_b64 s[6:7], 04976; GFX8-NEXT:    s_waitcnt lgkmcnt(0)4977; GFX8-NEXT:    v_mov_b32_e32 v0, s04978; GFX8-NEXT:    v_mov_b32_e32 v1, s14979; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]4980; GFX8-NEXT:    v_mov_b32_e32 v3, s14981; GFX8-NEXT:    v_mov_b32_e32 v2, s04982; GFX8-NEXT:  .LBB61_1: ; %atomicrmw.start4983; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=14984; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4985; GFX8-NEXT:    v_mov_b32_e32 v7, v14986; GFX8-NEXT:    v_mov_b32_e32 v6, v04987; GFX8-NEXT:    v_or_b32_e32 v5, s5, v74988; GFX8-NEXT:    v_or_b32_e32 v4, s4, v64989; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc4990; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)4991; GFX8-NEXT:    buffer_wbinvl1_vol4992; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]4993; GFX8-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]4994; GFX8-NEXT:    s_andn2_b64 exec, exec, s[6:7]4995; GFX8-NEXT:    s_cbranch_execnz .LBB61_14996; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end4997; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]4998; GFX8-NEXT:    v_mov_b32_e32 v2, s24999; GFX8-NEXT:    v_mov_b32_e32 v3, s35000; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]5001; GFX8-NEXT:    s_endpgm5002;5003; GFX12-LABEL: atomic_or_i64_ret:5004; GFX12:       ; %bb.0: ; %entry5005; GFX12-NEXT:    s_clause 0x15006; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x245007; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x345008; GFX12-NEXT:    s_wait_kmcnt 0x05009; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15010; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s55011; GFX12-NEXT:    flat_atomic_or_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV5012; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x05013; GFX12-NEXT:    global_inv scope:SCOPE_DEV5014; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s35015; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]5016; GFX12-NEXT:    s_endpgm5017entry:5018  %tmp0 = atomicrmw volatile or ptr %out, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !05019  store i64 %tmp0, ptr %out25020  ret void5021}5022 5023define amdgpu_kernel void @atomic_or_i64_addr64(ptr %out, i64 %in, i64 %index) {5024; GFX7-LABEL: atomic_or_i64_addr64:5025; GFX7:       ; %bb.0: ; %entry5026; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd5027; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x95028; GFX7-NEXT:    s_waitcnt lgkmcnt(0)5029; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 35030; GFX7-NEXT:    s_add_u32 s0, s0, s45031; GFX7-NEXT:    s_addc_u32 s1, s1, s55032; GFX7-NEXT:    v_mov_b32_e32 v5, s15033; GFX7-NEXT:    v_mov_b32_e32 v4, s05034; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]5035; GFX7-NEXT:    s_mov_b64 s[0:1], 05036; GFX7-NEXT:  .LBB62_1: ; %atomicrmw.start5037; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=15038; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5039; GFX7-NEXT:    v_or_b32_e32 v1, s3, v35040; GFX7-NEXT:    v_or_b32_e32 v0, s2, v25041; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc5042; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5043; GFX7-NEXT:    buffer_wbinvl1_vol5044; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]5045; GFX7-NEXT:    v_mov_b32_e32 v3, v15046; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]5047; GFX7-NEXT:    v_mov_b32_e32 v2, v05048; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]5049; GFX7-NEXT:    s_cbranch_execnz .LBB62_15050; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end5051; GFX7-NEXT:    s_endpgm5052;5053; GFX8-LABEL: atomic_or_i64_addr64:5054; GFX8:       ; %bb.0: ; %entry5055; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x345056; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245057; GFX8-NEXT:    s_waitcnt lgkmcnt(0)5058; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 35059; GFX8-NEXT:    s_add_u32 s0, s0, s45060; GFX8-NEXT:    s_addc_u32 s1, s1, s55061; GFX8-NEXT:    v_mov_b32_e32 v5, s15062; GFX8-NEXT:    v_mov_b32_e32 v4, s05063; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]5064; GFX8-NEXT:    s_mov_b64 s[0:1], 05065; GFX8-NEXT:  .LBB62_1: ; %atomicrmw.start5066; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=15067; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5068; GFX8-NEXT:    v_or_b32_e32 v1, s3, v35069; GFX8-NEXT:    v_or_b32_e32 v0, s2, v25070; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc5071; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5072; GFX8-NEXT:    buffer_wbinvl1_vol5073; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]5074; GFX8-NEXT:    v_mov_b32_e32 v3, v15075; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]5076; GFX8-NEXT:    v_mov_b32_e32 v2, v05077; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]5078; GFX8-NEXT:    s_cbranch_execnz .LBB62_15079; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end5080; GFX8-NEXT:    s_endpgm5081;5082; GFX12-LABEL: atomic_or_i64_addr64:5083; GFX12:       ; %bb.0: ; %entry5084; GFX12-NEXT:    s_clause 0x15085; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x345086; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x245087; GFX12-NEXT:    s_wait_kmcnt 0x05088; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 35089; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s35090; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]5091; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)5092; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s05093; GFX12-NEXT:    flat_atomic_or_b64 v[2:3], v[0:1] scope:SCOPE_DEV5094; GFX12-NEXT:    s_wait_storecnt_dscnt 0x05095; GFX12-NEXT:    global_inv scope:SCOPE_DEV5096; GFX12-NEXT:    s_endpgm5097entry:5098  %ptr = getelementptr inbounds i64, ptr %out, i64 %index5099  %tmp0 = atomicrmw volatile or ptr %ptr, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !05100  ret void5101}5102 5103define amdgpu_kernel void @atomic_or_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {5104; GFX7-LABEL: atomic_or_i64_ret_addr64:5105; GFX7:       ; %bb.0: ; %entry5106; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x95107; GFX7-NEXT:    s_waitcnt lgkmcnt(0)5108; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 35109; GFX7-NEXT:    s_add_u32 s0, s0, s65110; GFX7-NEXT:    s_addc_u32 s1, s1, s75111; GFX7-NEXT:    v_mov_b32_e32 v0, s05112; GFX7-NEXT:    v_mov_b32_e32 v1, s15113; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]5114; GFX7-NEXT:    s_mov_b64 s[0:1], 05115; GFX7-NEXT:  .LBB63_1: ; %atomicrmw.start5116; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=15117; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5118; GFX7-NEXT:    v_mov_b32_e32 v5, v35119; GFX7-NEXT:    v_mov_b32_e32 v4, v25120; GFX7-NEXT:    v_or_b32_e32 v3, s5, v55121; GFX7-NEXT:    v_or_b32_e32 v2, s4, v45122; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc5123; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5124; GFX7-NEXT:    buffer_wbinvl1_vol5125; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]5126; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]5127; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]5128; GFX7-NEXT:    s_cbranch_execnz .LBB63_15129; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end5130; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]5131; GFX7-NEXT:    v_mov_b32_e32 v0, s25132; GFX7-NEXT:    v_mov_b32_e32 v1, s35133; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]5134; GFX7-NEXT:    s_endpgm5135;5136; GFX8-LABEL: atomic_or_i64_ret_addr64:5137; GFX8:       ; %bb.0: ; %entry5138; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x245139; GFX8-NEXT:    s_waitcnt lgkmcnt(0)5140; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 35141; GFX8-NEXT:    s_add_u32 s0, s0, s65142; GFX8-NEXT:    s_addc_u32 s1, s1, s75143; GFX8-NEXT:    v_mov_b32_e32 v0, s05144; GFX8-NEXT:    v_mov_b32_e32 v1, s15145; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]5146; GFX8-NEXT:    s_mov_b64 s[0:1], 05147; GFX8-NEXT:  .LBB63_1: ; %atomicrmw.start5148; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=15149; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5150; GFX8-NEXT:    v_mov_b32_e32 v5, v35151; GFX8-NEXT:    v_mov_b32_e32 v4, v25152; GFX8-NEXT:    v_or_b32_e32 v3, s5, v55153; GFX8-NEXT:    v_or_b32_e32 v2, s4, v45154; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc5155; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5156; GFX8-NEXT:    buffer_wbinvl1_vol5157; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]5158; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]5159; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]5160; GFX8-NEXT:    s_cbranch_execnz .LBB63_15161; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end5162; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]5163; GFX8-NEXT:    v_mov_b32_e32 v0, s25164; GFX8-NEXT:    v_mov_b32_e32 v1, s35165; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]5166; GFX8-NEXT:    s_endpgm5167;5168; GFX12-LABEL: atomic_or_i64_ret_addr64:5169; GFX12:       ; %bb.0: ; %entry5170; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x245171; GFX12-NEXT:    s_wait_kmcnt 0x05172; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 35173; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s55174; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]5175; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)5176; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s05177; GFX12-NEXT:    flat_atomic_or_b64 v[0:1], v[2:3], v[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV5178; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x05179; GFX12-NEXT:    global_inv scope:SCOPE_DEV5180; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s35181; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]5182; GFX12-NEXT:    s_endpgm5183entry:5184  %ptr = getelementptr inbounds i64, ptr %out, i64 %index5185  %tmp0 = atomicrmw volatile or ptr %ptr, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !05186  store i64 %tmp0, ptr %out25187  ret void5188}5189 5190define amdgpu_kernel void @atomic_xchg_i64_offset(ptr %out, i64 %in) {5191; GFX7-LABEL: atomic_xchg_i64_offset:5192; GFX7:       ; %bb.0: ; %entry5193; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x95194; GFX7-NEXT:    s_waitcnt lgkmcnt(0)5195; GFX7-NEXT:    s_add_u32 s0, s0, 325196; GFX7-NEXT:    s_addc_u32 s1, s1, 05197; GFX7-NEXT:    v_mov_b32_e32 v3, s15198; GFX7-NEXT:    v_mov_b32_e32 v0, s25199; GFX7-NEXT:    v_mov_b32_e32 v1, s35200; GFX7-NEXT:    v_mov_b32_e32 v2, s05201; GFX7-NEXT:    flat_atomic_swap_x2 v[2:3], v[0:1]5202; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5203; GFX7-NEXT:    buffer_wbinvl1_vol5204; GFX7-NEXT:    s_endpgm5205;5206; GFX8-LABEL: atomic_xchg_i64_offset:5207; GFX8:       ; %bb.0: ; %entry5208; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245209; GFX8-NEXT:    s_waitcnt lgkmcnt(0)5210; GFX8-NEXT:    s_add_u32 s0, s0, 325211; GFX8-NEXT:    s_addc_u32 s1, s1, 05212; GFX8-NEXT:    v_mov_b32_e32 v3, s15213; GFX8-NEXT:    v_mov_b32_e32 v0, s25214; GFX8-NEXT:    v_mov_b32_e32 v1, s35215; GFX8-NEXT:    v_mov_b32_e32 v2, s05216; GFX8-NEXT:    flat_atomic_swap_x2 v[2:3], v[0:1]5217; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5218; GFX8-NEXT:    buffer_wbinvl1_vol5219; GFX8-NEXT:    s_endpgm5220;5221; GFX12-LABEL: atomic_xchg_i64_offset:5222; GFX12:       ; %bb.0: ; %entry5223; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x245224; GFX12-NEXT:    s_wait_kmcnt 0x05225; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15226; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s35227; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[2:3] offset:32 scope:SCOPE_DEV5228; GFX12-NEXT:    s_wait_storecnt_dscnt 0x05229; GFX12-NEXT:    global_inv scope:SCOPE_DEV5230; GFX12-NEXT:    s_endpgm5231entry:5232  %gep = getelementptr inbounds i64, ptr %out, i64 45233  %tmp0 = atomicrmw volatile xchg ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !05234  ret void5235}5236 5237define amdgpu_kernel void @atomic_xchg_f64_offset(ptr %out, double %in) {5238; GFX7-LABEL: atomic_xchg_f64_offset:5239; GFX7:       ; %bb.0: ; %entry5240; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x95241; GFX7-NEXT:    s_waitcnt lgkmcnt(0)5242; GFX7-NEXT:    s_add_u32 s0, s0, 325243; GFX7-NEXT:    s_addc_u32 s1, s1, 05244; GFX7-NEXT:    v_mov_b32_e32 v3, s15245; GFX7-NEXT:    v_mov_b32_e32 v0, s25246; GFX7-NEXT:    v_mov_b32_e32 v1, s35247; GFX7-NEXT:    v_mov_b32_e32 v2, s05248; GFX7-NEXT:    flat_atomic_swap_x2 v[2:3], v[0:1]5249; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5250; GFX7-NEXT:    buffer_wbinvl1_vol5251; GFX7-NEXT:    s_endpgm5252;5253; GFX8-LABEL: atomic_xchg_f64_offset:5254; GFX8:       ; %bb.0: ; %entry5255; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245256; GFX8-NEXT:    s_waitcnt lgkmcnt(0)5257; GFX8-NEXT:    s_add_u32 s0, s0, 325258; GFX8-NEXT:    s_addc_u32 s1, s1, 05259; GFX8-NEXT:    v_mov_b32_e32 v3, s15260; GFX8-NEXT:    v_mov_b32_e32 v0, s25261; GFX8-NEXT:    v_mov_b32_e32 v1, s35262; GFX8-NEXT:    v_mov_b32_e32 v2, s05263; GFX8-NEXT:    flat_atomic_swap_x2 v[2:3], v[0:1]5264; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5265; GFX8-NEXT:    buffer_wbinvl1_vol5266; GFX8-NEXT:    s_endpgm5267;5268; GFX12-LABEL: atomic_xchg_f64_offset:5269; GFX12:       ; %bb.0: ; %entry5270; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x245271; GFX12-NEXT:    s_wait_kmcnt 0x05272; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15273; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s35274; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[2:3] offset:32 scope:SCOPE_DEV5275; GFX12-NEXT:    s_wait_storecnt_dscnt 0x05276; GFX12-NEXT:    global_inv scope:SCOPE_DEV5277; GFX12-NEXT:    s_endpgm5278entry:5279  %gep = getelementptr inbounds double, ptr %out, i64 45280  %tmp0 = atomicrmw volatile xchg ptr %gep, double %in syncscope("agent") seq_cst, !noalias.addrspace !05281  ret void5282}5283 5284define amdgpu_kernel void @atomic_xchg_pointer_offset(ptr %out, ptr %in) {5285; GFX7-LABEL: atomic_xchg_pointer_offset:5286; GFX7:       ; %bb.0: ; %entry5287; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x95288; GFX7-NEXT:    s_waitcnt lgkmcnt(0)5289; GFX7-NEXT:    s_add_u32 s0, s0, 325290; GFX7-NEXT:    s_addc_u32 s1, s1, 05291; GFX7-NEXT:    v_mov_b32_e32 v3, s15292; GFX7-NEXT:    v_mov_b32_e32 v0, s25293; GFX7-NEXT:    v_mov_b32_e32 v1, s35294; GFX7-NEXT:    v_mov_b32_e32 v2, s05295; GFX7-NEXT:    flat_atomic_swap_x2 v[2:3], v[0:1]5296; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5297; GFX7-NEXT:    buffer_wbinvl1_vol5298; GFX7-NEXT:    s_endpgm5299;5300; GFX8-LABEL: atomic_xchg_pointer_offset:5301; GFX8:       ; %bb.0: ; %entry5302; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245303; GFX8-NEXT:    s_waitcnt lgkmcnt(0)5304; GFX8-NEXT:    s_add_u32 s0, s0, 325305; GFX8-NEXT:    s_addc_u32 s1, s1, 05306; GFX8-NEXT:    v_mov_b32_e32 v3, s15307; GFX8-NEXT:    v_mov_b32_e32 v0, s25308; GFX8-NEXT:    v_mov_b32_e32 v1, s35309; GFX8-NEXT:    v_mov_b32_e32 v2, s05310; GFX8-NEXT:    flat_atomic_swap_x2 v[2:3], v[0:1]5311; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5312; GFX8-NEXT:    buffer_wbinvl1_vol5313; GFX8-NEXT:    s_endpgm5314;5315; GFX12-LABEL: atomic_xchg_pointer_offset:5316; GFX12:       ; %bb.0: ; %entry5317; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x245318; GFX12-NEXT:    s_wait_kmcnt 0x05319; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15320; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s35321; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[2:3] offset:32 scope:SCOPE_DEV5322; GFX12-NEXT:    s_wait_storecnt_dscnt 0x05323; GFX12-NEXT:    global_inv scope:SCOPE_DEV5324; GFX12-NEXT:    s_endpgm5325entry:5326  %gep = getelementptr inbounds ptr, ptr %out, i32 45327  %val = atomicrmw volatile xchg ptr %gep, ptr %in syncscope("agent") seq_cst, !noalias.addrspace !05328  ret void5329}5330 5331define amdgpu_kernel void @atomic_xchg_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {5332; GFX7-LABEL: atomic_xchg_i64_ret_offset:5333; GFX7:       ; %bb.0: ; %entry5334; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd5335; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x95336; GFX7-NEXT:    s_waitcnt lgkmcnt(0)5337; GFX7-NEXT:    v_mov_b32_e32 v0, s65338; GFX7-NEXT:    s_add_u32 s0, s0, 325339; GFX7-NEXT:    s_addc_u32 s1, s1, 05340; GFX7-NEXT:    v_mov_b32_e32 v3, s15341; GFX7-NEXT:    v_mov_b32_e32 v1, s75342; GFX7-NEXT:    v_mov_b32_e32 v2, s05343; GFX7-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc5344; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5345; GFX7-NEXT:    buffer_wbinvl1_vol5346; GFX7-NEXT:    v_mov_b32_e32 v2, s25347; GFX7-NEXT:    v_mov_b32_e32 v3, s35348; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]5349; GFX7-NEXT:    s_endpgm5350;5351; GFX8-LABEL: atomic_xchg_i64_ret_offset:5352; GFX8:       ; %bb.0: ; %entry5353; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x345354; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245355; GFX8-NEXT:    s_waitcnt lgkmcnt(0)5356; GFX8-NEXT:    v_mov_b32_e32 v0, s65357; GFX8-NEXT:    s_add_u32 s0, s0, 325358; GFX8-NEXT:    s_addc_u32 s1, s1, 05359; GFX8-NEXT:    v_mov_b32_e32 v3, s15360; GFX8-NEXT:    v_mov_b32_e32 v1, s75361; GFX8-NEXT:    v_mov_b32_e32 v2, s05362; GFX8-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc5363; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5364; GFX8-NEXT:    buffer_wbinvl1_vol5365; GFX8-NEXT:    v_mov_b32_e32 v2, s25366; GFX8-NEXT:    v_mov_b32_e32 v3, s35367; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]5368; GFX8-NEXT:    s_endpgm5369;5370; GFX12-LABEL: atomic_xchg_i64_ret_offset:5371; GFX12:       ; %bb.0: ; %entry5372; GFX12-NEXT:    s_clause 0x15373; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x245374; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x345375; GFX12-NEXT:    s_wait_kmcnt 0x05376; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15377; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s55378; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[0:1], v[2:3] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV5379; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x05380; GFX12-NEXT:    global_inv scope:SCOPE_DEV5381; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s35382; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]5383; GFX12-NEXT:    s_endpgm5384entry:5385  %gep = getelementptr inbounds i64, ptr %out, i64 45386  %tmp0 = atomicrmw volatile xchg ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !05387  store i64 %tmp0, ptr %out25388  ret void5389}5390 5391define amdgpu_kernel void @atomic_xchg_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {5392; GFX7-LABEL: atomic_xchg_i64_addr64_offset:5393; GFX7:       ; %bb.0: ; %entry5394; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x95395; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd5396; GFX7-NEXT:    s_waitcnt lgkmcnt(0)5397; GFX7-NEXT:    v_mov_b32_e32 v0, s25398; GFX7-NEXT:    v_mov_b32_e32 v1, s35399; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 35400; GFX7-NEXT:    s_add_u32 s0, s0, s25401; GFX7-NEXT:    s_addc_u32 s1, s1, s35402; GFX7-NEXT:    s_add_u32 s0, s0, 325403; GFX7-NEXT:    s_addc_u32 s1, s1, 05404; GFX7-NEXT:    v_mov_b32_e32 v3, s15405; GFX7-NEXT:    v_mov_b32_e32 v2, s05406; GFX7-NEXT:    flat_atomic_swap_x2 v[2:3], v[0:1]5407; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5408; GFX7-NEXT:    buffer_wbinvl1_vol5409; GFX7-NEXT:    s_endpgm5410;5411; GFX8-LABEL: atomic_xchg_i64_addr64_offset:5412; GFX8:       ; %bb.0: ; %entry5413; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245414; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x345415; GFX8-NEXT:    s_waitcnt lgkmcnt(0)5416; GFX8-NEXT:    v_mov_b32_e32 v0, s25417; GFX8-NEXT:    v_mov_b32_e32 v1, s35418; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 35419; GFX8-NEXT:    s_add_u32 s0, s0, s25420; GFX8-NEXT:    s_addc_u32 s1, s1, s35421; GFX8-NEXT:    s_add_u32 s0, s0, 325422; GFX8-NEXT:    s_addc_u32 s1, s1, 05423; GFX8-NEXT:    v_mov_b32_e32 v3, s15424; GFX8-NEXT:    v_mov_b32_e32 v2, s05425; GFX8-NEXT:    flat_atomic_swap_x2 v[2:3], v[0:1]5426; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5427; GFX8-NEXT:    buffer_wbinvl1_vol5428; GFX8-NEXT:    s_endpgm5429;5430; GFX12-LABEL: atomic_xchg_i64_addr64_offset:5431; GFX12:       ; %bb.0: ; %entry5432; GFX12-NEXT:    s_clause 0x15433; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x345434; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x245435; GFX12-NEXT:    s_wait_kmcnt 0x05436; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 35437; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s35438; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]5439; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)5440; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s05441; GFX12-NEXT:    flat_atomic_swap_b64 v[2:3], v[0:1] offset:32 scope:SCOPE_DEV5442; GFX12-NEXT:    s_wait_storecnt_dscnt 0x05443; GFX12-NEXT:    global_inv scope:SCOPE_DEV5444; GFX12-NEXT:    s_endpgm5445entry:5446  %ptr = getelementptr inbounds i64, ptr %out, i64 %index5447  %gep = getelementptr inbounds i64, ptr %ptr, i64 45448  %tmp0 = atomicrmw volatile xchg ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !05449  ret void5450}5451 5452define amdgpu_kernel void @atomic_xchg_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {5453; GFX7-LABEL: atomic_xchg_i64_ret_addr64_offset:5454; GFX7:       ; %bb.0: ; %entry5455; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x95456; GFX7-NEXT:    s_waitcnt lgkmcnt(0)5457; GFX7-NEXT:    v_mov_b32_e32 v0, s45458; GFX7-NEXT:    v_mov_b32_e32 v1, s55459; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 35460; GFX7-NEXT:    s_add_u32 s0, s0, s45461; GFX7-NEXT:    s_addc_u32 s1, s1, s55462; GFX7-NEXT:    s_add_u32 s0, s0, 325463; GFX7-NEXT:    s_addc_u32 s1, s1, 05464; GFX7-NEXT:    v_mov_b32_e32 v3, s15465; GFX7-NEXT:    v_mov_b32_e32 v2, s05466; GFX7-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc5467; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5468; GFX7-NEXT:    buffer_wbinvl1_vol5469; GFX7-NEXT:    v_mov_b32_e32 v2, s25470; GFX7-NEXT:    v_mov_b32_e32 v3, s35471; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]5472; GFX7-NEXT:    s_endpgm5473;5474; GFX8-LABEL: atomic_xchg_i64_ret_addr64_offset:5475; GFX8:       ; %bb.0: ; %entry5476; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x245477; GFX8-NEXT:    s_waitcnt lgkmcnt(0)5478; GFX8-NEXT:    v_mov_b32_e32 v0, s45479; GFX8-NEXT:    v_mov_b32_e32 v1, s55480; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 35481; GFX8-NEXT:    s_add_u32 s0, s0, s45482; GFX8-NEXT:    s_addc_u32 s1, s1, s55483; GFX8-NEXT:    s_add_u32 s0, s0, 325484; GFX8-NEXT:    s_addc_u32 s1, s1, 05485; GFX8-NEXT:    v_mov_b32_e32 v3, s15486; GFX8-NEXT:    v_mov_b32_e32 v2, s05487; GFX8-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc5488; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5489; GFX8-NEXT:    buffer_wbinvl1_vol5490; GFX8-NEXT:    v_mov_b32_e32 v2, s25491; GFX8-NEXT:    v_mov_b32_e32 v3, s35492; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]5493; GFX8-NEXT:    s_endpgm5494;5495; GFX12-LABEL: atomic_xchg_i64_ret_addr64_offset:5496; GFX12:       ; %bb.0: ; %entry5497; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x245498; GFX12-NEXT:    s_wait_kmcnt 0x05499; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 35500; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s55501; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]5502; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)5503; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s05504; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[2:3], v[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV5505; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x05506; GFX12-NEXT:    global_inv scope:SCOPE_DEV5507; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s35508; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]5509; GFX12-NEXT:    s_endpgm5510entry:5511  %ptr = getelementptr inbounds i64, ptr %out, i64 %index5512  %gep = getelementptr inbounds i64, ptr %ptr, i64 45513  %tmp0 = atomicrmw volatile xchg ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !05514  store i64 %tmp0, ptr %out25515  ret void5516}5517 5518define amdgpu_kernel void @atomic_xchg_i64(ptr %out, i64 %in) {5519; GFX7-LABEL: atomic_xchg_i64:5520; GFX7:       ; %bb.0: ; %entry5521; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x95522; GFX7-NEXT:    s_waitcnt lgkmcnt(0)5523; GFX7-NEXT:    v_mov_b32_e32 v0, s05524; GFX7-NEXT:    v_mov_b32_e32 v1, s15525; GFX7-NEXT:    v_mov_b32_e32 v2, s25526; GFX7-NEXT:    v_mov_b32_e32 v3, s35527; GFX7-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]5528; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5529; GFX7-NEXT:    buffer_wbinvl1_vol5530; GFX7-NEXT:    s_endpgm5531;5532; GFX8-LABEL: atomic_xchg_i64:5533; GFX8:       ; %bb.0: ; %entry5534; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245535; GFX8-NEXT:    s_waitcnt lgkmcnt(0)5536; GFX8-NEXT:    v_mov_b32_e32 v0, s05537; GFX8-NEXT:    v_mov_b32_e32 v1, s15538; GFX8-NEXT:    v_mov_b32_e32 v2, s25539; GFX8-NEXT:    v_mov_b32_e32 v3, s35540; GFX8-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]5541; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5542; GFX8-NEXT:    buffer_wbinvl1_vol5543; GFX8-NEXT:    s_endpgm5544;5545; GFX12-LABEL: atomic_xchg_i64:5546; GFX12:       ; %bb.0: ; %entry5547; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x245548; GFX12-NEXT:    s_wait_kmcnt 0x05549; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15550; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s35551; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV5552; GFX12-NEXT:    s_wait_storecnt_dscnt 0x05553; GFX12-NEXT:    global_inv scope:SCOPE_DEV5554; GFX12-NEXT:    s_endpgm5555entry:5556  %tmp0 = atomicrmw volatile xchg ptr %out, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !05557  ret void5558}5559 5560define amdgpu_kernel void @atomic_xchg_i64_ret(ptr %out, ptr %out2, i64 %in) {5561; GFX7-LABEL: atomic_xchg_i64_ret:5562; GFX7:       ; %bb.0: ; %entry5563; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x95564; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd5565; GFX7-NEXT:    s_waitcnt lgkmcnt(0)5566; GFX7-NEXT:    v_mov_b32_e32 v0, s05567; GFX7-NEXT:    v_mov_b32_e32 v1, s15568; GFX7-NEXT:    v_mov_b32_e32 v2, s45569; GFX7-NEXT:    v_mov_b32_e32 v3, s55570; GFX7-NEXT:    flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc5571; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5572; GFX7-NEXT:    buffer_wbinvl1_vol5573; GFX7-NEXT:    v_mov_b32_e32 v2, s25574; GFX7-NEXT:    v_mov_b32_e32 v3, s35575; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]5576; GFX7-NEXT:    s_endpgm5577;5578; GFX8-LABEL: atomic_xchg_i64_ret:5579; GFX8:       ; %bb.0: ; %entry5580; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245581; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x345582; GFX8-NEXT:    s_waitcnt lgkmcnt(0)5583; GFX8-NEXT:    v_mov_b32_e32 v0, s05584; GFX8-NEXT:    v_mov_b32_e32 v1, s15585; GFX8-NEXT:    v_mov_b32_e32 v2, s45586; GFX8-NEXT:    v_mov_b32_e32 v3, s55587; GFX8-NEXT:    flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc5588; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5589; GFX8-NEXT:    buffer_wbinvl1_vol5590; GFX8-NEXT:    v_mov_b32_e32 v2, s25591; GFX8-NEXT:    v_mov_b32_e32 v3, s35592; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]5593; GFX8-NEXT:    s_endpgm5594;5595; GFX12-LABEL: atomic_xchg_i64_ret:5596; GFX12:       ; %bb.0: ; %entry5597; GFX12-NEXT:    s_clause 0x15598; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x245599; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x345600; GFX12-NEXT:    s_wait_kmcnt 0x05601; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15602; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s55603; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV5604; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x05605; GFX12-NEXT:    global_inv scope:SCOPE_DEV5606; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s35607; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]5608; GFX12-NEXT:    s_endpgm5609entry:5610  %tmp0 = atomicrmw volatile xchg ptr %out, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !05611  store i64 %tmp0, ptr %out25612  ret void5613}5614 5615define amdgpu_kernel void @atomic_xchg_i64_addr64(ptr %out, i64 %in, i64 %index) {5616; GFX7-LABEL: atomic_xchg_i64_addr64:5617; GFX7:       ; %bb.0: ; %entry5618; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x95619; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd5620; GFX7-NEXT:    s_waitcnt lgkmcnt(0)5621; GFX7-NEXT:    v_mov_b32_e32 v0, s25622; GFX7-NEXT:    v_mov_b32_e32 v1, s35623; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 35624; GFX7-NEXT:    s_add_u32 s0, s0, s25625; GFX7-NEXT:    s_addc_u32 s1, s1, s35626; GFX7-NEXT:    v_mov_b32_e32 v3, s15627; GFX7-NEXT:    v_mov_b32_e32 v2, s05628; GFX7-NEXT:    flat_atomic_swap_x2 v[2:3], v[0:1]5629; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5630; GFX7-NEXT:    buffer_wbinvl1_vol5631; GFX7-NEXT:    s_endpgm5632;5633; GFX8-LABEL: atomic_xchg_i64_addr64:5634; GFX8:       ; %bb.0: ; %entry5635; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245636; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x345637; GFX8-NEXT:    s_waitcnt lgkmcnt(0)5638; GFX8-NEXT:    v_mov_b32_e32 v0, s25639; GFX8-NEXT:    v_mov_b32_e32 v1, s35640; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 35641; GFX8-NEXT:    s_add_u32 s0, s0, s25642; GFX8-NEXT:    s_addc_u32 s1, s1, s35643; GFX8-NEXT:    v_mov_b32_e32 v3, s15644; GFX8-NEXT:    v_mov_b32_e32 v2, s05645; GFX8-NEXT:    flat_atomic_swap_x2 v[2:3], v[0:1]5646; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5647; GFX8-NEXT:    buffer_wbinvl1_vol5648; GFX8-NEXT:    s_endpgm5649;5650; GFX12-LABEL: atomic_xchg_i64_addr64:5651; GFX12:       ; %bb.0: ; %entry5652; GFX12-NEXT:    s_clause 0x15653; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x345654; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x245655; GFX12-NEXT:    s_wait_kmcnt 0x05656; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 35657; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s35658; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]5659; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)5660; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s05661; GFX12-NEXT:    flat_atomic_swap_b64 v[2:3], v[0:1] scope:SCOPE_DEV5662; GFX12-NEXT:    s_wait_storecnt_dscnt 0x05663; GFX12-NEXT:    global_inv scope:SCOPE_DEV5664; GFX12-NEXT:    s_endpgm5665entry:5666  %ptr = getelementptr inbounds i64, ptr %out, i64 %index5667  %tmp0 = atomicrmw volatile xchg ptr %ptr, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !05668  ret void5669}5670 5671define amdgpu_kernel void @atomic_xchg_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {5672; GFX7-LABEL: atomic_xchg_i64_ret_addr64:5673; GFX7:       ; %bb.0: ; %entry5674; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x95675; GFX7-NEXT:    s_waitcnt lgkmcnt(0)5676; GFX7-NEXT:    v_mov_b32_e32 v0, s45677; GFX7-NEXT:    v_mov_b32_e32 v1, s55678; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 35679; GFX7-NEXT:    s_add_u32 s0, s0, s45680; GFX7-NEXT:    s_addc_u32 s1, s1, s55681; GFX7-NEXT:    v_mov_b32_e32 v3, s15682; GFX7-NEXT:    v_mov_b32_e32 v2, s05683; GFX7-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc5684; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5685; GFX7-NEXT:    buffer_wbinvl1_vol5686; GFX7-NEXT:    v_mov_b32_e32 v2, s25687; GFX7-NEXT:    v_mov_b32_e32 v3, s35688; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]5689; GFX7-NEXT:    s_endpgm5690;5691; GFX8-LABEL: atomic_xchg_i64_ret_addr64:5692; GFX8:       ; %bb.0: ; %entry5693; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x245694; GFX8-NEXT:    s_waitcnt lgkmcnt(0)5695; GFX8-NEXT:    v_mov_b32_e32 v0, s45696; GFX8-NEXT:    v_mov_b32_e32 v1, s55697; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 35698; GFX8-NEXT:    s_add_u32 s0, s0, s45699; GFX8-NEXT:    s_addc_u32 s1, s1, s55700; GFX8-NEXT:    v_mov_b32_e32 v3, s15701; GFX8-NEXT:    v_mov_b32_e32 v2, s05702; GFX8-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc5703; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5704; GFX8-NEXT:    buffer_wbinvl1_vol5705; GFX8-NEXT:    v_mov_b32_e32 v2, s25706; GFX8-NEXT:    v_mov_b32_e32 v3, s35707; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]5708; GFX8-NEXT:    s_endpgm5709;5710; GFX12-LABEL: atomic_xchg_i64_ret_addr64:5711; GFX12:       ; %bb.0: ; %entry5712; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x245713; GFX12-NEXT:    s_wait_kmcnt 0x05714; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 35715; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s55716; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]5717; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)5718; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s05719; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[2:3], v[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV5720; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x05721; GFX12-NEXT:    global_inv scope:SCOPE_DEV5722; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s35723; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]5724; GFX12-NEXT:    s_endpgm5725entry:5726  %ptr = getelementptr inbounds i64, ptr %out, i64 %index5727  %tmp0 = atomicrmw volatile xchg ptr %ptr, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !05728  store i64 %tmp0, ptr %out25729  ret void5730}5731 5732define amdgpu_kernel void @atomic_xor_i64_offset(ptr %out, i64 %in) {5733; GFX7-LABEL: atomic_xor_i64_offset:5734; GFX7:       ; %bb.0: ; %entry5735; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x95736; GFX7-NEXT:    s_waitcnt lgkmcnt(0)5737; GFX7-NEXT:    s_add_u32 s0, s0, 325738; GFX7-NEXT:    s_addc_u32 s1, s1, 05739; GFX7-NEXT:    v_mov_b32_e32 v5, s15740; GFX7-NEXT:    v_mov_b32_e32 v4, s05741; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]5742; GFX7-NEXT:    s_mov_b64 s[0:1], 05743; GFX7-NEXT:  .LBB74_1: ; %atomicrmw.start5744; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=15745; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5746; GFX7-NEXT:    v_xor_b32_e32 v1, s3, v35747; GFX7-NEXT:    v_xor_b32_e32 v0, s2, v25748; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc5749; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5750; GFX7-NEXT:    buffer_wbinvl1_vol5751; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]5752; GFX7-NEXT:    v_mov_b32_e32 v3, v15753; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]5754; GFX7-NEXT:    v_mov_b32_e32 v2, v05755; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]5756; GFX7-NEXT:    s_cbranch_execnz .LBB74_15757; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end5758; GFX7-NEXT:    s_endpgm5759;5760; GFX8-LABEL: atomic_xor_i64_offset:5761; GFX8:       ; %bb.0: ; %entry5762; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245763; GFX8-NEXT:    s_waitcnt lgkmcnt(0)5764; GFX8-NEXT:    s_add_u32 s0, s0, 325765; GFX8-NEXT:    s_addc_u32 s1, s1, 05766; GFX8-NEXT:    v_mov_b32_e32 v5, s15767; GFX8-NEXT:    v_mov_b32_e32 v4, s05768; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]5769; GFX8-NEXT:    s_mov_b64 s[0:1], 05770; GFX8-NEXT:  .LBB74_1: ; %atomicrmw.start5771; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=15772; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5773; GFX8-NEXT:    v_xor_b32_e32 v1, s3, v35774; GFX8-NEXT:    v_xor_b32_e32 v0, s2, v25775; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc5776; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5777; GFX8-NEXT:    buffer_wbinvl1_vol5778; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]5779; GFX8-NEXT:    v_mov_b32_e32 v3, v15780; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]5781; GFX8-NEXT:    v_mov_b32_e32 v2, v05782; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]5783; GFX8-NEXT:    s_cbranch_execnz .LBB74_15784; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end5785; GFX8-NEXT:    s_endpgm5786;5787; GFX12-LABEL: atomic_xor_i64_offset:5788; GFX12:       ; %bb.0: ; %entry5789; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x245790; GFX12-NEXT:    s_wait_kmcnt 0x05791; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15792; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s35793; GFX12-NEXT:    flat_atomic_xor_b64 v[0:1], v[2:3] offset:32 scope:SCOPE_DEV5794; GFX12-NEXT:    s_wait_storecnt_dscnt 0x05795; GFX12-NEXT:    global_inv scope:SCOPE_DEV5796; GFX12-NEXT:    s_endpgm5797entry:5798  %gep = getelementptr inbounds i64, ptr %out, i64 45799  %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !05800  ret void5801}5802 5803define amdgpu_kernel void @atomic_xor_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {5804; GFX7-LABEL: atomic_xor_i64_ret_offset:5805; GFX7:       ; %bb.0: ; %entry5806; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x95807; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd5808; GFX7-NEXT:    s_waitcnt lgkmcnt(0)5809; GFX7-NEXT:    s_add_u32 s0, s0, 325810; GFX7-NEXT:    s_addc_u32 s1, s1, 05811; GFX7-NEXT:    v_mov_b32_e32 v0, s05812; GFX7-NEXT:    v_mov_b32_e32 v1, s15813; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]5814; GFX7-NEXT:    s_mov_b64 s[0:1], 05815; GFX7-NEXT:  .LBB75_1: ; %atomicrmw.start5816; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=15817; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5818; GFX7-NEXT:    v_mov_b32_e32 v5, v35819; GFX7-NEXT:    v_mov_b32_e32 v4, v25820; GFX7-NEXT:    v_xor_b32_e32 v3, s5, v55821; GFX7-NEXT:    v_xor_b32_e32 v2, s4, v45822; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc5823; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5824; GFX7-NEXT:    buffer_wbinvl1_vol5825; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]5826; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]5827; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]5828; GFX7-NEXT:    s_cbranch_execnz .LBB75_15829; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end5830; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]5831; GFX7-NEXT:    v_mov_b32_e32 v0, s25832; GFX7-NEXT:    v_mov_b32_e32 v1, s35833; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]5834; GFX7-NEXT:    s_endpgm5835;5836; GFX8-LABEL: atomic_xor_i64_ret_offset:5837; GFX8:       ; %bb.0: ; %entry5838; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245839; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x345840; GFX8-NEXT:    s_waitcnt lgkmcnt(0)5841; GFX8-NEXT:    s_add_u32 s0, s0, 325842; GFX8-NEXT:    s_addc_u32 s1, s1, 05843; GFX8-NEXT:    v_mov_b32_e32 v0, s05844; GFX8-NEXT:    v_mov_b32_e32 v1, s15845; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]5846; GFX8-NEXT:    s_mov_b64 s[0:1], 05847; GFX8-NEXT:  .LBB75_1: ; %atomicrmw.start5848; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=15849; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5850; GFX8-NEXT:    v_mov_b32_e32 v5, v35851; GFX8-NEXT:    v_mov_b32_e32 v4, v25852; GFX8-NEXT:    v_xor_b32_e32 v3, s5, v55853; GFX8-NEXT:    v_xor_b32_e32 v2, s4, v45854; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc5855; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5856; GFX8-NEXT:    buffer_wbinvl1_vol5857; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]5858; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]5859; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]5860; GFX8-NEXT:    s_cbranch_execnz .LBB75_15861; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end5862; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]5863; GFX8-NEXT:    v_mov_b32_e32 v0, s25864; GFX8-NEXT:    v_mov_b32_e32 v1, s35865; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]5866; GFX8-NEXT:    s_endpgm5867;5868; GFX12-LABEL: atomic_xor_i64_ret_offset:5869; GFX12:       ; %bb.0: ; %entry5870; GFX12-NEXT:    s_clause 0x15871; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x245872; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x345873; GFX12-NEXT:    s_wait_kmcnt 0x05874; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15875; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s55876; GFX12-NEXT:    flat_atomic_xor_b64 v[0:1], v[0:1], v[2:3] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV5877; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x05878; GFX12-NEXT:    global_inv scope:SCOPE_DEV5879; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s35880; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]5881; GFX12-NEXT:    s_endpgm5882entry:5883  %gep = getelementptr inbounds i64, ptr %out, i64 45884  %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !05885  store i64 %tmp0, ptr %out25886  ret void5887}5888 5889define amdgpu_kernel void @atomic_xor_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {5890; GFX7-LABEL: atomic_xor_i64_addr64_offset:5891; GFX7:       ; %bb.0: ; %entry5892; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd5893; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x95894; GFX7-NEXT:    s_waitcnt lgkmcnt(0)5895; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 35896; GFX7-NEXT:    s_add_u32 s0, s0, s45897; GFX7-NEXT:    s_addc_u32 s1, s1, s55898; GFX7-NEXT:    s_add_u32 s0, s0, 325899; GFX7-NEXT:    s_addc_u32 s1, s1, 05900; GFX7-NEXT:    v_mov_b32_e32 v5, s15901; GFX7-NEXT:    v_mov_b32_e32 v4, s05902; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]5903; GFX7-NEXT:    s_mov_b64 s[0:1], 05904; GFX7-NEXT:  .LBB76_1: ; %atomicrmw.start5905; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=15906; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5907; GFX7-NEXT:    v_xor_b32_e32 v1, s3, v35908; GFX7-NEXT:    v_xor_b32_e32 v0, s2, v25909; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc5910; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5911; GFX7-NEXT:    buffer_wbinvl1_vol5912; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]5913; GFX7-NEXT:    v_mov_b32_e32 v3, v15914; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]5915; GFX7-NEXT:    v_mov_b32_e32 v2, v05916; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]5917; GFX7-NEXT:    s_cbranch_execnz .LBB76_15918; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end5919; GFX7-NEXT:    s_endpgm5920;5921; GFX8-LABEL: atomic_xor_i64_addr64_offset:5922; GFX8:       ; %bb.0: ; %entry5923; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x345924; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245925; GFX8-NEXT:    s_waitcnt lgkmcnt(0)5926; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 35927; GFX8-NEXT:    s_add_u32 s0, s0, s45928; GFX8-NEXT:    s_addc_u32 s1, s1, s55929; GFX8-NEXT:    s_add_u32 s0, s0, 325930; GFX8-NEXT:    s_addc_u32 s1, s1, 05931; GFX8-NEXT:    v_mov_b32_e32 v5, s15932; GFX8-NEXT:    v_mov_b32_e32 v4, s05933; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]5934; GFX8-NEXT:    s_mov_b64 s[0:1], 05935; GFX8-NEXT:  .LBB76_1: ; %atomicrmw.start5936; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=15937; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5938; GFX8-NEXT:    v_xor_b32_e32 v1, s3, v35939; GFX8-NEXT:    v_xor_b32_e32 v0, s2, v25940; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc5941; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5942; GFX8-NEXT:    buffer_wbinvl1_vol5943; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]5944; GFX8-NEXT:    v_mov_b32_e32 v3, v15945; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]5946; GFX8-NEXT:    v_mov_b32_e32 v2, v05947; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]5948; GFX8-NEXT:    s_cbranch_execnz .LBB76_15949; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end5950; GFX8-NEXT:    s_endpgm5951;5952; GFX12-LABEL: atomic_xor_i64_addr64_offset:5953; GFX12:       ; %bb.0: ; %entry5954; GFX12-NEXT:    s_clause 0x15955; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x345956; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x245957; GFX12-NEXT:    s_wait_kmcnt 0x05958; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 35959; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s35960; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]5961; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)5962; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s05963; GFX12-NEXT:    flat_atomic_xor_b64 v[2:3], v[0:1] offset:32 scope:SCOPE_DEV5964; GFX12-NEXT:    s_wait_storecnt_dscnt 0x05965; GFX12-NEXT:    global_inv scope:SCOPE_DEV5966; GFX12-NEXT:    s_endpgm5967entry:5968  %ptr = getelementptr inbounds i64, ptr %out, i64 %index5969  %gep = getelementptr inbounds i64, ptr %ptr, i64 45970  %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !05971  ret void5972}5973 5974define amdgpu_kernel void @atomic_xor_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {5975; GFX7-LABEL: atomic_xor_i64_ret_addr64_offset:5976; GFX7:       ; %bb.0: ; %entry5977; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x95978; GFX7-NEXT:    s_waitcnt lgkmcnt(0)5979; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 35980; GFX7-NEXT:    s_add_u32 s0, s0, s65981; GFX7-NEXT:    s_addc_u32 s1, s1, s75982; GFX7-NEXT:    s_add_u32 s0, s0, 325983; GFX7-NEXT:    s_addc_u32 s1, s1, 05984; GFX7-NEXT:    v_mov_b32_e32 v0, s05985; GFX7-NEXT:    v_mov_b32_e32 v1, s15986; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]5987; GFX7-NEXT:    s_mov_b64 s[0:1], 05988; GFX7-NEXT:  .LBB77_1: ; %atomicrmw.start5989; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=15990; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5991; GFX7-NEXT:    v_mov_b32_e32 v5, v35992; GFX7-NEXT:    v_mov_b32_e32 v4, v25993; GFX7-NEXT:    v_xor_b32_e32 v3, s5, v55994; GFX7-NEXT:    v_xor_b32_e32 v2, s4, v45995; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc5996; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)5997; GFX7-NEXT:    buffer_wbinvl1_vol5998; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]5999; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]6000; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]6001; GFX7-NEXT:    s_cbranch_execnz .LBB77_16002; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end6003; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]6004; GFX7-NEXT:    v_mov_b32_e32 v0, s26005; GFX7-NEXT:    v_mov_b32_e32 v1, s36006; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]6007; GFX7-NEXT:    s_endpgm6008;6009; GFX8-LABEL: atomic_xor_i64_ret_addr64_offset:6010; GFX8:       ; %bb.0: ; %entry6011; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x246012; GFX8-NEXT:    s_waitcnt lgkmcnt(0)6013; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 36014; GFX8-NEXT:    s_add_u32 s0, s0, s66015; GFX8-NEXT:    s_addc_u32 s1, s1, s76016; GFX8-NEXT:    s_add_u32 s0, s0, 326017; GFX8-NEXT:    s_addc_u32 s1, s1, 06018; GFX8-NEXT:    v_mov_b32_e32 v0, s06019; GFX8-NEXT:    v_mov_b32_e32 v1, s16020; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]6021; GFX8-NEXT:    s_mov_b64 s[0:1], 06022; GFX8-NEXT:  .LBB77_1: ; %atomicrmw.start6023; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=16024; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6025; GFX8-NEXT:    v_mov_b32_e32 v5, v36026; GFX8-NEXT:    v_mov_b32_e32 v4, v26027; GFX8-NEXT:    v_xor_b32_e32 v3, s5, v56028; GFX8-NEXT:    v_xor_b32_e32 v2, s4, v46029; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc6030; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6031; GFX8-NEXT:    buffer_wbinvl1_vol6032; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]6033; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]6034; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]6035; GFX8-NEXT:    s_cbranch_execnz .LBB77_16036; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end6037; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]6038; GFX8-NEXT:    v_mov_b32_e32 v0, s26039; GFX8-NEXT:    v_mov_b32_e32 v1, s36040; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]6041; GFX8-NEXT:    s_endpgm6042;6043; GFX12-LABEL: atomic_xor_i64_ret_addr64_offset:6044; GFX12:       ; %bb.0: ; %entry6045; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x246046; GFX12-NEXT:    s_wait_kmcnt 0x06047; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 36048; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s56049; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]6050; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)6051; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s06052; GFX12-NEXT:    flat_atomic_xor_b64 v[0:1], v[2:3], v[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV6053; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x06054; GFX12-NEXT:    global_inv scope:SCOPE_DEV6055; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s36056; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]6057; GFX12-NEXT:    s_endpgm6058entry:6059  %ptr = getelementptr inbounds i64, ptr %out, i64 %index6060  %gep = getelementptr inbounds i64, ptr %ptr, i64 46061  %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !06062  store i64 %tmp0, ptr %out26063  ret void6064}6065 6066define amdgpu_kernel void @atomic_xor_i64(ptr %out, i64 %in) {6067; GFX7-LABEL: atomic_xor_i64:6068; GFX7:       ; %bb.0: ; %entry6069; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96070; GFX7-NEXT:    s_mov_b64 s[4:5], 06071; GFX7-NEXT:    s_waitcnt lgkmcnt(0)6072; GFX7-NEXT:    v_mov_b32_e32 v0, s06073; GFX7-NEXT:    v_mov_b32_e32 v1, s16074; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]6075; GFX7-NEXT:    v_mov_b32_e32 v5, s16076; GFX7-NEXT:    v_mov_b32_e32 v4, s06077; GFX7-NEXT:  .LBB78_1: ; %atomicrmw.start6078; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=16079; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6080; GFX7-NEXT:    v_xor_b32_e32 v1, s3, v36081; GFX7-NEXT:    v_xor_b32_e32 v0, s2, v26082; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc6083; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6084; GFX7-NEXT:    buffer_wbinvl1_vol6085; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6086; GFX7-NEXT:    v_mov_b32_e32 v3, v16087; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]6088; GFX7-NEXT:    v_mov_b32_e32 v2, v06089; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]6090; GFX7-NEXT:    s_cbranch_execnz .LBB78_16091; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end6092; GFX7-NEXT:    s_endpgm6093;6094; GFX8-LABEL: atomic_xor_i64:6095; GFX8:       ; %bb.0: ; %entry6096; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246097; GFX8-NEXT:    s_mov_b64 s[4:5], 06098; GFX8-NEXT:    s_waitcnt lgkmcnt(0)6099; GFX8-NEXT:    v_mov_b32_e32 v0, s06100; GFX8-NEXT:    v_mov_b32_e32 v1, s16101; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]6102; GFX8-NEXT:    v_mov_b32_e32 v5, s16103; GFX8-NEXT:    v_mov_b32_e32 v4, s06104; GFX8-NEXT:  .LBB78_1: ; %atomicrmw.start6105; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=16106; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6107; GFX8-NEXT:    v_xor_b32_e32 v1, s3, v36108; GFX8-NEXT:    v_xor_b32_e32 v0, s2, v26109; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc6110; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6111; GFX8-NEXT:    buffer_wbinvl1_vol6112; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6113; GFX8-NEXT:    v_mov_b32_e32 v3, v16114; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]6115; GFX8-NEXT:    v_mov_b32_e32 v2, v06116; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]6117; GFX8-NEXT:    s_cbranch_execnz .LBB78_16118; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end6119; GFX8-NEXT:    s_endpgm6120;6121; GFX12-LABEL: atomic_xor_i64:6122; GFX12:       ; %bb.0: ; %entry6123; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246124; GFX12-NEXT:    s_wait_kmcnt 0x06125; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16126; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s36127; GFX12-NEXT:    flat_atomic_xor_b64 v[0:1], v[2:3] scope:SCOPE_DEV6128; GFX12-NEXT:    s_wait_storecnt_dscnt 0x06129; GFX12-NEXT:    global_inv scope:SCOPE_DEV6130; GFX12-NEXT:    s_endpgm6131entry:6132  %tmp0 = atomicrmw volatile xor ptr %out, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !06133  ret void6134}6135 6136define amdgpu_kernel void @atomic_xor_i64_ret(ptr %out, ptr %out2, i64 %in) {6137; GFX7-LABEL: atomic_xor_i64_ret:6138; GFX7:       ; %bb.0: ; %entry6139; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96140; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd6141; GFX7-NEXT:    s_mov_b64 s[6:7], 06142; GFX7-NEXT:    s_waitcnt lgkmcnt(0)6143; GFX7-NEXT:    v_mov_b32_e32 v0, s06144; GFX7-NEXT:    v_mov_b32_e32 v1, s16145; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]6146; GFX7-NEXT:    v_mov_b32_e32 v3, s16147; GFX7-NEXT:    v_mov_b32_e32 v2, s06148; GFX7-NEXT:  .LBB79_1: ; %atomicrmw.start6149; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=16150; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6151; GFX7-NEXT:    v_mov_b32_e32 v7, v16152; GFX7-NEXT:    v_mov_b32_e32 v6, v06153; GFX7-NEXT:    v_xor_b32_e32 v5, s5, v76154; GFX7-NEXT:    v_xor_b32_e32 v4, s4, v66155; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc6156; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6157; GFX7-NEXT:    buffer_wbinvl1_vol6158; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]6159; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]6160; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]6161; GFX7-NEXT:    s_cbranch_execnz .LBB79_16162; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end6163; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]6164; GFX7-NEXT:    v_mov_b32_e32 v2, s26165; GFX7-NEXT:    v_mov_b32_e32 v3, s36166; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]6167; GFX7-NEXT:    s_endpgm6168;6169; GFX8-LABEL: atomic_xor_i64_ret:6170; GFX8:       ; %bb.0: ; %entry6171; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246172; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x346173; GFX8-NEXT:    s_mov_b64 s[6:7], 06174; GFX8-NEXT:    s_waitcnt lgkmcnt(0)6175; GFX8-NEXT:    v_mov_b32_e32 v0, s06176; GFX8-NEXT:    v_mov_b32_e32 v1, s16177; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]6178; GFX8-NEXT:    v_mov_b32_e32 v3, s16179; GFX8-NEXT:    v_mov_b32_e32 v2, s06180; GFX8-NEXT:  .LBB79_1: ; %atomicrmw.start6181; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=16182; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6183; GFX8-NEXT:    v_mov_b32_e32 v7, v16184; GFX8-NEXT:    v_mov_b32_e32 v6, v06185; GFX8-NEXT:    v_xor_b32_e32 v5, s5, v76186; GFX8-NEXT:    v_xor_b32_e32 v4, s4, v66187; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc6188; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6189; GFX8-NEXT:    buffer_wbinvl1_vol6190; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]6191; GFX8-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]6192; GFX8-NEXT:    s_andn2_b64 exec, exec, s[6:7]6193; GFX8-NEXT:    s_cbranch_execnz .LBB79_16194; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end6195; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]6196; GFX8-NEXT:    v_mov_b32_e32 v2, s26197; GFX8-NEXT:    v_mov_b32_e32 v3, s36198; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]6199; GFX8-NEXT:    s_endpgm6200;6201; GFX12-LABEL: atomic_xor_i64_ret:6202; GFX12:       ; %bb.0: ; %entry6203; GFX12-NEXT:    s_clause 0x16204; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246205; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x346206; GFX12-NEXT:    s_wait_kmcnt 0x06207; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16208; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s56209; GFX12-NEXT:    flat_atomic_xor_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV6210; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x06211; GFX12-NEXT:    global_inv scope:SCOPE_DEV6212; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s36213; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]6214; GFX12-NEXT:    s_endpgm6215entry:6216  %tmp0 = atomicrmw volatile xor ptr %out, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !06217  store i64 %tmp0, ptr %out26218  ret void6219}6220 6221define amdgpu_kernel void @atomic_xor_i64_addr64(ptr %out, i64 %in, i64 %index) {6222; GFX7-LABEL: atomic_xor_i64_addr64:6223; GFX7:       ; %bb.0: ; %entry6224; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd6225; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96226; GFX7-NEXT:    s_waitcnt lgkmcnt(0)6227; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 36228; GFX7-NEXT:    s_add_u32 s0, s0, s46229; GFX7-NEXT:    s_addc_u32 s1, s1, s56230; GFX7-NEXT:    v_mov_b32_e32 v5, s16231; GFX7-NEXT:    v_mov_b32_e32 v4, s06232; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]6233; GFX7-NEXT:    s_mov_b64 s[0:1], 06234; GFX7-NEXT:  .LBB80_1: ; %atomicrmw.start6235; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=16236; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6237; GFX7-NEXT:    v_xor_b32_e32 v1, s3, v36238; GFX7-NEXT:    v_xor_b32_e32 v0, s2, v26239; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc6240; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6241; GFX7-NEXT:    buffer_wbinvl1_vol6242; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6243; GFX7-NEXT:    v_mov_b32_e32 v3, v16244; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]6245; GFX7-NEXT:    v_mov_b32_e32 v2, v06246; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]6247; GFX7-NEXT:    s_cbranch_execnz .LBB80_16248; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end6249; GFX7-NEXT:    s_endpgm6250;6251; GFX8-LABEL: atomic_xor_i64_addr64:6252; GFX8:       ; %bb.0: ; %entry6253; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x346254; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246255; GFX8-NEXT:    s_waitcnt lgkmcnt(0)6256; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 36257; GFX8-NEXT:    s_add_u32 s0, s0, s46258; GFX8-NEXT:    s_addc_u32 s1, s1, s56259; GFX8-NEXT:    v_mov_b32_e32 v5, s16260; GFX8-NEXT:    v_mov_b32_e32 v4, s06261; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]6262; GFX8-NEXT:    s_mov_b64 s[0:1], 06263; GFX8-NEXT:  .LBB80_1: ; %atomicrmw.start6264; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=16265; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6266; GFX8-NEXT:    v_xor_b32_e32 v1, s3, v36267; GFX8-NEXT:    v_xor_b32_e32 v0, s2, v26268; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc6269; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6270; GFX8-NEXT:    buffer_wbinvl1_vol6271; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6272; GFX8-NEXT:    v_mov_b32_e32 v3, v16273; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]6274; GFX8-NEXT:    v_mov_b32_e32 v2, v06275; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]6276; GFX8-NEXT:    s_cbranch_execnz .LBB80_16277; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end6278; GFX8-NEXT:    s_endpgm6279;6280; GFX12-LABEL: atomic_xor_i64_addr64:6281; GFX12:       ; %bb.0: ; %entry6282; GFX12-NEXT:    s_clause 0x16283; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x346284; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246285; GFX12-NEXT:    s_wait_kmcnt 0x06286; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 36287; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s36288; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]6289; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)6290; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s06291; GFX12-NEXT:    flat_atomic_xor_b64 v[2:3], v[0:1] scope:SCOPE_DEV6292; GFX12-NEXT:    s_wait_storecnt_dscnt 0x06293; GFX12-NEXT:    global_inv scope:SCOPE_DEV6294; GFX12-NEXT:    s_endpgm6295entry:6296  %ptr = getelementptr inbounds i64, ptr %out, i64 %index6297  %tmp0 = atomicrmw volatile xor ptr %ptr, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !06298  ret void6299}6300 6301define amdgpu_kernel void @atomic_xor_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {6302; GFX7-LABEL: atomic_xor_i64_ret_addr64:6303; GFX7:       ; %bb.0: ; %entry6304; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x96305; GFX7-NEXT:    s_waitcnt lgkmcnt(0)6306; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 36307; GFX7-NEXT:    s_add_u32 s0, s0, s66308; GFX7-NEXT:    s_addc_u32 s1, s1, s76309; GFX7-NEXT:    v_mov_b32_e32 v0, s06310; GFX7-NEXT:    v_mov_b32_e32 v1, s16311; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]6312; GFX7-NEXT:    s_mov_b64 s[0:1], 06313; GFX7-NEXT:  .LBB81_1: ; %atomicrmw.start6314; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=16315; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6316; GFX7-NEXT:    v_mov_b32_e32 v5, v36317; GFX7-NEXT:    v_mov_b32_e32 v4, v26318; GFX7-NEXT:    v_xor_b32_e32 v3, s5, v56319; GFX7-NEXT:    v_xor_b32_e32 v2, s4, v46320; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc6321; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6322; GFX7-NEXT:    buffer_wbinvl1_vol6323; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]6324; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]6325; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]6326; GFX7-NEXT:    s_cbranch_execnz .LBB81_16327; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end6328; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]6329; GFX7-NEXT:    v_mov_b32_e32 v0, s26330; GFX7-NEXT:    v_mov_b32_e32 v1, s36331; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]6332; GFX7-NEXT:    s_endpgm6333;6334; GFX8-LABEL: atomic_xor_i64_ret_addr64:6335; GFX8:       ; %bb.0: ; %entry6336; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x246337; GFX8-NEXT:    s_waitcnt lgkmcnt(0)6338; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 36339; GFX8-NEXT:    s_add_u32 s0, s0, s66340; GFX8-NEXT:    s_addc_u32 s1, s1, s76341; GFX8-NEXT:    v_mov_b32_e32 v0, s06342; GFX8-NEXT:    v_mov_b32_e32 v1, s16343; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]6344; GFX8-NEXT:    s_mov_b64 s[0:1], 06345; GFX8-NEXT:  .LBB81_1: ; %atomicrmw.start6346; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=16347; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6348; GFX8-NEXT:    v_mov_b32_e32 v5, v36349; GFX8-NEXT:    v_mov_b32_e32 v4, v26350; GFX8-NEXT:    v_xor_b32_e32 v3, s5, v56351; GFX8-NEXT:    v_xor_b32_e32 v2, s4, v46352; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc6353; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6354; GFX8-NEXT:    buffer_wbinvl1_vol6355; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]6356; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]6357; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]6358; GFX8-NEXT:    s_cbranch_execnz .LBB81_16359; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end6360; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]6361; GFX8-NEXT:    v_mov_b32_e32 v0, s26362; GFX8-NEXT:    v_mov_b32_e32 v1, s36363; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]6364; GFX8-NEXT:    s_endpgm6365;6366; GFX12-LABEL: atomic_xor_i64_ret_addr64:6367; GFX12:       ; %bb.0: ; %entry6368; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x246369; GFX12-NEXT:    s_wait_kmcnt 0x06370; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 36371; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s56372; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]6373; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)6374; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s06375; GFX12-NEXT:    flat_atomic_xor_b64 v[0:1], v[2:3], v[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV6376; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x06377; GFX12-NEXT:    global_inv scope:SCOPE_DEV6378; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s36379; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]6380; GFX12-NEXT:    s_endpgm6381entry:6382  %ptr = getelementptr inbounds i64, ptr %out, i64 %index6383  %tmp0 = atomicrmw volatile xor ptr %ptr, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !06384  store i64 %tmp0, ptr %out26385  ret void6386}6387 6388define amdgpu_kernel void @atomic_load_i64_offset(ptr %in, ptr %out) {6389; GFX7-LABEL: atomic_load_i64_offset:6390; GFX7:       ; %bb.0: ; %entry6391; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96392; GFX7-NEXT:    s_waitcnt lgkmcnt(0)6393; GFX7-NEXT:    s_add_u32 s0, s0, 326394; GFX7-NEXT:    s_addc_u32 s1, s1, 06395; GFX7-NEXT:    v_mov_b32_e32 v0, s06396; GFX7-NEXT:    v_mov_b32_e32 v1, s16397; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc6398; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6399; GFX7-NEXT:    buffer_wbinvl1_vol6400; GFX7-NEXT:    v_mov_b32_e32 v2, s26401; GFX7-NEXT:    v_mov_b32_e32 v3, s36402; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]6403; GFX7-NEXT:    s_endpgm6404;6405; GFX8-LABEL: atomic_load_i64_offset:6406; GFX8:       ; %bb.0: ; %entry6407; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246408; GFX8-NEXT:    s_waitcnt lgkmcnt(0)6409; GFX8-NEXT:    s_add_u32 s0, s0, 326410; GFX8-NEXT:    s_addc_u32 s1, s1, 06411; GFX8-NEXT:    v_mov_b32_e32 v0, s06412; GFX8-NEXT:    v_mov_b32_e32 v1, s16413; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc6414; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6415; GFX8-NEXT:    buffer_wbinvl1_vol6416; GFX8-NEXT:    v_mov_b32_e32 v2, s26417; GFX8-NEXT:    v_mov_b32_e32 v3, s36418; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]6419; GFX8-NEXT:    s_endpgm6420;6421; GFX12-LABEL: atomic_load_i64_offset:6422; GFX12:       ; %bb.0: ; %entry6423; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246424; GFX12-NEXT:    s_wait_kmcnt 0x06425; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16426; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s36427; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] offset:32 scope:SCOPE_SYS6428; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x06429; GFX12-NEXT:    global_inv scope:SCOPE_SYS6430; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]6431; GFX12-NEXT:    s_endpgm6432entry:6433  %gep = getelementptr inbounds i64, ptr %in, i64 46434  %val = load atomic i64, ptr %gep  seq_cst, align 86435  store i64 %val, ptr %out6436  ret void6437}6438 6439define amdgpu_kernel void @atomic_load_i64(ptr %in, ptr %out) {6440; GFX7-LABEL: atomic_load_i64:6441; GFX7:       ; %bb.0: ; %entry6442; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96443; GFX7-NEXT:    s_waitcnt lgkmcnt(0)6444; GFX7-NEXT:    v_mov_b32_e32 v0, s06445; GFX7-NEXT:    v_mov_b32_e32 v1, s16446; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc6447; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6448; GFX7-NEXT:    buffer_wbinvl1_vol6449; GFX7-NEXT:    v_mov_b32_e32 v2, s26450; GFX7-NEXT:    v_mov_b32_e32 v3, s36451; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]6452; GFX7-NEXT:    s_endpgm6453;6454; GFX8-LABEL: atomic_load_i64:6455; GFX8:       ; %bb.0: ; %entry6456; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246457; GFX8-NEXT:    s_waitcnt lgkmcnt(0)6458; GFX8-NEXT:    v_mov_b32_e32 v0, s06459; GFX8-NEXT:    v_mov_b32_e32 v1, s16460; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc6461; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6462; GFX8-NEXT:    buffer_wbinvl1_vol6463; GFX8-NEXT:    v_mov_b32_e32 v2, s26464; GFX8-NEXT:    v_mov_b32_e32 v3, s36465; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]6466; GFX8-NEXT:    s_endpgm6467;6468; GFX12-LABEL: atomic_load_i64:6469; GFX12:       ; %bb.0: ; %entry6470; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246471; GFX12-NEXT:    s_wait_kmcnt 0x06472; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16473; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s36474; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV6475; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x06476; GFX12-NEXT:    global_inv scope:SCOPE_DEV6477; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]6478; GFX12-NEXT:    s_endpgm6479entry:6480  %val = load atomic i64, ptr %in syncscope("agent") seq_cst, align 86481  store i64 %val, ptr %out6482  ret void6483}6484 6485define amdgpu_kernel void @atomic_load_i64_addr64_offset(ptr %in, ptr %out, i64 %index) {6486; GFX7-LABEL: atomic_load_i64_addr64_offset:6487; GFX7:       ; %bb.0: ; %entry6488; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd6489; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96490; GFX7-NEXT:    s_waitcnt lgkmcnt(0)6491; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 36492; GFX7-NEXT:    s_add_u32 s0, s0, s46493; GFX7-NEXT:    s_addc_u32 s1, s1, s56494; GFX7-NEXT:    s_add_u32 s0, s0, 326495; GFX7-NEXT:    s_addc_u32 s1, s1, 06496; GFX7-NEXT:    v_mov_b32_e32 v0, s06497; GFX7-NEXT:    v_mov_b32_e32 v1, s16498; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc6499; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6500; GFX7-NEXT:    buffer_wbinvl1_vol6501; GFX7-NEXT:    v_mov_b32_e32 v2, s26502; GFX7-NEXT:    v_mov_b32_e32 v3, s36503; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]6504; GFX7-NEXT:    s_endpgm6505;6506; GFX8-LABEL: atomic_load_i64_addr64_offset:6507; GFX8:       ; %bb.0: ; %entry6508; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x346509; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246510; GFX8-NEXT:    s_waitcnt lgkmcnt(0)6511; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 36512; GFX8-NEXT:    s_add_u32 s0, s0, s46513; GFX8-NEXT:    s_addc_u32 s1, s1, s56514; GFX8-NEXT:    s_add_u32 s0, s0, 326515; GFX8-NEXT:    s_addc_u32 s1, s1, 06516; GFX8-NEXT:    v_mov_b32_e32 v0, s06517; GFX8-NEXT:    v_mov_b32_e32 v1, s16518; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc6519; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6520; GFX8-NEXT:    buffer_wbinvl1_vol6521; GFX8-NEXT:    v_mov_b32_e32 v2, s26522; GFX8-NEXT:    v_mov_b32_e32 v3, s36523; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]6524; GFX8-NEXT:    s_endpgm6525;6526; GFX12-LABEL: atomic_load_i64_addr64_offset:6527; GFX12:       ; %bb.0: ; %entry6528; GFX12-NEXT:    s_clause 0x16529; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x346530; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246531; GFX12-NEXT:    s_wait_kmcnt 0x06532; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 36533; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s36534; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]6535; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)6536; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16537; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] offset:32 scope:SCOPE_SYS6538; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x06539; GFX12-NEXT:    global_inv scope:SCOPE_SYS6540; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]6541; GFX12-NEXT:    s_endpgm6542entry:6543  %ptr = getelementptr inbounds i64, ptr %in, i64 %index6544  %gep = getelementptr inbounds i64, ptr %ptr, i64 46545  %val = load atomic i64, ptr %gep seq_cst, align 86546  store i64 %val, ptr %out6547  ret void6548}6549 6550define amdgpu_kernel void @atomic_load_i64_addr64(ptr %in, ptr %out, i64 %index) {6551; GFX7-LABEL: atomic_load_i64_addr64:6552; GFX7:       ; %bb.0: ; %entry6553; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd6554; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96555; GFX7-NEXT:    s_waitcnt lgkmcnt(0)6556; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 36557; GFX7-NEXT:    s_add_u32 s0, s0, s46558; GFX7-NEXT:    s_addc_u32 s1, s1, s56559; GFX7-NEXT:    v_mov_b32_e32 v0, s06560; GFX7-NEXT:    v_mov_b32_e32 v1, s16561; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc6562; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6563; GFX7-NEXT:    buffer_wbinvl1_vol6564; GFX7-NEXT:    v_mov_b32_e32 v2, s26565; GFX7-NEXT:    v_mov_b32_e32 v3, s36566; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]6567; GFX7-NEXT:    s_endpgm6568;6569; GFX8-LABEL: atomic_load_i64_addr64:6570; GFX8:       ; %bb.0: ; %entry6571; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x346572; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246573; GFX8-NEXT:    s_waitcnt lgkmcnt(0)6574; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 36575; GFX8-NEXT:    s_add_u32 s0, s0, s46576; GFX8-NEXT:    s_addc_u32 s1, s1, s56577; GFX8-NEXT:    v_mov_b32_e32 v0, s06578; GFX8-NEXT:    v_mov_b32_e32 v1, s16579; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc6580; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6581; GFX8-NEXT:    buffer_wbinvl1_vol6582; GFX8-NEXT:    v_mov_b32_e32 v2, s26583; GFX8-NEXT:    v_mov_b32_e32 v3, s36584; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]6585; GFX8-NEXT:    s_endpgm6586;6587; GFX12-LABEL: atomic_load_i64_addr64:6588; GFX12:       ; %bb.0: ; %entry6589; GFX12-NEXT:    s_clause 0x16590; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x346591; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246592; GFX12-NEXT:    s_wait_kmcnt 0x06593; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 36594; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s36595; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]6596; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)6597; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16598; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] scope:SCOPE_SYS6599; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x06600; GFX12-NEXT:    global_inv scope:SCOPE_SYS6601; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]6602; GFX12-NEXT:    s_endpgm6603entry:6604  %ptr = getelementptr inbounds i64, ptr %in, i64 %index6605  %val = load atomic i64, ptr %ptr seq_cst, align 86606  store i64 %val, ptr %out6607  ret void6608}6609 6610define amdgpu_kernel void @atomic_store_i64_offset(i64 %in, ptr %out) {6611; GFX7-LABEL: atomic_store_i64_offset:6612; GFX7:       ; %bb.0: ; %entry6613; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96614; GFX7-NEXT:    s_waitcnt lgkmcnt(0)6615; GFX7-NEXT:    v_mov_b32_e32 v0, s06616; GFX7-NEXT:    s_add_u32 s0, s2, 326617; GFX7-NEXT:    v_mov_b32_e32 v1, s16618; GFX7-NEXT:    s_addc_u32 s1, s3, 06619; GFX7-NEXT:    v_mov_b32_e32 v3, s16620; GFX7-NEXT:    v_mov_b32_e32 v2, s06621; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]6622; GFX7-NEXT:    s_endpgm6623;6624; GFX8-LABEL: atomic_store_i64_offset:6625; GFX8:       ; %bb.0: ; %entry6626; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246627; GFX8-NEXT:    s_waitcnt lgkmcnt(0)6628; GFX8-NEXT:    v_mov_b32_e32 v0, s06629; GFX8-NEXT:    s_add_u32 s0, s2, 326630; GFX8-NEXT:    v_mov_b32_e32 v1, s16631; GFX8-NEXT:    s_addc_u32 s1, s3, 06632; GFX8-NEXT:    v_mov_b32_e32 v3, s16633; GFX8-NEXT:    v_mov_b32_e32 v2, s06634; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]6635; GFX8-NEXT:    s_endpgm6636;6637; GFX12-LABEL: atomic_store_i64_offset:6638; GFX12:       ; %bb.0: ; %entry6639; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246640; GFX12-NEXT:    s_wait_kmcnt 0x06641; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16642; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s36643; GFX12-NEXT:    global_wb scope:SCOPE_SYS6644; GFX12-NEXT:    s_wait_storecnt 0x06645; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] offset:32 scope:SCOPE_SYS6646; GFX12-NEXT:    s_endpgm6647entry:6648  %gep = getelementptr inbounds i64, ptr %out, i64 46649  store atomic i64 %in, ptr %gep  seq_cst, align 86650  ret void6651}6652 6653define amdgpu_kernel void @atomic_store_i64(i64 %in, ptr %out) {6654; GFX7-LABEL: atomic_store_i64:6655; GFX7:       ; %bb.0: ; %entry6656; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96657; GFX7-NEXT:    s_waitcnt lgkmcnt(0)6658; GFX7-NEXT:    v_mov_b32_e32 v0, s06659; GFX7-NEXT:    v_mov_b32_e32 v1, s16660; GFX7-NEXT:    v_mov_b32_e32 v2, s26661; GFX7-NEXT:    v_mov_b32_e32 v3, s36662; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]6663; GFX7-NEXT:    s_endpgm6664;6665; GFX8-LABEL: atomic_store_i64:6666; GFX8:       ; %bb.0: ; %entry6667; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246668; GFX8-NEXT:    s_waitcnt lgkmcnt(0)6669; GFX8-NEXT:    v_mov_b32_e32 v0, s06670; GFX8-NEXT:    v_mov_b32_e32 v1, s16671; GFX8-NEXT:    v_mov_b32_e32 v2, s26672; GFX8-NEXT:    v_mov_b32_e32 v3, s36673; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]6674; GFX8-NEXT:    s_endpgm6675;6676; GFX12-LABEL: atomic_store_i64:6677; GFX12:       ; %bb.0: ; %entry6678; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246679; GFX12-NEXT:    s_wait_kmcnt 0x06680; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16681; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s36682; GFX12-NEXT:    global_wb scope:SCOPE_SYS6683; GFX12-NEXT:    s_wait_storecnt 0x06684; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] scope:SCOPE_SYS6685; GFX12-NEXT:    s_endpgm6686entry:6687  store atomic i64 %in, ptr %out seq_cst, align 86688  ret void6689}6690 6691define amdgpu_kernel void @atomic_store_i64_addr64_offset(i64 %in, ptr %out, i64 %index) {6692; GFX7-LABEL: atomic_store_i64_addr64_offset:6693; GFX7:       ; %bb.0: ; %entry6694; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96695; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd6696; GFX7-NEXT:    s_waitcnt lgkmcnt(0)6697; GFX7-NEXT:    v_mov_b32_e32 v0, s06698; GFX7-NEXT:    v_mov_b32_e32 v1, s16699; GFX7-NEXT:    s_lshl_b64 s[0:1], s[4:5], 36700; GFX7-NEXT:    s_add_u32 s0, s2, s06701; GFX7-NEXT:    s_addc_u32 s1, s3, s16702; GFX7-NEXT:    s_add_u32 s0, s0, 326703; GFX7-NEXT:    s_addc_u32 s1, s1, 06704; GFX7-NEXT:    v_mov_b32_e32 v3, s16705; GFX7-NEXT:    v_mov_b32_e32 v2, s06706; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]6707; GFX7-NEXT:    s_endpgm6708;6709; GFX8-LABEL: atomic_store_i64_addr64_offset:6710; GFX8:       ; %bb.0: ; %entry6711; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246712; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x346713; GFX8-NEXT:    s_waitcnt lgkmcnt(0)6714; GFX8-NEXT:    v_mov_b32_e32 v0, s06715; GFX8-NEXT:    v_mov_b32_e32 v1, s16716; GFX8-NEXT:    s_lshl_b64 s[0:1], s[4:5], 36717; GFX8-NEXT:    s_add_u32 s0, s2, s06718; GFX8-NEXT:    s_addc_u32 s1, s3, s16719; GFX8-NEXT:    s_add_u32 s0, s0, 326720; GFX8-NEXT:    s_addc_u32 s1, s1, 06721; GFX8-NEXT:    v_mov_b32_e32 v3, s16722; GFX8-NEXT:    v_mov_b32_e32 v2, s06723; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]6724; GFX8-NEXT:    s_endpgm6725;6726; GFX12-LABEL: atomic_store_i64_addr64_offset:6727; GFX12:       ; %bb.0: ; %entry6728; GFX12-NEXT:    s_clause 0x16729; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x346730; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246731; GFX12-NEXT:    s_wait_kmcnt 0x06732; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 36733; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16734; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[2:3], s[4:5]6735; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)6736; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s36737; GFX12-NEXT:    global_wb scope:SCOPE_SYS6738; GFX12-NEXT:    s_wait_storecnt 0x06739; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] offset:32 scope:SCOPE_SYS6740; GFX12-NEXT:    s_endpgm6741entry:6742  %ptr = getelementptr inbounds i64, ptr %out, i64 %index6743  %gep = getelementptr inbounds i64, ptr %ptr, i64 46744  store atomic i64 %in, ptr %gep seq_cst, align 86745  ret void6746}6747 6748define amdgpu_kernel void @atomic_store_i64_addr64(i64 %in, ptr %out, i64 %index) {6749; GFX7-LABEL: atomic_store_i64_addr64:6750; GFX7:       ; %bb.0: ; %entry6751; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96752; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd6753; GFX7-NEXT:    s_waitcnt lgkmcnt(0)6754; GFX7-NEXT:    v_mov_b32_e32 v0, s06755; GFX7-NEXT:    v_mov_b32_e32 v1, s16756; GFX7-NEXT:    s_lshl_b64 s[0:1], s[4:5], 36757; GFX7-NEXT:    s_add_u32 s0, s2, s06758; GFX7-NEXT:    s_addc_u32 s1, s3, s16759; GFX7-NEXT:    v_mov_b32_e32 v3, s16760; GFX7-NEXT:    v_mov_b32_e32 v2, s06761; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]6762; GFX7-NEXT:    s_endpgm6763;6764; GFX8-LABEL: atomic_store_i64_addr64:6765; GFX8:       ; %bb.0: ; %entry6766; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246767; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x346768; GFX8-NEXT:    s_waitcnt lgkmcnt(0)6769; GFX8-NEXT:    v_mov_b32_e32 v0, s06770; GFX8-NEXT:    v_mov_b32_e32 v1, s16771; GFX8-NEXT:    s_lshl_b64 s[0:1], s[4:5], 36772; GFX8-NEXT:    s_add_u32 s0, s2, s06773; GFX8-NEXT:    s_addc_u32 s1, s3, s16774; GFX8-NEXT:    v_mov_b32_e32 v3, s16775; GFX8-NEXT:    v_mov_b32_e32 v2, s06776; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]6777; GFX8-NEXT:    s_endpgm6778;6779; GFX12-LABEL: atomic_store_i64_addr64:6780; GFX12:       ; %bb.0: ; %entry6781; GFX12-NEXT:    s_clause 0x16782; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x346783; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246784; GFX12-NEXT:    s_wait_kmcnt 0x06785; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 36786; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16787; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[2:3], s[4:5]6788; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)6789; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s36790; GFX12-NEXT:    global_wb scope:SCOPE_SYS6791; GFX12-NEXT:    s_wait_storecnt 0x06792; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] scope:SCOPE_SYS6793; GFX12-NEXT:    s_endpgm6794entry:6795  %ptr = getelementptr inbounds i64, ptr %out, i64 %index6796  store atomic i64 %in, ptr %ptr seq_cst, align 86797  ret void6798}6799 6800define amdgpu_kernel void @atomic_cmpxchg_i64_offset(ptr %out, i64 %in, i64 %old) {6801; GFX7-LABEL: atomic_cmpxchg_i64_offset:6802; GFX7:       ; %bb.0: ; %entry6803; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96804; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd6805; GFX7-NEXT:    s_waitcnt lgkmcnt(0)6806; GFX7-NEXT:    s_add_u32 s0, s0, 326807; GFX7-NEXT:    s_addc_u32 s1, s1, 06808; GFX7-NEXT:    v_mov_b32_e32 v5, s16809; GFX7-NEXT:    v_mov_b32_e32 v0, s26810; GFX7-NEXT:    v_mov_b32_e32 v1, s36811; GFX7-NEXT:    v_mov_b32_e32 v2, s46812; GFX7-NEXT:    v_mov_b32_e32 v3, s56813; GFX7-NEXT:    v_mov_b32_e32 v4, s06814; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]6815; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6816; GFX7-NEXT:    buffer_wbinvl1_vol6817; GFX7-NEXT:    s_endpgm6818;6819; GFX8-LABEL: atomic_cmpxchg_i64_offset:6820; GFX8:       ; %bb.0: ; %entry6821; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246822; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x346823; GFX8-NEXT:    s_waitcnt lgkmcnt(0)6824; GFX8-NEXT:    s_add_u32 s0, s0, 326825; GFX8-NEXT:    s_addc_u32 s1, s1, 06826; GFX8-NEXT:    v_mov_b32_e32 v5, s16827; GFX8-NEXT:    v_mov_b32_e32 v0, s26828; GFX8-NEXT:    v_mov_b32_e32 v1, s36829; GFX8-NEXT:    v_mov_b32_e32 v2, s46830; GFX8-NEXT:    v_mov_b32_e32 v3, s56831; GFX8-NEXT:    v_mov_b32_e32 v4, s06832; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]6833; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6834; GFX8-NEXT:    buffer_wbinvl1_vol6835; GFX8-NEXT:    s_endpgm6836;6837; GFX12-LABEL: atomic_cmpxchg_i64_offset:6838; GFX12:       ; %bb.0: ; %entry6839; GFX12-NEXT:    s_clause 0x16840; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246841; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x346842; GFX12-NEXT:    s_wait_kmcnt 0x06843; GFX12-NEXT:    v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v5, s16844; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s36845; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s56846; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[4:5], v[0:3] offset:32 scope:SCOPE_DEV6847; GFX12-NEXT:    s_wait_storecnt_dscnt 0x06848; GFX12-NEXT:    global_inv scope:SCOPE_DEV6849; GFX12-NEXT:    s_endpgm6850entry:6851  %gep = getelementptr inbounds i64, ptr %out, i64 46852  %val = cmpxchg volatile ptr %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst, !noalias.addrspace !06853  ret void6854}6855 6856define amdgpu_kernel void @atomic_cmpxchg_i64_soffset(ptr %out, i64 %in, i64 %old) {6857; GFX7-LABEL: atomic_cmpxchg_i64_soffset:6858; GFX7:       ; %bb.0: ; %entry6859; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96860; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd6861; GFX7-NEXT:    s_waitcnt lgkmcnt(0)6862; GFX7-NEXT:    s_add_u32 s0, s0, 0x119406863; GFX7-NEXT:    s_addc_u32 s1, s1, 06864; GFX7-NEXT:    v_mov_b32_e32 v5, s16865; GFX7-NEXT:    v_mov_b32_e32 v0, s26866; GFX7-NEXT:    v_mov_b32_e32 v1, s36867; GFX7-NEXT:    v_mov_b32_e32 v2, s46868; GFX7-NEXT:    v_mov_b32_e32 v3, s56869; GFX7-NEXT:    v_mov_b32_e32 v4, s06870; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]6871; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6872; GFX7-NEXT:    buffer_wbinvl1_vol6873; GFX7-NEXT:    s_endpgm6874;6875; GFX8-LABEL: atomic_cmpxchg_i64_soffset:6876; GFX8:       ; %bb.0: ; %entry6877; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246878; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x346879; GFX8-NEXT:    s_waitcnt lgkmcnt(0)6880; GFX8-NEXT:    s_add_u32 s0, s0, 0x119406881; GFX8-NEXT:    s_addc_u32 s1, s1, 06882; GFX8-NEXT:    v_mov_b32_e32 v5, s16883; GFX8-NEXT:    v_mov_b32_e32 v0, s26884; GFX8-NEXT:    v_mov_b32_e32 v1, s36885; GFX8-NEXT:    v_mov_b32_e32 v2, s46886; GFX8-NEXT:    v_mov_b32_e32 v3, s56887; GFX8-NEXT:    v_mov_b32_e32 v4, s06888; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]6889; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6890; GFX8-NEXT:    buffer_wbinvl1_vol6891; GFX8-NEXT:    s_endpgm6892;6893; GFX12-LABEL: atomic_cmpxchg_i64_soffset:6894; GFX12:       ; %bb.0: ; %entry6895; GFX12-NEXT:    s_clause 0x16896; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246897; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x346898; GFX12-NEXT:    s_wait_kmcnt 0x06899; GFX12-NEXT:    v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v5, s16900; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s36901; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s56902; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[4:5], v[0:3] offset:72000 scope:SCOPE_DEV6903; GFX12-NEXT:    s_wait_storecnt_dscnt 0x06904; GFX12-NEXT:    global_inv scope:SCOPE_DEV6905; GFX12-NEXT:    s_endpgm6906entry:6907  %gep = getelementptr inbounds i64, ptr %out, i64 90006908  %val = cmpxchg volatile ptr %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst, !noalias.addrspace !06909  ret void6910}6911 6912define amdgpu_kernel void @atomic_cmpxchg_i64_ret_offset(ptr %out, ptr %out2, i64 %in, i64 %old) {6913; GFX7-LABEL: atomic_cmpxchg_i64_ret_offset:6914; GFX7:       ; %bb.0: ; %entry6915; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x96916; GFX7-NEXT:    s_waitcnt lgkmcnt(0)6917; GFX7-NEXT:    s_add_u32 s0, s0, 326918; GFX7-NEXT:    s_addc_u32 s1, s1, 06919; GFX7-NEXT:    v_mov_b32_e32 v5, s16920; GFX7-NEXT:    v_mov_b32_e32 v0, s46921; GFX7-NEXT:    v_mov_b32_e32 v1, s56922; GFX7-NEXT:    v_mov_b32_e32 v2, s66923; GFX7-NEXT:    v_mov_b32_e32 v3, s76924; GFX7-NEXT:    v_mov_b32_e32 v4, s06925; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc6926; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6927; GFX7-NEXT:    buffer_wbinvl1_vol6928; GFX7-NEXT:    v_mov_b32_e32 v2, s26929; GFX7-NEXT:    v_mov_b32_e32 v3, s36930; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]6931; GFX7-NEXT:    s_endpgm6932;6933; GFX8-LABEL: atomic_cmpxchg_i64_ret_offset:6934; GFX8:       ; %bb.0: ; %entry6935; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x246936; GFX8-NEXT:    s_waitcnt lgkmcnt(0)6937; GFX8-NEXT:    s_add_u32 s0, s0, 326938; GFX8-NEXT:    s_addc_u32 s1, s1, 06939; GFX8-NEXT:    v_mov_b32_e32 v5, s16940; GFX8-NEXT:    v_mov_b32_e32 v0, s46941; GFX8-NEXT:    v_mov_b32_e32 v1, s56942; GFX8-NEXT:    v_mov_b32_e32 v2, s66943; GFX8-NEXT:    v_mov_b32_e32 v3, s76944; GFX8-NEXT:    v_mov_b32_e32 v4, s06945; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc6946; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6947; GFX8-NEXT:    buffer_wbinvl1_vol6948; GFX8-NEXT:    v_mov_b32_e32 v2, s26949; GFX8-NEXT:    v_mov_b32_e32 v3, s36950; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]6951; GFX8-NEXT:    s_endpgm6952;6953; GFX12-LABEL: atomic_cmpxchg_i64_ret_offset:6954; GFX12:       ; %bb.0: ; %entry6955; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x246956; GFX12-NEXT:    s_wait_kmcnt 0x06957; GFX12-NEXT:    v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v5, s16958; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s56959; GFX12-NEXT:    v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s76960; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[0:3] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV6961; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x06962; GFX12-NEXT:    global_inv scope:SCOPE_DEV6963; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s36964; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]6965; GFX12-NEXT:    s_endpgm6966entry:6967  %gep = getelementptr inbounds i64, ptr %out, i64 46968  %val = cmpxchg volatile ptr %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst, !noalias.addrspace !06969  %extract0 = extractvalue { i64, i1 } %val, 06970  store i64 %extract0, ptr %out26971  ret void6972}6973 6974define amdgpu_kernel void @atomic_cmpxchg_i64_addr64_offset(ptr %out, i64 %in, i64 %index, i64 %old) {6975; GFX7-LABEL: atomic_cmpxchg_i64_addr64_offset:6976; GFX7:       ; %bb.0: ; %entry6977; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x96978; GFX7-NEXT:    s_waitcnt lgkmcnt(0)6979; GFX7-NEXT:    s_lshl_b64 s[4:5], s[4:5], 36980; GFX7-NEXT:    s_add_u32 s0, s0, s46981; GFX7-NEXT:    s_addc_u32 s1, s1, s56982; GFX7-NEXT:    s_add_u32 s0, s0, 326983; GFX7-NEXT:    s_addc_u32 s1, s1, 06984; GFX7-NEXT:    v_mov_b32_e32 v5, s16985; GFX7-NEXT:    v_mov_b32_e32 v0, s26986; GFX7-NEXT:    v_mov_b32_e32 v1, s36987; GFX7-NEXT:    v_mov_b32_e32 v2, s66988; GFX7-NEXT:    v_mov_b32_e32 v3, s76989; GFX7-NEXT:    v_mov_b32_e32 v4, s06990; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]6991; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)6992; GFX7-NEXT:    buffer_wbinvl1_vol6993; GFX7-NEXT:    s_endpgm6994;6995; GFX8-LABEL: atomic_cmpxchg_i64_addr64_offset:6996; GFX8:       ; %bb.0: ; %entry6997; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x246998; GFX8-NEXT:    s_waitcnt lgkmcnt(0)6999; GFX8-NEXT:    s_lshl_b64 s[4:5], s[4:5], 37000; GFX8-NEXT:    s_add_u32 s0, s0, s47001; GFX8-NEXT:    s_addc_u32 s1, s1, s57002; GFX8-NEXT:    s_add_u32 s0, s0, 327003; GFX8-NEXT:    s_addc_u32 s1, s1, 07004; GFX8-NEXT:    v_mov_b32_e32 v5, s17005; GFX8-NEXT:    v_mov_b32_e32 v0, s27006; GFX8-NEXT:    v_mov_b32_e32 v1, s37007; GFX8-NEXT:    v_mov_b32_e32 v2, s67008; GFX8-NEXT:    v_mov_b32_e32 v3, s77009; GFX8-NEXT:    v_mov_b32_e32 v4, s07010; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]7011; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7012; GFX8-NEXT:    buffer_wbinvl1_vol7013; GFX8-NEXT:    s_endpgm7014;7015; GFX12-LABEL: atomic_cmpxchg_i64_addr64_offset:7016; GFX12:       ; %bb.0: ; %entry7017; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x247018; GFX12-NEXT:    s_wait_kmcnt 0x07019; GFX12-NEXT:    s_lshl_b64 s[4:5], s[4:5], 37020; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s37021; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]7022; GFX12-NEXT:    v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s77023; GFX12-NEXT:    v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s07024; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[4:5], v[0:3] offset:32 scope:SCOPE_DEV7025; GFX12-NEXT:    s_wait_storecnt_dscnt 0x07026; GFX12-NEXT:    global_inv scope:SCOPE_DEV7027; GFX12-NEXT:    s_endpgm7028entry:7029  %ptr = getelementptr inbounds i64, ptr %out, i64 %index7030  %gep = getelementptr inbounds i64, ptr %ptr, i64 47031  %val = cmpxchg volatile ptr %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst, !noalias.addrspace !07032  ret void7033}7034 7035define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index, i64 %old) {7036; GFX7-LABEL: atomic_cmpxchg_i64_ret_addr64_offset:7037; GFX7:       ; %bb.0: ; %entry7038; GFX7-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x97039; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x117040; GFX7-NEXT:    s_waitcnt lgkmcnt(0)7041; GFX7-NEXT:    s_lshl_b64 s[2:3], s[14:15], 37042; GFX7-NEXT:    v_mov_b32_e32 v2, s07043; GFX7-NEXT:    s_add_u32 s0, s8, s27044; GFX7-NEXT:    s_addc_u32 s3, s9, s37045; GFX7-NEXT:    s_add_u32 s2, s0, 327046; GFX7-NEXT:    s_addc_u32 s3, s3, 07047; GFX7-NEXT:    v_mov_b32_e32 v5, s37048; GFX7-NEXT:    v_mov_b32_e32 v0, s127049; GFX7-NEXT:    v_mov_b32_e32 v1, s137050; GFX7-NEXT:    v_mov_b32_e32 v3, s17051; GFX7-NEXT:    v_mov_b32_e32 v4, s27052; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc7053; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7054; GFX7-NEXT:    buffer_wbinvl1_vol7055; GFX7-NEXT:    v_mov_b32_e32 v2, s107056; GFX7-NEXT:    v_mov_b32_e32 v3, s117057; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]7058; GFX7-NEXT:    s_endpgm7059;7060; GFX8-LABEL: atomic_cmpxchg_i64_ret_addr64_offset:7061; GFX8:       ; %bb.0: ; %entry7062; GFX8-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x247063; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x447064; GFX8-NEXT:    s_waitcnt lgkmcnt(0)7065; GFX8-NEXT:    s_lshl_b64 s[2:3], s[14:15], 37066; GFX8-NEXT:    v_mov_b32_e32 v2, s07067; GFX8-NEXT:    s_add_u32 s0, s8, s27068; GFX8-NEXT:    s_addc_u32 s3, s9, s37069; GFX8-NEXT:    s_add_u32 s2, s0, 327070; GFX8-NEXT:    s_addc_u32 s3, s3, 07071; GFX8-NEXT:    v_mov_b32_e32 v5, s37072; GFX8-NEXT:    v_mov_b32_e32 v0, s127073; GFX8-NEXT:    v_mov_b32_e32 v1, s137074; GFX8-NEXT:    v_mov_b32_e32 v3, s17075; GFX8-NEXT:    v_mov_b32_e32 v4, s27076; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc7077; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7078; GFX8-NEXT:    buffer_wbinvl1_vol7079; GFX8-NEXT:    v_mov_b32_e32 v2, s107080; GFX8-NEXT:    v_mov_b32_e32 v3, s117081; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]7082; GFX8-NEXT:    s_endpgm7083;7084; GFX12-LABEL: atomic_cmpxchg_i64_ret_addr64_offset:7085; GFX12:       ; %bb.0: ; %entry7086; GFX12-NEXT:    s_clause 0x17087; GFX12-NEXT:    s_load_b256 s[8:15], s[4:5], 0x247088; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x447089; GFX12-NEXT:    s_wait_kmcnt 0x07090; GFX12-NEXT:    s_lshl_b64 s[2:3], s[14:15], 37091; GFX12-NEXT:    v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s137092; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[8:9], s[2:3]7093; GFX12-NEXT:    v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s17094; GFX12-NEXT:    v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s27095; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[0:3] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV7096; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x07097; GFX12-NEXT:    global_inv scope:SCOPE_DEV7098; GFX12-NEXT:    v_dual_mov_b32 v2, s10 :: v_dual_mov_b32 v3, s117099; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]7100; GFX12-NEXT:    s_endpgm7101entry:7102  %ptr = getelementptr inbounds i64, ptr %out, i64 %index7103  %gep = getelementptr inbounds i64, ptr %ptr, i64 47104  %val = cmpxchg volatile ptr %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst, !noalias.addrspace !07105  %extract0 = extractvalue { i64, i1 } %val, 07106  store i64 %extract0, ptr %out27107  ret void7108}7109 7110define amdgpu_kernel void @atomic_cmpxchg_i64(ptr %out, i64 %in, i64 %old) {7111; GFX7-LABEL: atomic_cmpxchg_i64:7112; GFX7:       ; %bb.0: ; %entry7113; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x97114; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd7115; GFX7-NEXT:    s_waitcnt lgkmcnt(0)7116; GFX7-NEXT:    v_mov_b32_e32 v4, s07117; GFX7-NEXT:    v_mov_b32_e32 v5, s17118; GFX7-NEXT:    v_mov_b32_e32 v0, s27119; GFX7-NEXT:    v_mov_b32_e32 v1, s37120; GFX7-NEXT:    v_mov_b32_e32 v2, s47121; GFX7-NEXT:    v_mov_b32_e32 v3, s57122; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]7123; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7124; GFX7-NEXT:    buffer_wbinvl1_vol7125; GFX7-NEXT:    s_endpgm7126;7127; GFX8-LABEL: atomic_cmpxchg_i64:7128; GFX8:       ; %bb.0: ; %entry7129; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247130; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x347131; GFX8-NEXT:    s_waitcnt lgkmcnt(0)7132; GFX8-NEXT:    v_mov_b32_e32 v4, s07133; GFX8-NEXT:    v_mov_b32_e32 v5, s17134; GFX8-NEXT:    v_mov_b32_e32 v0, s27135; GFX8-NEXT:    v_mov_b32_e32 v1, s37136; GFX8-NEXT:    v_mov_b32_e32 v2, s47137; GFX8-NEXT:    v_mov_b32_e32 v3, s57138; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]7139; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7140; GFX8-NEXT:    buffer_wbinvl1_vol7141; GFX8-NEXT:    s_endpgm7142;7143; GFX12-LABEL: atomic_cmpxchg_i64:7144; GFX12:       ; %bb.0: ; %entry7145; GFX12-NEXT:    s_clause 0x17146; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x247147; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x347148; GFX12-NEXT:    s_wait_kmcnt 0x07149; GFX12-NEXT:    v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v5, s17150; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s37151; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s57152; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[4:5], v[0:3] scope:SCOPE_DEV7153; GFX12-NEXT:    s_wait_storecnt_dscnt 0x07154; GFX12-NEXT:    global_inv scope:SCOPE_DEV7155; GFX12-NEXT:    s_endpgm7156entry:7157  %val = cmpxchg volatile ptr %out, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst, !noalias.addrspace !07158  ret void7159}7160 7161define amdgpu_kernel void @atomic_cmpxchg_i64_ret(ptr %out, ptr %out2, i64 %in, i64 %old) {7162; GFX7-LABEL: atomic_cmpxchg_i64_ret:7163; GFX7:       ; %bb.0: ; %entry7164; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x97165; GFX7-NEXT:    s_waitcnt lgkmcnt(0)7166; GFX7-NEXT:    v_mov_b32_e32 v4, s07167; GFX7-NEXT:    v_mov_b32_e32 v5, s17168; GFX7-NEXT:    v_mov_b32_e32 v0, s47169; GFX7-NEXT:    v_mov_b32_e32 v1, s57170; GFX7-NEXT:    v_mov_b32_e32 v2, s67171; GFX7-NEXT:    v_mov_b32_e32 v3, s77172; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc7173; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7174; GFX7-NEXT:    buffer_wbinvl1_vol7175; GFX7-NEXT:    v_mov_b32_e32 v2, s27176; GFX7-NEXT:    v_mov_b32_e32 v3, s37177; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]7178; GFX7-NEXT:    s_endpgm7179;7180; GFX8-LABEL: atomic_cmpxchg_i64_ret:7181; GFX8:       ; %bb.0: ; %entry7182; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x247183; GFX8-NEXT:    s_waitcnt lgkmcnt(0)7184; GFX8-NEXT:    v_mov_b32_e32 v4, s07185; GFX8-NEXT:    v_mov_b32_e32 v5, s17186; GFX8-NEXT:    v_mov_b32_e32 v0, s47187; GFX8-NEXT:    v_mov_b32_e32 v1, s57188; GFX8-NEXT:    v_mov_b32_e32 v2, s67189; GFX8-NEXT:    v_mov_b32_e32 v3, s77190; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc7191; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7192; GFX8-NEXT:    buffer_wbinvl1_vol7193; GFX8-NEXT:    v_mov_b32_e32 v2, s27194; GFX8-NEXT:    v_mov_b32_e32 v3, s37195; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]7196; GFX8-NEXT:    s_endpgm7197;7198; GFX12-LABEL: atomic_cmpxchg_i64_ret:7199; GFX12:       ; %bb.0: ; %entry7200; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x247201; GFX12-NEXT:    s_wait_kmcnt 0x07202; GFX12-NEXT:    v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v5, s17203; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s57204; GFX12-NEXT:    v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s77205; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[0:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV7206; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x07207; GFX12-NEXT:    global_inv scope:SCOPE_DEV7208; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s37209; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]7210; GFX12-NEXT:    s_endpgm7211entry:7212  %val = cmpxchg volatile ptr %out, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst, !noalias.addrspace !07213  %extract0 = extractvalue { i64, i1 } %val, 07214  store i64 %extract0, ptr %out27215  ret void7216}7217 7218define amdgpu_kernel void @atomic_cmpxchg_i64_addr64(ptr %out, i64 %in, i64 %index, i64 %old) {7219; GFX7-LABEL: atomic_cmpxchg_i64_addr64:7220; GFX7:       ; %bb.0: ; %entry7221; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x97222; GFX7-NEXT:    s_waitcnt lgkmcnt(0)7223; GFX7-NEXT:    s_lshl_b64 s[4:5], s[4:5], 37224; GFX7-NEXT:    s_add_u32 s0, s0, s47225; GFX7-NEXT:    s_addc_u32 s1, s1, s57226; GFX7-NEXT:    v_mov_b32_e32 v5, s17227; GFX7-NEXT:    v_mov_b32_e32 v0, s27228; GFX7-NEXT:    v_mov_b32_e32 v1, s37229; GFX7-NEXT:    v_mov_b32_e32 v2, s67230; GFX7-NEXT:    v_mov_b32_e32 v3, s77231; GFX7-NEXT:    v_mov_b32_e32 v4, s07232; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]7233; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7234; GFX7-NEXT:    buffer_wbinvl1_vol7235; GFX7-NEXT:    s_endpgm7236;7237; GFX8-LABEL: atomic_cmpxchg_i64_addr64:7238; GFX8:       ; %bb.0: ; %entry7239; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x247240; GFX8-NEXT:    s_waitcnt lgkmcnt(0)7241; GFX8-NEXT:    s_lshl_b64 s[4:5], s[4:5], 37242; GFX8-NEXT:    s_add_u32 s0, s0, s47243; GFX8-NEXT:    s_addc_u32 s1, s1, s57244; GFX8-NEXT:    v_mov_b32_e32 v5, s17245; GFX8-NEXT:    v_mov_b32_e32 v0, s27246; GFX8-NEXT:    v_mov_b32_e32 v1, s37247; GFX8-NEXT:    v_mov_b32_e32 v2, s67248; GFX8-NEXT:    v_mov_b32_e32 v3, s77249; GFX8-NEXT:    v_mov_b32_e32 v4, s07250; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]7251; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7252; GFX8-NEXT:    buffer_wbinvl1_vol7253; GFX8-NEXT:    s_endpgm7254;7255; GFX12-LABEL: atomic_cmpxchg_i64_addr64:7256; GFX12:       ; %bb.0: ; %entry7257; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x247258; GFX12-NEXT:    s_wait_kmcnt 0x07259; GFX12-NEXT:    s_lshl_b64 s[4:5], s[4:5], 37260; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s37261; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]7262; GFX12-NEXT:    v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s77263; GFX12-NEXT:    v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s07264; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[4:5], v[0:3] scope:SCOPE_DEV7265; GFX12-NEXT:    s_wait_storecnt_dscnt 0x07266; GFX12-NEXT:    global_inv scope:SCOPE_DEV7267; GFX12-NEXT:    s_endpgm7268entry:7269  %ptr = getelementptr inbounds i64, ptr %out, i64 %index7270  %val = cmpxchg volatile ptr %ptr, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst, !noalias.addrspace !07271  ret void7272}7273 7274define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index, i64 %old) {7275; GFX7-LABEL: atomic_cmpxchg_i64_ret_addr64:7276; GFX7:       ; %bb.0: ; %entry7277; GFX7-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x97278; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x117279; GFX7-NEXT:    s_waitcnt lgkmcnt(0)7280; GFX7-NEXT:    s_lshl_b64 s[2:3], s[14:15], 37281; GFX7-NEXT:    s_add_u32 s2, s8, s27282; GFX7-NEXT:    s_addc_u32 s3, s9, s37283; GFX7-NEXT:    v_mov_b32_e32 v5, s37284; GFX7-NEXT:    v_mov_b32_e32 v0, s127285; GFX7-NEXT:    v_mov_b32_e32 v1, s137286; GFX7-NEXT:    v_mov_b32_e32 v2, s07287; GFX7-NEXT:    v_mov_b32_e32 v3, s17288; GFX7-NEXT:    v_mov_b32_e32 v4, s27289; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc7290; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7291; GFX7-NEXT:    buffer_wbinvl1_vol7292; GFX7-NEXT:    v_mov_b32_e32 v2, s107293; GFX7-NEXT:    v_mov_b32_e32 v3, s117294; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]7295; GFX7-NEXT:    s_endpgm7296;7297; GFX8-LABEL: atomic_cmpxchg_i64_ret_addr64:7298; GFX8:       ; %bb.0: ; %entry7299; GFX8-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x247300; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x447301; GFX8-NEXT:    s_waitcnt lgkmcnt(0)7302; GFX8-NEXT:    s_lshl_b64 s[2:3], s[14:15], 37303; GFX8-NEXT:    s_add_u32 s2, s8, s27304; GFX8-NEXT:    s_addc_u32 s3, s9, s37305; GFX8-NEXT:    v_mov_b32_e32 v5, s37306; GFX8-NEXT:    v_mov_b32_e32 v0, s127307; GFX8-NEXT:    v_mov_b32_e32 v1, s137308; GFX8-NEXT:    v_mov_b32_e32 v2, s07309; GFX8-NEXT:    v_mov_b32_e32 v3, s17310; GFX8-NEXT:    v_mov_b32_e32 v4, s27311; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc7312; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7313; GFX8-NEXT:    buffer_wbinvl1_vol7314; GFX8-NEXT:    v_mov_b32_e32 v2, s107315; GFX8-NEXT:    v_mov_b32_e32 v3, s117316; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]7317; GFX8-NEXT:    s_endpgm7318;7319; GFX12-LABEL: atomic_cmpxchg_i64_ret_addr64:7320; GFX12:       ; %bb.0: ; %entry7321; GFX12-NEXT:    s_clause 0x17322; GFX12-NEXT:    s_load_b256 s[8:15], s[4:5], 0x247323; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x447324; GFX12-NEXT:    s_wait_kmcnt 0x07325; GFX12-NEXT:    s_lshl_b64 s[2:3], s[14:15], 37326; GFX12-NEXT:    v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s137327; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[8:9], s[2:3]7328; GFX12-NEXT:    v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s17329; GFX12-NEXT:    v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s27330; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[0:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV7331; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x07332; GFX12-NEXT:    global_inv scope:SCOPE_DEV7333; GFX12-NEXT:    v_dual_mov_b32 v2, s10 :: v_dual_mov_b32 v3, s117334; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]7335; GFX12-NEXT:    s_endpgm7336entry:7337  %ptr = getelementptr inbounds i64, ptr %out, i64 %index7338  %val = cmpxchg volatile ptr %ptr, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst, !noalias.addrspace !07339  %extract0 = extractvalue { i64, i1 } %val, 07340  store i64 %extract0, ptr %out27341  ret void7342}7343 7344define amdgpu_kernel void @atomic_load_f64_offset(ptr %in, ptr %out) {7345; GFX7-LABEL: atomic_load_f64_offset:7346; GFX7:       ; %bb.0: ; %entry7347; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x97348; GFX7-NEXT:    s_waitcnt lgkmcnt(0)7349; GFX7-NEXT:    s_add_u32 s0, s0, 327350; GFX7-NEXT:    s_addc_u32 s1, s1, 07351; GFX7-NEXT:    v_mov_b32_e32 v0, s07352; GFX7-NEXT:    v_mov_b32_e32 v1, s17353; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc7354; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7355; GFX7-NEXT:    buffer_wbinvl1_vol7356; GFX7-NEXT:    v_mov_b32_e32 v2, s27357; GFX7-NEXT:    v_mov_b32_e32 v3, s37358; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]7359; GFX7-NEXT:    s_endpgm7360;7361; GFX8-LABEL: atomic_load_f64_offset:7362; GFX8:       ; %bb.0: ; %entry7363; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247364; GFX8-NEXT:    s_waitcnt lgkmcnt(0)7365; GFX8-NEXT:    s_add_u32 s0, s0, 327366; GFX8-NEXT:    s_addc_u32 s1, s1, 07367; GFX8-NEXT:    v_mov_b32_e32 v0, s07368; GFX8-NEXT:    v_mov_b32_e32 v1, s17369; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc7370; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7371; GFX8-NEXT:    buffer_wbinvl1_vol7372; GFX8-NEXT:    v_mov_b32_e32 v2, s27373; GFX8-NEXT:    v_mov_b32_e32 v3, s37374; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]7375; GFX8-NEXT:    s_endpgm7376;7377; GFX12-LABEL: atomic_load_f64_offset:7378; GFX12:       ; %bb.0: ; %entry7379; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x247380; GFX12-NEXT:    s_wait_kmcnt 0x07381; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17382; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s37383; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] offset:32 scope:SCOPE_SYS7384; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x07385; GFX12-NEXT:    global_inv scope:SCOPE_SYS7386; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]7387; GFX12-NEXT:    s_endpgm7388entry:7389  %gep = getelementptr inbounds double, ptr %in, i64 47390  %val = load atomic double, ptr %gep  seq_cst, align 8, !noalias.addrspace !07391  store double %val, ptr %out7392  ret void7393}7394 7395define amdgpu_kernel void @atomic_load_f64(ptr %in, ptr %out) {7396; GFX7-LABEL: atomic_load_f64:7397; GFX7:       ; %bb.0: ; %entry7398; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x97399; GFX7-NEXT:    s_waitcnt lgkmcnt(0)7400; GFX7-NEXT:    v_mov_b32_e32 v0, s07401; GFX7-NEXT:    v_mov_b32_e32 v1, s17402; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc7403; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7404; GFX7-NEXT:    buffer_wbinvl1_vol7405; GFX7-NEXT:    v_mov_b32_e32 v2, s27406; GFX7-NEXT:    v_mov_b32_e32 v3, s37407; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]7408; GFX7-NEXT:    s_endpgm7409;7410; GFX8-LABEL: atomic_load_f64:7411; GFX8:       ; %bb.0: ; %entry7412; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247413; GFX8-NEXT:    s_waitcnt lgkmcnt(0)7414; GFX8-NEXT:    v_mov_b32_e32 v0, s07415; GFX8-NEXT:    v_mov_b32_e32 v1, s17416; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc7417; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7418; GFX8-NEXT:    buffer_wbinvl1_vol7419; GFX8-NEXT:    v_mov_b32_e32 v2, s27420; GFX8-NEXT:    v_mov_b32_e32 v3, s37421; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]7422; GFX8-NEXT:    s_endpgm7423;7424; GFX12-LABEL: atomic_load_f64:7425; GFX12:       ; %bb.0: ; %entry7426; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x247427; GFX12-NEXT:    s_wait_kmcnt 0x07428; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17429; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s37430; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV7431; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x07432; GFX12-NEXT:    global_inv scope:SCOPE_DEV7433; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]7434; GFX12-NEXT:    s_endpgm7435entry:7436  %val = load atomic double, ptr %in syncscope("agent") seq_cst, align 8, !noalias.addrspace !07437  store double %val, ptr %out7438  ret void7439}7440 7441define amdgpu_kernel void @atomic_load_f64_addr64_offset(ptr %in, ptr %out, i64 %index) {7442; GFX7-LABEL: atomic_load_f64_addr64_offset:7443; GFX7:       ; %bb.0: ; %entry7444; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd7445; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x97446; GFX7-NEXT:    s_waitcnt lgkmcnt(0)7447; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 37448; GFX7-NEXT:    s_add_u32 s0, s0, s47449; GFX7-NEXT:    s_addc_u32 s1, s1, s57450; GFX7-NEXT:    s_add_u32 s0, s0, 327451; GFX7-NEXT:    s_addc_u32 s1, s1, 07452; GFX7-NEXT:    v_mov_b32_e32 v0, s07453; GFX7-NEXT:    v_mov_b32_e32 v1, s17454; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc7455; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7456; GFX7-NEXT:    buffer_wbinvl1_vol7457; GFX7-NEXT:    v_mov_b32_e32 v2, s27458; GFX7-NEXT:    v_mov_b32_e32 v3, s37459; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]7460; GFX7-NEXT:    s_endpgm7461;7462; GFX8-LABEL: atomic_load_f64_addr64_offset:7463; GFX8:       ; %bb.0: ; %entry7464; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x347465; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247466; GFX8-NEXT:    s_waitcnt lgkmcnt(0)7467; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 37468; GFX8-NEXT:    s_add_u32 s0, s0, s47469; GFX8-NEXT:    s_addc_u32 s1, s1, s57470; GFX8-NEXT:    s_add_u32 s0, s0, 327471; GFX8-NEXT:    s_addc_u32 s1, s1, 07472; GFX8-NEXT:    v_mov_b32_e32 v0, s07473; GFX8-NEXT:    v_mov_b32_e32 v1, s17474; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc7475; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7476; GFX8-NEXT:    buffer_wbinvl1_vol7477; GFX8-NEXT:    v_mov_b32_e32 v2, s27478; GFX8-NEXT:    v_mov_b32_e32 v3, s37479; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]7480; GFX8-NEXT:    s_endpgm7481;7482; GFX12-LABEL: atomic_load_f64_addr64_offset:7483; GFX12:       ; %bb.0: ; %entry7484; GFX12-NEXT:    s_clause 0x17485; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x347486; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x247487; GFX12-NEXT:    s_wait_kmcnt 0x07488; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 37489; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s37490; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]7491; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)7492; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17493; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] offset:32 scope:SCOPE_SYS7494; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x07495; GFX12-NEXT:    global_inv scope:SCOPE_SYS7496; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]7497; GFX12-NEXT:    s_endpgm7498entry:7499  %ptr = getelementptr inbounds double, ptr %in, i64 %index7500  %gep = getelementptr inbounds double, ptr %ptr, i64 47501  %val = load atomic double, ptr %gep seq_cst, align 8, !noalias.addrspace !07502  store double %val, ptr %out7503  ret void7504}7505 7506define amdgpu_kernel void @atomic_load_f64_addr64(ptr %in, ptr %out, i64 %index) {7507; GFX7-LABEL: atomic_load_f64_addr64:7508; GFX7:       ; %bb.0: ; %entry7509; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd7510; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x97511; GFX7-NEXT:    s_waitcnt lgkmcnt(0)7512; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 37513; GFX7-NEXT:    s_add_u32 s0, s0, s47514; GFX7-NEXT:    s_addc_u32 s1, s1, s57515; GFX7-NEXT:    v_mov_b32_e32 v0, s07516; GFX7-NEXT:    v_mov_b32_e32 v1, s17517; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc7518; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7519; GFX7-NEXT:    buffer_wbinvl1_vol7520; GFX7-NEXT:    v_mov_b32_e32 v2, s27521; GFX7-NEXT:    v_mov_b32_e32 v3, s37522; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]7523; GFX7-NEXT:    s_endpgm7524;7525; GFX8-LABEL: atomic_load_f64_addr64:7526; GFX8:       ; %bb.0: ; %entry7527; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x347528; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247529; GFX8-NEXT:    s_waitcnt lgkmcnt(0)7530; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 37531; GFX8-NEXT:    s_add_u32 s0, s0, s47532; GFX8-NEXT:    s_addc_u32 s1, s1, s57533; GFX8-NEXT:    v_mov_b32_e32 v0, s07534; GFX8-NEXT:    v_mov_b32_e32 v1, s17535; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc7536; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7537; GFX8-NEXT:    buffer_wbinvl1_vol7538; GFX8-NEXT:    v_mov_b32_e32 v2, s27539; GFX8-NEXT:    v_mov_b32_e32 v3, s37540; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]7541; GFX8-NEXT:    s_endpgm7542;7543; GFX12-LABEL: atomic_load_f64_addr64:7544; GFX12:       ; %bb.0: ; %entry7545; GFX12-NEXT:    s_clause 0x17546; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x347547; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x247548; GFX12-NEXT:    s_wait_kmcnt 0x07549; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 37550; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s37551; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]7552; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)7553; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17554; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] scope:SCOPE_SYS7555; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x07556; GFX12-NEXT:    global_inv scope:SCOPE_SYS7557; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]7558; GFX12-NEXT:    s_endpgm7559entry:7560  %ptr = getelementptr inbounds double, ptr %in, i64 %index7561  %val = load atomic double, ptr %ptr seq_cst, align 8, !noalias.addrspace !07562  store double %val, ptr %out7563  ret void7564}7565 7566define amdgpu_kernel void @atomic_store_f64_offset(double %in, ptr %out) {7567; GFX7-LABEL: atomic_store_f64_offset:7568; GFX7:       ; %bb.0: ; %entry7569; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x97570; GFX7-NEXT:    s_waitcnt lgkmcnt(0)7571; GFX7-NEXT:    v_mov_b32_e32 v0, s07572; GFX7-NEXT:    s_add_u32 s0, s2, 327573; GFX7-NEXT:    v_mov_b32_e32 v1, s17574; GFX7-NEXT:    s_addc_u32 s1, s3, 07575; GFX7-NEXT:    v_mov_b32_e32 v3, s17576; GFX7-NEXT:    v_mov_b32_e32 v2, s07577; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]7578; GFX7-NEXT:    s_endpgm7579;7580; GFX8-LABEL: atomic_store_f64_offset:7581; GFX8:       ; %bb.0: ; %entry7582; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247583; GFX8-NEXT:    s_waitcnt lgkmcnt(0)7584; GFX8-NEXT:    v_mov_b32_e32 v0, s07585; GFX8-NEXT:    s_add_u32 s0, s2, 327586; GFX8-NEXT:    v_mov_b32_e32 v1, s17587; GFX8-NEXT:    s_addc_u32 s1, s3, 07588; GFX8-NEXT:    v_mov_b32_e32 v3, s17589; GFX8-NEXT:    v_mov_b32_e32 v2, s07590; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]7591; GFX8-NEXT:    s_endpgm7592;7593; GFX12-LABEL: atomic_store_f64_offset:7594; GFX12:       ; %bb.0: ; %entry7595; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x247596; GFX12-NEXT:    s_wait_kmcnt 0x07597; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17598; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s37599; GFX12-NEXT:    global_wb scope:SCOPE_SYS7600; GFX12-NEXT:    s_wait_storecnt 0x07601; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] offset:32 scope:SCOPE_SYS7602; GFX12-NEXT:    s_endpgm7603entry:7604  %gep = getelementptr inbounds double, ptr %out, i64 47605  store atomic double %in, ptr %gep  seq_cst, align 8, !noalias.addrspace !07606  ret void7607}7608 7609define amdgpu_kernel void @atomic_store_f64(double %in, ptr %out) {7610; GFX7-LABEL: atomic_store_f64:7611; GFX7:       ; %bb.0: ; %entry7612; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x97613; GFX7-NEXT:    s_waitcnt lgkmcnt(0)7614; GFX7-NEXT:    v_mov_b32_e32 v0, s07615; GFX7-NEXT:    v_mov_b32_e32 v1, s17616; GFX7-NEXT:    v_mov_b32_e32 v2, s27617; GFX7-NEXT:    v_mov_b32_e32 v3, s37618; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]7619; GFX7-NEXT:    s_endpgm7620;7621; GFX8-LABEL: atomic_store_f64:7622; GFX8:       ; %bb.0: ; %entry7623; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247624; GFX8-NEXT:    s_waitcnt lgkmcnt(0)7625; GFX8-NEXT:    v_mov_b32_e32 v0, s07626; GFX8-NEXT:    v_mov_b32_e32 v1, s17627; GFX8-NEXT:    v_mov_b32_e32 v2, s27628; GFX8-NEXT:    v_mov_b32_e32 v3, s37629; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]7630; GFX8-NEXT:    s_endpgm7631;7632; GFX12-LABEL: atomic_store_f64:7633; GFX12:       ; %bb.0: ; %entry7634; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x247635; GFX12-NEXT:    s_wait_kmcnt 0x07636; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17637; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s37638; GFX12-NEXT:    global_wb scope:SCOPE_SYS7639; GFX12-NEXT:    s_wait_storecnt 0x07640; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] scope:SCOPE_SYS7641; GFX12-NEXT:    s_endpgm7642entry:7643  store atomic double %in, ptr %out seq_cst, align 8, !noalias.addrspace !07644  ret void7645}7646 7647define amdgpu_kernel void @atomic_store_f64_addr64_offset(double %in, ptr %out, i64 %index) {7648; GFX7-LABEL: atomic_store_f64_addr64_offset:7649; GFX7:       ; %bb.0: ; %entry7650; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x97651; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd7652; GFX7-NEXT:    s_waitcnt lgkmcnt(0)7653; GFX7-NEXT:    v_mov_b32_e32 v0, s07654; GFX7-NEXT:    v_mov_b32_e32 v1, s17655; GFX7-NEXT:    s_lshl_b64 s[0:1], s[4:5], 37656; GFX7-NEXT:    s_add_u32 s0, s2, s07657; GFX7-NEXT:    s_addc_u32 s1, s3, s17658; GFX7-NEXT:    s_add_u32 s0, s0, 327659; GFX7-NEXT:    s_addc_u32 s1, s1, 07660; GFX7-NEXT:    v_mov_b32_e32 v3, s17661; GFX7-NEXT:    v_mov_b32_e32 v2, s07662; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]7663; GFX7-NEXT:    s_endpgm7664;7665; GFX8-LABEL: atomic_store_f64_addr64_offset:7666; GFX8:       ; %bb.0: ; %entry7667; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247668; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x347669; GFX8-NEXT:    s_waitcnt lgkmcnt(0)7670; GFX8-NEXT:    v_mov_b32_e32 v0, s07671; GFX8-NEXT:    v_mov_b32_e32 v1, s17672; GFX8-NEXT:    s_lshl_b64 s[0:1], s[4:5], 37673; GFX8-NEXT:    s_add_u32 s0, s2, s07674; GFX8-NEXT:    s_addc_u32 s1, s3, s17675; GFX8-NEXT:    s_add_u32 s0, s0, 327676; GFX8-NEXT:    s_addc_u32 s1, s1, 07677; GFX8-NEXT:    v_mov_b32_e32 v3, s17678; GFX8-NEXT:    v_mov_b32_e32 v2, s07679; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]7680; GFX8-NEXT:    s_endpgm7681;7682; GFX12-LABEL: atomic_store_f64_addr64_offset:7683; GFX12:       ; %bb.0: ; %entry7684; GFX12-NEXT:    s_clause 0x17685; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x347686; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x247687; GFX12-NEXT:    s_wait_kmcnt 0x07688; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 37689; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17690; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[2:3], s[4:5]7691; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)7692; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s37693; GFX12-NEXT:    global_wb scope:SCOPE_SYS7694; GFX12-NEXT:    s_wait_storecnt 0x07695; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] offset:32 scope:SCOPE_SYS7696; GFX12-NEXT:    s_endpgm7697entry:7698  %ptr = getelementptr inbounds double, ptr %out, i64 %index7699  %gep = getelementptr inbounds double, ptr %ptr, i64 47700  store atomic double %in, ptr %gep seq_cst, align 8, !noalias.addrspace !07701  ret void7702}7703 7704define amdgpu_kernel void @atomic_store_f64_addr64(double %in, ptr %out, i64 %index) {7705; GFX7-LABEL: atomic_store_f64_addr64:7706; GFX7:       ; %bb.0: ; %entry7707; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x97708; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd7709; GFX7-NEXT:    s_waitcnt lgkmcnt(0)7710; GFX7-NEXT:    v_mov_b32_e32 v0, s07711; GFX7-NEXT:    v_mov_b32_e32 v1, s17712; GFX7-NEXT:    s_lshl_b64 s[0:1], s[4:5], 37713; GFX7-NEXT:    s_add_u32 s0, s2, s07714; GFX7-NEXT:    s_addc_u32 s1, s3, s17715; GFX7-NEXT:    v_mov_b32_e32 v3, s17716; GFX7-NEXT:    v_mov_b32_e32 v2, s07717; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]7718; GFX7-NEXT:    s_endpgm7719;7720; GFX8-LABEL: atomic_store_f64_addr64:7721; GFX8:       ; %bb.0: ; %entry7722; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247723; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x347724; GFX8-NEXT:    s_waitcnt lgkmcnt(0)7725; GFX8-NEXT:    v_mov_b32_e32 v0, s07726; GFX8-NEXT:    v_mov_b32_e32 v1, s17727; GFX8-NEXT:    s_lshl_b64 s[0:1], s[4:5], 37728; GFX8-NEXT:    s_add_u32 s0, s2, s07729; GFX8-NEXT:    s_addc_u32 s1, s3, s17730; GFX8-NEXT:    v_mov_b32_e32 v3, s17731; GFX8-NEXT:    v_mov_b32_e32 v2, s07732; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]7733; GFX8-NEXT:    s_endpgm7734;7735; GFX12-LABEL: atomic_store_f64_addr64:7736; GFX12:       ; %bb.0: ; %entry7737; GFX12-NEXT:    s_clause 0x17738; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x347739; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x247740; GFX12-NEXT:    s_wait_kmcnt 0x07741; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 37742; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17743; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[2:3], s[4:5]7744; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)7745; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s37746; GFX12-NEXT:    global_wb scope:SCOPE_SYS7747; GFX12-NEXT:    s_wait_storecnt 0x07748; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] scope:SCOPE_SYS7749; GFX12-NEXT:    s_endpgm7750entry:7751  %ptr = getelementptr inbounds double, ptr %out, i64 %index7752  store atomic double %in, ptr %ptr seq_cst, align 8, !noalias.addrspace !07753  ret void7754}7755 7756define amdgpu_kernel void @atomic_inc_i64_offset(ptr %out, i64 %in) {7757; GFX7-LABEL: atomic_inc_i64_offset:7758; GFX7:       ; %bb.0: ; %entry7759; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x97760; GFX7-NEXT:    s_waitcnt lgkmcnt(0)7761; GFX7-NEXT:    s_add_u32 s0, s0, 327762; GFX7-NEXT:    s_addc_u32 s1, s1, 07763; GFX7-NEXT:    v_mov_b32_e32 v5, s17764; GFX7-NEXT:    v_mov_b32_e32 v4, s07765; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]7766; GFX7-NEXT:    s_mov_b64 s[0:1], 07767; GFX7-NEXT:  .LBB107_1: ; %atomicrmw.start7768; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=17769; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7770; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 1, v27771; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc7772; GFX7-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]7773; GFX7-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc7774; GFX7-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc7775; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc7776; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7777; GFX7-NEXT:    buffer_wbinvl1_vol7778; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]7779; GFX7-NEXT:    v_mov_b32_e32 v3, v17780; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]7781; GFX7-NEXT:    v_mov_b32_e32 v2, v07782; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]7783; GFX7-NEXT:    s_cbranch_execnz .LBB107_17784; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end7785; GFX7-NEXT:    s_endpgm7786;7787; GFX8-LABEL: atomic_inc_i64_offset:7788; GFX8:       ; %bb.0: ; %entry7789; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247790; GFX8-NEXT:    s_waitcnt lgkmcnt(0)7791; GFX8-NEXT:    s_add_u32 s0, s0, 327792; GFX8-NEXT:    s_addc_u32 s1, s1, 07793; GFX8-NEXT:    v_mov_b32_e32 v5, s17794; GFX8-NEXT:    v_mov_b32_e32 v4, s07795; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]7796; GFX8-NEXT:    s_mov_b64 s[0:1], 07797; GFX8-NEXT:  .LBB107_1: ; %atomicrmw.start7798; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=17799; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7800; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 1, v27801; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc7802; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]7803; GFX8-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc7804; GFX8-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc7805; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc7806; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7807; GFX8-NEXT:    buffer_wbinvl1_vol7808; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]7809; GFX8-NEXT:    v_mov_b32_e32 v3, v17810; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]7811; GFX8-NEXT:    v_mov_b32_e32 v2, v07812; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]7813; GFX8-NEXT:    s_cbranch_execnz .LBB107_17814; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end7815; GFX8-NEXT:    s_endpgm7816;7817; GFX12-LABEL: atomic_inc_i64_offset:7818; GFX12:       ; %bb.0: ; %entry7819; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x247820; GFX12-NEXT:    s_wait_kmcnt 0x07821; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17822; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s37823; GFX12-NEXT:    flat_atomic_inc_u64 v[0:1], v[2:3] offset:32 scope:SCOPE_DEV7824; GFX12-NEXT:    s_wait_storecnt_dscnt 0x07825; GFX12-NEXT:    global_inv scope:SCOPE_DEV7826; GFX12-NEXT:    s_endpgm7827entry:7828  %gep = getelementptr inbounds i64, ptr %out, i64 47829  %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !07830  ret void7831}7832 7833define amdgpu_kernel void @atomic_inc_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {7834; GFX7-LABEL: atomic_inc_i64_ret_offset:7835; GFX7:       ; %bb.0: ; %entry7836; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x97837; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd7838; GFX7-NEXT:    s_waitcnt lgkmcnt(0)7839; GFX7-NEXT:    s_add_u32 s0, s0, 327840; GFX7-NEXT:    s_addc_u32 s1, s1, 07841; GFX7-NEXT:    v_mov_b32_e32 v0, s07842; GFX7-NEXT:    v_mov_b32_e32 v1, s17843; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]7844; GFX7-NEXT:    s_mov_b64 s[0:1], 07845; GFX7-NEXT:  .LBB108_1: ; %atomicrmw.start7846; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=17847; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7848; GFX7-NEXT:    v_mov_b32_e32 v5, v37849; GFX7-NEXT:    v_mov_b32_e32 v4, v27850; GFX7-NEXT:    v_add_i32_e32 v2, vcc, 1, v47851; GFX7-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc7852; GFX7-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]7853; GFX7-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc7854; GFX7-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc7855; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc7856; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7857; GFX7-NEXT:    buffer_wbinvl1_vol7858; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]7859; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]7860; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]7861; GFX7-NEXT:    s_cbranch_execnz .LBB108_17862; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end7863; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]7864; GFX7-NEXT:    v_mov_b32_e32 v0, s27865; GFX7-NEXT:    v_mov_b32_e32 v1, s37866; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]7867; GFX7-NEXT:    s_endpgm7868;7869; GFX8-LABEL: atomic_inc_i64_ret_offset:7870; GFX8:       ; %bb.0: ; %entry7871; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247872; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x347873; GFX8-NEXT:    s_waitcnt lgkmcnt(0)7874; GFX8-NEXT:    s_add_u32 s0, s0, 327875; GFX8-NEXT:    s_addc_u32 s1, s1, 07876; GFX8-NEXT:    v_mov_b32_e32 v0, s07877; GFX8-NEXT:    v_mov_b32_e32 v1, s17878; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]7879; GFX8-NEXT:    s_mov_b64 s[0:1], 07880; GFX8-NEXT:  .LBB108_1: ; %atomicrmw.start7881; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=17882; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7883; GFX8-NEXT:    v_mov_b32_e32 v5, v37884; GFX8-NEXT:    v_mov_b32_e32 v4, v27885; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v47886; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc7887; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]7888; GFX8-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc7889; GFX8-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc7890; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc7891; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7892; GFX8-NEXT:    buffer_wbinvl1_vol7893; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]7894; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]7895; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]7896; GFX8-NEXT:    s_cbranch_execnz .LBB108_17897; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end7898; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]7899; GFX8-NEXT:    v_mov_b32_e32 v0, s27900; GFX8-NEXT:    v_mov_b32_e32 v1, s37901; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]7902; GFX8-NEXT:    s_endpgm7903;7904; GFX12-LABEL: atomic_inc_i64_ret_offset:7905; GFX12:       ; %bb.0: ; %entry7906; GFX12-NEXT:    s_clause 0x17907; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x247908; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x347909; GFX12-NEXT:    s_wait_kmcnt 0x07910; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17911; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s57912; GFX12-NEXT:    flat_atomic_inc_u64 v[0:1], v[0:1], v[2:3] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV7913; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x07914; GFX12-NEXT:    global_inv scope:SCOPE_DEV7915; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s37916; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]7917; GFX12-NEXT:    s_endpgm7918entry:7919  %gep = getelementptr inbounds i64, ptr %out, i64 47920  %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !07921  store i64 %tmp0, ptr %out27922  ret void7923}7924 7925define amdgpu_kernel void @atomic_inc_i64_incr64_offset(ptr %out, i64 %in, i64 %index) {7926; GFX7-LABEL: atomic_inc_i64_incr64_offset:7927; GFX7:       ; %bb.0: ; %entry7928; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd7929; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x97930; GFX7-NEXT:    s_waitcnt lgkmcnt(0)7931; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 37932; GFX7-NEXT:    s_add_u32 s0, s0, s47933; GFX7-NEXT:    s_addc_u32 s1, s1, s57934; GFX7-NEXT:    s_add_u32 s0, s0, 327935; GFX7-NEXT:    s_addc_u32 s1, s1, 07936; GFX7-NEXT:    v_mov_b32_e32 v5, s17937; GFX7-NEXT:    v_mov_b32_e32 v4, s07938; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]7939; GFX7-NEXT:    s_mov_b64 s[0:1], 07940; GFX7-NEXT:  .LBB109_1: ; %atomicrmw.start7941; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=17942; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7943; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 1, v27944; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc7945; GFX7-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]7946; GFX7-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc7947; GFX7-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc7948; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc7949; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7950; GFX7-NEXT:    buffer_wbinvl1_vol7951; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]7952; GFX7-NEXT:    v_mov_b32_e32 v3, v17953; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]7954; GFX7-NEXT:    v_mov_b32_e32 v2, v07955; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]7956; GFX7-NEXT:    s_cbranch_execnz .LBB109_17957; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end7958; GFX7-NEXT:    s_endpgm7959;7960; GFX8-LABEL: atomic_inc_i64_incr64_offset:7961; GFX8:       ; %bb.0: ; %entry7962; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x347963; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247964; GFX8-NEXT:    s_waitcnt lgkmcnt(0)7965; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 37966; GFX8-NEXT:    s_add_u32 s0, s0, s47967; GFX8-NEXT:    s_addc_u32 s1, s1, s57968; GFX8-NEXT:    s_add_u32 s0, s0, 327969; GFX8-NEXT:    s_addc_u32 s1, s1, 07970; GFX8-NEXT:    v_mov_b32_e32 v5, s17971; GFX8-NEXT:    v_mov_b32_e32 v4, s07972; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]7973; GFX8-NEXT:    s_mov_b64 s[0:1], 07974; GFX8-NEXT:  .LBB109_1: ; %atomicrmw.start7975; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=17976; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7977; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 1, v27978; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc7979; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]7980; GFX8-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc7981; GFX8-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc7982; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc7983; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)7984; GFX8-NEXT:    buffer_wbinvl1_vol7985; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]7986; GFX8-NEXT:    v_mov_b32_e32 v3, v17987; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]7988; GFX8-NEXT:    v_mov_b32_e32 v2, v07989; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]7990; GFX8-NEXT:    s_cbranch_execnz .LBB109_17991; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end7992; GFX8-NEXT:    s_endpgm7993;7994; GFX12-LABEL: atomic_inc_i64_incr64_offset:7995; GFX12:       ; %bb.0: ; %entry7996; GFX12-NEXT:    s_clause 0x17997; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x347998; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x247999; GFX12-NEXT:    s_wait_kmcnt 0x08000; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 38001; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s38002; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]8003; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)8004; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s08005; GFX12-NEXT:    flat_atomic_inc_u64 v[2:3], v[0:1] offset:32 scope:SCOPE_DEV8006; GFX12-NEXT:    s_wait_storecnt_dscnt 0x08007; GFX12-NEXT:    global_inv scope:SCOPE_DEV8008; GFX12-NEXT:    s_endpgm8009entry:8010  %ptr = getelementptr inbounds i64, ptr %out, i64 %index8011  %gep = getelementptr inbounds i64, ptr %ptr, i64 48012  %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !08013  ret void8014}8015 8016define amdgpu_kernel void @atomic_inc_i64_ret_incr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {8017; GFX7-LABEL: atomic_inc_i64_ret_incr64_offset:8018; GFX7:       ; %bb.0: ; %entry8019; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x98020; GFX7-NEXT:    s_waitcnt lgkmcnt(0)8021; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 38022; GFX7-NEXT:    s_add_u32 s0, s0, s68023; GFX7-NEXT:    s_addc_u32 s1, s1, s78024; GFX7-NEXT:    s_add_u32 s0, s0, 328025; GFX7-NEXT:    s_addc_u32 s1, s1, 08026; GFX7-NEXT:    v_mov_b32_e32 v0, s08027; GFX7-NEXT:    v_mov_b32_e32 v1, s18028; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]8029; GFX7-NEXT:    s_mov_b64 s[0:1], 08030; GFX7-NEXT:  .LBB110_1: ; %atomicrmw.start8031; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=18032; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8033; GFX7-NEXT:    v_mov_b32_e32 v5, v38034; GFX7-NEXT:    v_mov_b32_e32 v4, v28035; GFX7-NEXT:    v_add_i32_e32 v2, vcc, 1, v48036; GFX7-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc8037; GFX7-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]8038; GFX7-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc8039; GFX7-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc8040; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc8041; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8042; GFX7-NEXT:    buffer_wbinvl1_vol8043; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]8044; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]8045; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]8046; GFX7-NEXT:    s_cbranch_execnz .LBB110_18047; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end8048; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]8049; GFX7-NEXT:    v_mov_b32_e32 v0, s28050; GFX7-NEXT:    v_mov_b32_e32 v1, s38051; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]8052; GFX7-NEXT:    s_endpgm8053;8054; GFX8-LABEL: atomic_inc_i64_ret_incr64_offset:8055; GFX8:       ; %bb.0: ; %entry8056; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x248057; GFX8-NEXT:    s_waitcnt lgkmcnt(0)8058; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 38059; GFX8-NEXT:    s_add_u32 s0, s0, s68060; GFX8-NEXT:    s_addc_u32 s1, s1, s78061; GFX8-NEXT:    s_add_u32 s0, s0, 328062; GFX8-NEXT:    s_addc_u32 s1, s1, 08063; GFX8-NEXT:    v_mov_b32_e32 v0, s08064; GFX8-NEXT:    v_mov_b32_e32 v1, s18065; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]8066; GFX8-NEXT:    s_mov_b64 s[0:1], 08067; GFX8-NEXT:  .LBB110_1: ; %atomicrmw.start8068; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=18069; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8070; GFX8-NEXT:    v_mov_b32_e32 v5, v38071; GFX8-NEXT:    v_mov_b32_e32 v4, v28072; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v48073; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc8074; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]8075; GFX8-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc8076; GFX8-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc8077; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc8078; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8079; GFX8-NEXT:    buffer_wbinvl1_vol8080; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]8081; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]8082; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]8083; GFX8-NEXT:    s_cbranch_execnz .LBB110_18084; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end8085; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]8086; GFX8-NEXT:    v_mov_b32_e32 v0, s28087; GFX8-NEXT:    v_mov_b32_e32 v1, s38088; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]8089; GFX8-NEXT:    s_endpgm8090;8091; GFX12-LABEL: atomic_inc_i64_ret_incr64_offset:8092; GFX12:       ; %bb.0: ; %entry8093; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x248094; GFX12-NEXT:    s_wait_kmcnt 0x08095; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 38096; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s58097; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]8098; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)8099; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s08100; GFX12-NEXT:    flat_atomic_inc_u64 v[0:1], v[2:3], v[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV8101; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x08102; GFX12-NEXT:    global_inv scope:SCOPE_DEV8103; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s38104; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]8105; GFX12-NEXT:    s_endpgm8106entry:8107  %ptr = getelementptr inbounds i64, ptr %out, i64 %index8108  %gep = getelementptr inbounds i64, ptr %ptr, i64 48109  %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !08110  store i64 %tmp0, ptr %out28111  ret void8112}8113 8114define amdgpu_kernel void @atomic_inc_i64(ptr %out, i64 %in) {8115; GFX7-LABEL: atomic_inc_i64:8116; GFX7:       ; %bb.0: ; %entry8117; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x98118; GFX7-NEXT:    s_mov_b64 s[4:5], 08119; GFX7-NEXT:    s_waitcnt lgkmcnt(0)8120; GFX7-NEXT:    v_mov_b32_e32 v0, s08121; GFX7-NEXT:    v_mov_b32_e32 v1, s18122; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]8123; GFX7-NEXT:    v_mov_b32_e32 v5, s18124; GFX7-NEXT:    v_mov_b32_e32 v4, s08125; GFX7-NEXT:  .LBB111_1: ; %atomicrmw.start8126; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=18127; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8128; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 1, v28129; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc8130; GFX7-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]8131; GFX7-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc8132; GFX7-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc8133; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8134; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8135; GFX7-NEXT:    buffer_wbinvl1_vol8136; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8137; GFX7-NEXT:    v_mov_b32_e32 v3, v18138; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]8139; GFX7-NEXT:    v_mov_b32_e32 v2, v08140; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]8141; GFX7-NEXT:    s_cbranch_execnz .LBB111_18142; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end8143; GFX7-NEXT:    s_endpgm8144;8145; GFX8-LABEL: atomic_inc_i64:8146; GFX8:       ; %bb.0: ; %entry8147; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x248148; GFX8-NEXT:    s_mov_b64 s[4:5], 08149; GFX8-NEXT:    s_waitcnt lgkmcnt(0)8150; GFX8-NEXT:    v_mov_b32_e32 v0, s08151; GFX8-NEXT:    v_mov_b32_e32 v1, s18152; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]8153; GFX8-NEXT:    v_mov_b32_e32 v5, s18154; GFX8-NEXT:    v_mov_b32_e32 v4, s08155; GFX8-NEXT:  .LBB111_1: ; %atomicrmw.start8156; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=18157; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8158; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 1, v28159; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc8160; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]8161; GFX8-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc8162; GFX8-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc8163; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8164; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8165; GFX8-NEXT:    buffer_wbinvl1_vol8166; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8167; GFX8-NEXT:    v_mov_b32_e32 v3, v18168; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]8169; GFX8-NEXT:    v_mov_b32_e32 v2, v08170; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]8171; GFX8-NEXT:    s_cbranch_execnz .LBB111_18172; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end8173; GFX8-NEXT:    s_endpgm8174;8175; GFX12-LABEL: atomic_inc_i64:8176; GFX12:       ; %bb.0: ; %entry8177; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x248178; GFX12-NEXT:    s_wait_kmcnt 0x08179; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18180; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s38181; GFX12-NEXT:    flat_atomic_inc_u64 v[0:1], v[2:3] scope:SCOPE_DEV8182; GFX12-NEXT:    s_wait_storecnt_dscnt 0x08183; GFX12-NEXT:    global_inv scope:SCOPE_DEV8184; GFX12-NEXT:    s_endpgm8185entry:8186  %tmp0 = atomicrmw volatile uinc_wrap ptr %out, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !08187  ret void8188}8189 8190define amdgpu_kernel void @atomic_inc_i64_ret(ptr %out, ptr %out2, i64 %in) {8191; GFX7-LABEL: atomic_inc_i64_ret:8192; GFX7:       ; %bb.0: ; %entry8193; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x98194; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd8195; GFX7-NEXT:    s_mov_b64 s[6:7], 08196; GFX7-NEXT:    s_waitcnt lgkmcnt(0)8197; GFX7-NEXT:    v_mov_b32_e32 v0, s08198; GFX7-NEXT:    v_mov_b32_e32 v1, s18199; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]8200; GFX7-NEXT:  .LBB112_1: ; %atomicrmw.start8201; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=18202; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8203; GFX7-NEXT:    v_mov_b32_e32 v5, v38204; GFX7-NEXT:    v_mov_b32_e32 v4, v28205; GFX7-NEXT:    v_add_i32_e32 v2, vcc, 1, v48206; GFX7-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc8207; GFX7-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]8208; GFX7-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc8209; GFX7-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc8210; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc8211; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8212; GFX7-NEXT:    buffer_wbinvl1_vol8213; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]8214; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]8215; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]8216; GFX7-NEXT:    s_cbranch_execnz .LBB112_18217; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end8218; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]8219; GFX7-NEXT:    v_mov_b32_e32 v0, s28220; GFX7-NEXT:    v_mov_b32_e32 v1, s38221; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]8222; GFX7-NEXT:    s_endpgm8223;8224; GFX8-LABEL: atomic_inc_i64_ret:8225; GFX8:       ; %bb.0: ; %entry8226; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x248227; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x348228; GFX8-NEXT:    s_mov_b64 s[6:7], 08229; GFX8-NEXT:    s_waitcnt lgkmcnt(0)8230; GFX8-NEXT:    v_mov_b32_e32 v0, s08231; GFX8-NEXT:    v_mov_b32_e32 v1, s18232; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]8233; GFX8-NEXT:  .LBB112_1: ; %atomicrmw.start8234; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=18235; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8236; GFX8-NEXT:    v_mov_b32_e32 v5, v38237; GFX8-NEXT:    v_mov_b32_e32 v4, v28238; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v48239; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc8240; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]8241; GFX8-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc8242; GFX8-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc8243; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc8244; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8245; GFX8-NEXT:    buffer_wbinvl1_vol8246; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]8247; GFX8-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]8248; GFX8-NEXT:    s_andn2_b64 exec, exec, s[6:7]8249; GFX8-NEXT:    s_cbranch_execnz .LBB112_18250; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end8251; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]8252; GFX8-NEXT:    v_mov_b32_e32 v0, s28253; GFX8-NEXT:    v_mov_b32_e32 v1, s38254; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]8255; GFX8-NEXT:    s_endpgm8256;8257; GFX12-LABEL: atomic_inc_i64_ret:8258; GFX12:       ; %bb.0: ; %entry8259; GFX12-NEXT:    s_clause 0x18260; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x248261; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x348262; GFX12-NEXT:    s_wait_kmcnt 0x08263; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18264; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s58265; GFX12-NEXT:    flat_atomic_inc_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV8266; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x08267; GFX12-NEXT:    global_inv scope:SCOPE_DEV8268; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s38269; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]8270; GFX12-NEXT:    s_endpgm8271entry:8272  %tmp0 = atomicrmw volatile uinc_wrap ptr %out, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !08273  store i64 %tmp0, ptr %out28274  ret void8275}8276 8277define amdgpu_kernel void @atomic_inc_i64_incr64(ptr %out, i64 %in, i64 %index) {8278; GFX7-LABEL: atomic_inc_i64_incr64:8279; GFX7:       ; %bb.0: ; %entry8280; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd8281; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x98282; GFX7-NEXT:    s_waitcnt lgkmcnt(0)8283; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 38284; GFX7-NEXT:    s_add_u32 s0, s0, s48285; GFX7-NEXT:    s_addc_u32 s1, s1, s58286; GFX7-NEXT:    v_mov_b32_e32 v5, s18287; GFX7-NEXT:    v_mov_b32_e32 v4, s08288; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]8289; GFX7-NEXT:    s_mov_b64 s[0:1], 08290; GFX7-NEXT:  .LBB113_1: ; %atomicrmw.start8291; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=18292; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8293; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 1, v28294; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc8295; GFX7-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]8296; GFX7-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc8297; GFX7-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc8298; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8299; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8300; GFX7-NEXT:    buffer_wbinvl1_vol8301; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8302; GFX7-NEXT:    v_mov_b32_e32 v3, v18303; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]8304; GFX7-NEXT:    v_mov_b32_e32 v2, v08305; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]8306; GFX7-NEXT:    s_cbranch_execnz .LBB113_18307; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end8308; GFX7-NEXT:    s_endpgm8309;8310; GFX8-LABEL: atomic_inc_i64_incr64:8311; GFX8:       ; %bb.0: ; %entry8312; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x348313; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x248314; GFX8-NEXT:    s_waitcnt lgkmcnt(0)8315; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 38316; GFX8-NEXT:    s_add_u32 s0, s0, s48317; GFX8-NEXT:    s_addc_u32 s1, s1, s58318; GFX8-NEXT:    v_mov_b32_e32 v5, s18319; GFX8-NEXT:    v_mov_b32_e32 v4, s08320; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]8321; GFX8-NEXT:    s_mov_b64 s[0:1], 08322; GFX8-NEXT:  .LBB113_1: ; %atomicrmw.start8323; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=18324; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8325; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 1, v28326; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc8327; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]8328; GFX8-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc8329; GFX8-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc8330; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8331; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8332; GFX8-NEXT:    buffer_wbinvl1_vol8333; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8334; GFX8-NEXT:    v_mov_b32_e32 v3, v18335; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]8336; GFX8-NEXT:    v_mov_b32_e32 v2, v08337; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]8338; GFX8-NEXT:    s_cbranch_execnz .LBB113_18339; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end8340; GFX8-NEXT:    s_endpgm8341;8342; GFX12-LABEL: atomic_inc_i64_incr64:8343; GFX12:       ; %bb.0: ; %entry8344; GFX12-NEXT:    s_clause 0x18345; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x348346; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x248347; GFX12-NEXT:    s_wait_kmcnt 0x08348; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 38349; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s38350; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]8351; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)8352; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s08353; GFX12-NEXT:    flat_atomic_inc_u64 v[2:3], v[0:1] scope:SCOPE_DEV8354; GFX12-NEXT:    s_wait_storecnt_dscnt 0x08355; GFX12-NEXT:    global_inv scope:SCOPE_DEV8356; GFX12-NEXT:    s_endpgm8357entry:8358  %ptr = getelementptr inbounds i64, ptr %out, i64 %index8359  %tmp0 = atomicrmw volatile uinc_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !08360  ret void8361}8362 8363define amdgpu_kernel void @atomic_inc_i64_ret_incr64(ptr %out, ptr %out2, i64 %in, i64 %index) {8364; GFX7-LABEL: atomic_inc_i64_ret_incr64:8365; GFX7:       ; %bb.0: ; %entry8366; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x98367; GFX7-NEXT:    s_waitcnt lgkmcnt(0)8368; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 38369; GFX7-NEXT:    s_add_u32 s0, s0, s68370; GFX7-NEXT:    s_addc_u32 s1, s1, s78371; GFX7-NEXT:    v_mov_b32_e32 v0, s08372; GFX7-NEXT:    v_mov_b32_e32 v1, s18373; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]8374; GFX7-NEXT:    s_mov_b64 s[0:1], 08375; GFX7-NEXT:  .LBB114_1: ; %atomicrmw.start8376; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=18377; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8378; GFX7-NEXT:    v_mov_b32_e32 v5, v38379; GFX7-NEXT:    v_mov_b32_e32 v4, v28380; GFX7-NEXT:    v_add_i32_e32 v2, vcc, 1, v48381; GFX7-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc8382; GFX7-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]8383; GFX7-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc8384; GFX7-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc8385; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc8386; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8387; GFX7-NEXT:    buffer_wbinvl1_vol8388; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]8389; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]8390; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]8391; GFX7-NEXT:    s_cbranch_execnz .LBB114_18392; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end8393; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]8394; GFX7-NEXT:    v_mov_b32_e32 v0, s28395; GFX7-NEXT:    v_mov_b32_e32 v1, s38396; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]8397; GFX7-NEXT:    s_endpgm8398;8399; GFX8-LABEL: atomic_inc_i64_ret_incr64:8400; GFX8:       ; %bb.0: ; %entry8401; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x248402; GFX8-NEXT:    s_waitcnt lgkmcnt(0)8403; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 38404; GFX8-NEXT:    s_add_u32 s0, s0, s68405; GFX8-NEXT:    s_addc_u32 s1, s1, s78406; GFX8-NEXT:    v_mov_b32_e32 v0, s08407; GFX8-NEXT:    v_mov_b32_e32 v1, s18408; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]8409; GFX8-NEXT:    s_mov_b64 s[0:1], 08410; GFX8-NEXT:  .LBB114_1: ; %atomicrmw.start8411; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=18412; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8413; GFX8-NEXT:    v_mov_b32_e32 v5, v38414; GFX8-NEXT:    v_mov_b32_e32 v4, v28415; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v48416; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc8417; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]8418; GFX8-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc8419; GFX8-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc8420; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc8421; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8422; GFX8-NEXT:    buffer_wbinvl1_vol8423; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]8424; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]8425; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]8426; GFX8-NEXT:    s_cbranch_execnz .LBB114_18427; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end8428; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]8429; GFX8-NEXT:    v_mov_b32_e32 v0, s28430; GFX8-NEXT:    v_mov_b32_e32 v1, s38431; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]8432; GFX8-NEXT:    s_endpgm8433;8434; GFX12-LABEL: atomic_inc_i64_ret_incr64:8435; GFX12:       ; %bb.0: ; %entry8436; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x248437; GFX12-NEXT:    s_wait_kmcnt 0x08438; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 38439; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s58440; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]8441; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)8442; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s08443; GFX12-NEXT:    flat_atomic_inc_u64 v[0:1], v[2:3], v[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV8444; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x08445; GFX12-NEXT:    global_inv scope:SCOPE_DEV8446; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s38447; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]8448; GFX12-NEXT:    s_endpgm8449entry:8450  %ptr = getelementptr inbounds i64, ptr %out, i64 %index8451  %tmp0 = atomicrmw volatile uinc_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !08452  store i64 %tmp0, ptr %out28453  ret void8454}8455 8456define amdgpu_kernel void @atomic_dec_i64_offset(ptr %out, i64 %in) {8457; GFX7-LABEL: atomic_dec_i64_offset:8458; GFX7:       ; %bb.0: ; %entry8459; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x98460; GFX7-NEXT:    s_waitcnt lgkmcnt(0)8461; GFX7-NEXT:    s_add_u32 s0, s4, 328462; GFX7-NEXT:    s_addc_u32 s1, s5, 08463; GFX7-NEXT:    v_mov_b32_e32 v5, s18464; GFX7-NEXT:    v_mov_b32_e32 v4, s08465; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]8466; GFX7-NEXT:    s_mov_b64 s[4:5], 08467; GFX7-NEXT:    v_mov_b32_e32 v6, s78468; GFX7-NEXT:    v_mov_b32_e32 v7, s68469; GFX7-NEXT:  .LBB115_1: ; %atomicrmw.start8470; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=18471; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8472; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]8473; GFX7-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[6:7], v[2:3]8474; GFX7-NEXT:    v_add_i32_e64 v0, s[2:3], -1, v28475; GFX7-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v3, s[2:3]8476; GFX7-NEXT:    s_or_b64 vcc, vcc, s[0:1]8477; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc8478; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc8479; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8480; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8481; GFX7-NEXT:    buffer_wbinvl1_vol8482; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8483; GFX7-NEXT:    v_mov_b32_e32 v3, v18484; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]8485; GFX7-NEXT:    v_mov_b32_e32 v2, v08486; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]8487; GFX7-NEXT:    s_cbranch_execnz .LBB115_18488; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end8489; GFX7-NEXT:    s_endpgm8490;8491; GFX8-LABEL: atomic_dec_i64_offset:8492; GFX8:       ; %bb.0: ; %entry8493; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x248494; GFX8-NEXT:    s_waitcnt lgkmcnt(0)8495; GFX8-NEXT:    s_add_u32 s0, s4, 328496; GFX8-NEXT:    s_addc_u32 s1, s5, 08497; GFX8-NEXT:    v_mov_b32_e32 v5, s18498; GFX8-NEXT:    v_mov_b32_e32 v4, s08499; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]8500; GFX8-NEXT:    s_mov_b64 s[4:5], 08501; GFX8-NEXT:    v_mov_b32_e32 v6, s78502; GFX8-NEXT:    v_mov_b32_e32 v7, s68503; GFX8-NEXT:  .LBB115_1: ; %atomicrmw.start8504; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=18505; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8506; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]8507; GFX8-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[6:7], v[2:3]8508; GFX8-NEXT:    v_add_u32_e64 v0, s[2:3], -1, v28509; GFX8-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v3, s[2:3]8510; GFX8-NEXT:    s_or_b64 vcc, vcc, s[0:1]8511; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc8512; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc8513; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8514; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8515; GFX8-NEXT:    buffer_wbinvl1_vol8516; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8517; GFX8-NEXT:    v_mov_b32_e32 v3, v18518; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]8519; GFX8-NEXT:    v_mov_b32_e32 v2, v08520; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]8521; GFX8-NEXT:    s_cbranch_execnz .LBB115_18522; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end8523; GFX8-NEXT:    s_endpgm8524;8525; GFX12-LABEL: atomic_dec_i64_offset:8526; GFX12:       ; %bb.0: ; %entry8527; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x248528; GFX12-NEXT:    s_wait_kmcnt 0x08529; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18530; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s38531; GFX12-NEXT:    flat_atomic_dec_u64 v[0:1], v[2:3] offset:32 scope:SCOPE_DEV8532; GFX12-NEXT:    s_wait_storecnt_dscnt 0x08533; GFX12-NEXT:    global_inv scope:SCOPE_DEV8534; GFX12-NEXT:    s_endpgm8535entry:8536  %gep = getelementptr inbounds i64, ptr %out, i64 48537  %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !08538  ret void8539}8540 8541define amdgpu_kernel void @atomic_dec_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {8542; GFX7-LABEL: atomic_dec_i64_ret_offset:8543; GFX7:       ; %bb.0: ; %entry8544; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x98545; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd8546; GFX7-NEXT:    s_mov_b64 s[6:7], 08547; GFX7-NEXT:    s_waitcnt lgkmcnt(0)8548; GFX7-NEXT:    s_add_u32 s0, s8, 328549; GFX7-NEXT:    s_addc_u32 s1, s9, 08550; GFX7-NEXT:    v_mov_b32_e32 v0, s08551; GFX7-NEXT:    v_mov_b32_e32 v1, s18552; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]8553; GFX7-NEXT:    v_mov_b32_e32 v4, s58554; GFX7-NEXT:    v_mov_b32_e32 v5, s48555; GFX7-NEXT:  .LBB116_1: ; %atomicrmw.start8556; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=18557; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8558; GFX7-NEXT:    v_mov_b32_e32 v9, v38559; GFX7-NEXT:    v_mov_b32_e32 v8, v28560; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]8561; GFX7-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[4:5], v[8:9]8562; GFX7-NEXT:    v_add_i32_e64 v2, s[2:3], -1, v88563; GFX7-NEXT:    v_addc_u32_e64 v3, s[2:3], -1, v9, s[2:3]8564; GFX7-NEXT:    s_or_b64 vcc, vcc, s[0:1]8565; GFX7-NEXT:    v_cndmask_b32_e32 v7, v3, v4, vcc8566; GFX7-NEXT:    v_cndmask_b32_e32 v6, v2, v5, vcc8567; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc8568; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8569; GFX7-NEXT:    buffer_wbinvl1_vol8570; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]8571; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]8572; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]8573; GFX7-NEXT:    s_cbranch_execnz .LBB116_18574; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end8575; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]8576; GFX7-NEXT:    v_mov_b32_e32 v0, s108577; GFX7-NEXT:    v_mov_b32_e32 v1, s118578; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]8579; GFX7-NEXT:    s_endpgm8580;8581; GFX8-LABEL: atomic_dec_i64_ret_offset:8582; GFX8:       ; %bb.0: ; %entry8583; GFX8-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x248584; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x348585; GFX8-NEXT:    s_mov_b64 s[6:7], 08586; GFX8-NEXT:    s_waitcnt lgkmcnt(0)8587; GFX8-NEXT:    s_add_u32 s0, s8, 328588; GFX8-NEXT:    s_addc_u32 s1, s9, 08589; GFX8-NEXT:    v_mov_b32_e32 v0, s08590; GFX8-NEXT:    v_mov_b32_e32 v1, s18591; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]8592; GFX8-NEXT:    v_mov_b32_e32 v4, s58593; GFX8-NEXT:    v_mov_b32_e32 v5, s48594; GFX8-NEXT:  .LBB116_1: ; %atomicrmw.start8595; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=18596; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8597; GFX8-NEXT:    v_mov_b32_e32 v9, v38598; GFX8-NEXT:    v_mov_b32_e32 v8, v28599; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]8600; GFX8-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[4:5], v[8:9]8601; GFX8-NEXT:    v_add_u32_e64 v2, s[2:3], -1, v88602; GFX8-NEXT:    v_addc_u32_e64 v3, s[2:3], -1, v9, s[2:3]8603; GFX8-NEXT:    s_or_b64 vcc, vcc, s[0:1]8604; GFX8-NEXT:    v_cndmask_b32_e32 v7, v3, v4, vcc8605; GFX8-NEXT:    v_cndmask_b32_e32 v6, v2, v5, vcc8606; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc8607; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8608; GFX8-NEXT:    buffer_wbinvl1_vol8609; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]8610; GFX8-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]8611; GFX8-NEXT:    s_andn2_b64 exec, exec, s[6:7]8612; GFX8-NEXT:    s_cbranch_execnz .LBB116_18613; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end8614; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]8615; GFX8-NEXT:    v_mov_b32_e32 v0, s108616; GFX8-NEXT:    v_mov_b32_e32 v1, s118617; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]8618; GFX8-NEXT:    s_endpgm8619;8620; GFX12-LABEL: atomic_dec_i64_ret_offset:8621; GFX12:       ; %bb.0: ; %entry8622; GFX12-NEXT:    s_clause 0x18623; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x248624; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x348625; GFX12-NEXT:    s_wait_kmcnt 0x08626; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18627; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s58628; GFX12-NEXT:    flat_atomic_dec_u64 v[0:1], v[0:1], v[2:3] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV8629; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x08630; GFX12-NEXT:    global_inv scope:SCOPE_DEV8631; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s38632; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]8633; GFX12-NEXT:    s_endpgm8634entry:8635  %gep = getelementptr inbounds i64, ptr %out, i64 48636  %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !08637  store i64 %tmp0, ptr %out28638  ret void8639}8640 8641define amdgpu_kernel void @atomic_dec_i64_decr64_offset(ptr %out, i64 %in, i64 %index) {8642; GFX7-LABEL: atomic_dec_i64_decr64_offset:8643; GFX7:       ; %bb.0: ; %entry8644; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xd8645; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x98646; GFX7-NEXT:    s_waitcnt lgkmcnt(0)8647; GFX7-NEXT:    s_lshl_b64 s[0:1], s[0:1], 38648; GFX7-NEXT:    s_add_u32 s0, s4, s08649; GFX7-NEXT:    s_addc_u32 s1, s5, s18650; GFX7-NEXT:    s_add_u32 s0, s0, 328651; GFX7-NEXT:    s_addc_u32 s1, s1, 08652; GFX7-NEXT:    v_mov_b32_e32 v5, s18653; GFX7-NEXT:    v_mov_b32_e32 v4, s08654; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]8655; GFX7-NEXT:    s_mov_b64 s[4:5], 08656; GFX7-NEXT:    v_mov_b32_e32 v6, s78657; GFX7-NEXT:    v_mov_b32_e32 v7, s68658; GFX7-NEXT:  .LBB117_1: ; %atomicrmw.start8659; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=18660; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8661; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]8662; GFX7-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[6:7], v[2:3]8663; GFX7-NEXT:    v_add_i32_e64 v0, s[2:3], -1, v28664; GFX7-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v3, s[2:3]8665; GFX7-NEXT:    s_or_b64 vcc, vcc, s[0:1]8666; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc8667; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc8668; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8669; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8670; GFX7-NEXT:    buffer_wbinvl1_vol8671; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8672; GFX7-NEXT:    v_mov_b32_e32 v3, v18673; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]8674; GFX7-NEXT:    v_mov_b32_e32 v2, v08675; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]8676; GFX7-NEXT:    s_cbranch_execnz .LBB117_18677; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end8678; GFX7-NEXT:    s_endpgm8679;8680; GFX8-LABEL: atomic_dec_i64_decr64_offset:8681; GFX8:       ; %bb.0: ; %entry8682; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x348683; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x248684; GFX8-NEXT:    s_waitcnt lgkmcnt(0)8685; GFX8-NEXT:    s_lshl_b64 s[0:1], s[0:1], 38686; GFX8-NEXT:    s_add_u32 s0, s4, s08687; GFX8-NEXT:    s_addc_u32 s1, s5, s18688; GFX8-NEXT:    s_add_u32 s0, s0, 328689; GFX8-NEXT:    s_addc_u32 s1, s1, 08690; GFX8-NEXT:    v_mov_b32_e32 v5, s18691; GFX8-NEXT:    v_mov_b32_e32 v4, s08692; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]8693; GFX8-NEXT:    s_mov_b64 s[4:5], 08694; GFX8-NEXT:    v_mov_b32_e32 v6, s78695; GFX8-NEXT:    v_mov_b32_e32 v7, s68696; GFX8-NEXT:  .LBB117_1: ; %atomicrmw.start8697; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=18698; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8699; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]8700; GFX8-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[6:7], v[2:3]8701; GFX8-NEXT:    v_add_u32_e64 v0, s[2:3], -1, v28702; GFX8-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v3, s[2:3]8703; GFX8-NEXT:    s_or_b64 vcc, vcc, s[0:1]8704; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc8705; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc8706; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8707; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8708; GFX8-NEXT:    buffer_wbinvl1_vol8709; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8710; GFX8-NEXT:    v_mov_b32_e32 v3, v18711; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]8712; GFX8-NEXT:    v_mov_b32_e32 v2, v08713; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]8714; GFX8-NEXT:    s_cbranch_execnz .LBB117_18715; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end8716; GFX8-NEXT:    s_endpgm8717;8718; GFX12-LABEL: atomic_dec_i64_decr64_offset:8719; GFX12:       ; %bb.0: ; %entry8720; GFX12-NEXT:    s_clause 0x18721; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x348722; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x248723; GFX12-NEXT:    s_wait_kmcnt 0x08724; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 38725; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s38726; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]8727; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)8728; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s08729; GFX12-NEXT:    flat_atomic_dec_u64 v[2:3], v[0:1] offset:32 scope:SCOPE_DEV8730; GFX12-NEXT:    s_wait_storecnt_dscnt 0x08731; GFX12-NEXT:    global_inv scope:SCOPE_DEV8732; GFX12-NEXT:    s_endpgm8733entry:8734  %ptr = getelementptr inbounds i64, ptr %out, i64 %index8735  %gep = getelementptr inbounds i64, ptr %ptr, i64 48736  %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !08737  ret void8738}8739 8740define amdgpu_kernel void @atomic_dec_i64_ret_decr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {8741; GFX7-LABEL: atomic_dec_i64_ret_decr64_offset:8742; GFX7:       ; %bb.0: ; %entry8743; GFX7-NEXT:    s_load_dwordx8 s[4:11], s[4:5], 0x98744; GFX7-NEXT:    s_waitcnt lgkmcnt(0)8745; GFX7-NEXT:    s_lshl_b64 s[0:1], s[10:11], 38746; GFX7-NEXT:    s_add_u32 s0, s4, s08747; GFX7-NEXT:    s_addc_u32 s1, s5, s18748; GFX7-NEXT:    s_add_u32 s0, s0, 328749; GFX7-NEXT:    s_addc_u32 s1, s1, 08750; GFX7-NEXT:    v_mov_b32_e32 v0, s08751; GFX7-NEXT:    v_mov_b32_e32 v1, s18752; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]8753; GFX7-NEXT:    s_mov_b64 s[4:5], 08754; GFX7-NEXT:    v_mov_b32_e32 v4, s98755; GFX7-NEXT:    v_mov_b32_e32 v5, s88756; GFX7-NEXT:  .LBB118_1: ; %atomicrmw.start8757; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=18758; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8759; GFX7-NEXT:    v_mov_b32_e32 v9, v38760; GFX7-NEXT:    v_mov_b32_e32 v8, v28761; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]8762; GFX7-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[8:9], v[8:9]8763; GFX7-NEXT:    v_add_i32_e64 v2, s[2:3], -1, v88764; GFX7-NEXT:    v_addc_u32_e64 v3, s[2:3], -1, v9, s[2:3]8765; GFX7-NEXT:    s_or_b64 vcc, vcc, s[0:1]8766; GFX7-NEXT:    v_cndmask_b32_e32 v7, v3, v4, vcc8767; GFX7-NEXT:    v_cndmask_b32_e32 v6, v2, v5, vcc8768; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc8769; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8770; GFX7-NEXT:    buffer_wbinvl1_vol8771; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]8772; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]8773; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]8774; GFX7-NEXT:    s_cbranch_execnz .LBB118_18775; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end8776; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]8777; GFX7-NEXT:    v_mov_b32_e32 v0, s68778; GFX7-NEXT:    v_mov_b32_e32 v1, s78779; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]8780; GFX7-NEXT:    s_endpgm8781;8782; GFX8-LABEL: atomic_dec_i64_ret_decr64_offset:8783; GFX8:       ; %bb.0: ; %entry8784; GFX8-NEXT:    s_load_dwordx8 s[4:11], s[4:5], 0x248785; GFX8-NEXT:    s_waitcnt lgkmcnt(0)8786; GFX8-NEXT:    s_lshl_b64 s[0:1], s[10:11], 38787; GFX8-NEXT:    s_add_u32 s0, s4, s08788; GFX8-NEXT:    s_addc_u32 s1, s5, s18789; GFX8-NEXT:    s_add_u32 s0, s0, 328790; GFX8-NEXT:    s_addc_u32 s1, s1, 08791; GFX8-NEXT:    v_mov_b32_e32 v0, s08792; GFX8-NEXT:    v_mov_b32_e32 v1, s18793; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]8794; GFX8-NEXT:    s_mov_b64 s[4:5], 08795; GFX8-NEXT:    v_mov_b32_e32 v4, s98796; GFX8-NEXT:    v_mov_b32_e32 v5, s88797; GFX8-NEXT:  .LBB118_1: ; %atomicrmw.start8798; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=18799; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8800; GFX8-NEXT:    v_mov_b32_e32 v9, v38801; GFX8-NEXT:    v_mov_b32_e32 v8, v28802; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]8803; GFX8-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[8:9], v[8:9]8804; GFX8-NEXT:    v_add_u32_e64 v2, s[2:3], -1, v88805; GFX8-NEXT:    v_addc_u32_e64 v3, s[2:3], -1, v9, s[2:3]8806; GFX8-NEXT:    s_or_b64 vcc, vcc, s[0:1]8807; GFX8-NEXT:    v_cndmask_b32_e32 v7, v3, v4, vcc8808; GFX8-NEXT:    v_cndmask_b32_e32 v6, v2, v5, vcc8809; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc8810; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8811; GFX8-NEXT:    buffer_wbinvl1_vol8812; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]8813; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]8814; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]8815; GFX8-NEXT:    s_cbranch_execnz .LBB118_18816; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end8817; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]8818; GFX8-NEXT:    v_mov_b32_e32 v0, s68819; GFX8-NEXT:    v_mov_b32_e32 v1, s78820; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]8821; GFX8-NEXT:    s_endpgm8822;8823; GFX12-LABEL: atomic_dec_i64_ret_decr64_offset:8824; GFX12:       ; %bb.0: ; %entry8825; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x248826; GFX12-NEXT:    s_wait_kmcnt 0x08827; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 38828; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s58829; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]8830; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)8831; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s08832; GFX12-NEXT:    flat_atomic_dec_u64 v[0:1], v[2:3], v[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV8833; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x08834; GFX12-NEXT:    global_inv scope:SCOPE_DEV8835; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s38836; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]8837; GFX12-NEXT:    s_endpgm8838entry:8839  %ptr = getelementptr inbounds i64, ptr %out, i64 %index8840  %gep = getelementptr inbounds i64, ptr %ptr, i64 48841  %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !08842  store i64 %tmp0, ptr %out28843  ret void8844}8845 8846define amdgpu_kernel void @atomic_dec_i64(ptr %out, i64 %in) {8847; GFX7-LABEL: atomic_dec_i64:8848; GFX7:       ; %bb.0: ; %entry8849; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x98850; GFX7-NEXT:    s_mov_b64 s[8:9], 08851; GFX7-NEXT:    s_waitcnt lgkmcnt(0)8852; GFX7-NEXT:    v_mov_b32_e32 v0, s48853; GFX7-NEXT:    v_mov_b32_e32 v1, s58854; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]8855; GFX7-NEXT:    v_mov_b32_e32 v4, s48856; GFX7-NEXT:    v_mov_b32_e32 v6, s78857; GFX7-NEXT:    v_mov_b32_e32 v7, s68858; GFX7-NEXT:    v_mov_b32_e32 v5, s58859; GFX7-NEXT:  .LBB119_1: ; %atomicrmw.start8860; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=18861; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8862; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]8863; GFX7-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[6:7], v[2:3]8864; GFX7-NEXT:    v_add_i32_e64 v0, s[2:3], -1, v28865; GFX7-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v3, s[2:3]8866; GFX7-NEXT:    s_or_b64 vcc, vcc, s[0:1]8867; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc8868; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc8869; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8870; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8871; GFX7-NEXT:    buffer_wbinvl1_vol8872; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8873; GFX7-NEXT:    v_mov_b32_e32 v3, v18874; GFX7-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]8875; GFX7-NEXT:    v_mov_b32_e32 v2, v08876; GFX7-NEXT:    s_andn2_b64 exec, exec, s[8:9]8877; GFX7-NEXT:    s_cbranch_execnz .LBB119_18878; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end8879; GFX7-NEXT:    s_endpgm8880;8881; GFX8-LABEL: atomic_dec_i64:8882; GFX8:       ; %bb.0: ; %entry8883; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x248884; GFX8-NEXT:    s_mov_b64 s[8:9], 08885; GFX8-NEXT:    s_waitcnt lgkmcnt(0)8886; GFX8-NEXT:    v_mov_b32_e32 v0, s48887; GFX8-NEXT:    v_mov_b32_e32 v1, s58888; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]8889; GFX8-NEXT:    v_mov_b32_e32 v4, s48890; GFX8-NEXT:    v_mov_b32_e32 v6, s78891; GFX8-NEXT:    v_mov_b32_e32 v7, s68892; GFX8-NEXT:    v_mov_b32_e32 v5, s58893; GFX8-NEXT:  .LBB119_1: ; %atomicrmw.start8894; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=18895; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8896; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]8897; GFX8-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[6:7], v[2:3]8898; GFX8-NEXT:    v_add_u32_e64 v0, s[2:3], -1, v28899; GFX8-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v3, s[2:3]8900; GFX8-NEXT:    s_or_b64 vcc, vcc, s[0:1]8901; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc8902; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc8903; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8904; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8905; GFX8-NEXT:    buffer_wbinvl1_vol8906; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8907; GFX8-NEXT:    v_mov_b32_e32 v3, v18908; GFX8-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]8909; GFX8-NEXT:    v_mov_b32_e32 v2, v08910; GFX8-NEXT:    s_andn2_b64 exec, exec, s[8:9]8911; GFX8-NEXT:    s_cbranch_execnz .LBB119_18912; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end8913; GFX8-NEXT:    s_endpgm8914;8915; GFX12-LABEL: atomic_dec_i64:8916; GFX12:       ; %bb.0: ; %entry8917; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x248918; GFX12-NEXT:    s_wait_kmcnt 0x08919; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18920; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s38921; GFX12-NEXT:    flat_atomic_dec_u64 v[0:1], v[2:3] scope:SCOPE_DEV8922; GFX12-NEXT:    s_wait_storecnt_dscnt 0x08923; GFX12-NEXT:    global_inv scope:SCOPE_DEV8924; GFX12-NEXT:    s_endpgm8925entry:8926  %tmp0 = atomicrmw volatile udec_wrap ptr %out, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !08927  ret void8928}8929 8930define amdgpu_kernel void @atomic_dec_i64_ret(ptr %out, ptr %out2, i64 %in) {8931; GFX7-LABEL: atomic_dec_i64_ret:8932; GFX7:       ; %bb.0: ; %entry8933; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x98934; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd8935; GFX7-NEXT:    s_mov_b64 s[6:7], 08936; GFX7-NEXT:    s_waitcnt lgkmcnt(0)8937; GFX7-NEXT:    v_mov_b32_e32 v0, s88938; GFX7-NEXT:    v_mov_b32_e32 v1, s98939; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]8940; GFX7-NEXT:    v_mov_b32_e32 v4, s58941; GFX7-NEXT:    v_mov_b32_e32 v5, s48942; GFX7-NEXT:  .LBB120_1: ; %atomicrmw.start8943; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=18944; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8945; GFX7-NEXT:    v_mov_b32_e32 v9, v38946; GFX7-NEXT:    v_mov_b32_e32 v8, v28947; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]8948; GFX7-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[4:5], v[8:9]8949; GFX7-NEXT:    v_add_i32_e64 v2, s[2:3], -1, v88950; GFX7-NEXT:    v_addc_u32_e64 v3, s[2:3], -1, v9, s[2:3]8951; GFX7-NEXT:    s_or_b64 vcc, vcc, s[0:1]8952; GFX7-NEXT:    v_cndmask_b32_e32 v7, v3, v4, vcc8953; GFX7-NEXT:    v_cndmask_b32_e32 v6, v2, v5, vcc8954; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc8955; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8956; GFX7-NEXT:    buffer_wbinvl1_vol8957; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]8958; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]8959; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]8960; GFX7-NEXT:    s_cbranch_execnz .LBB120_18961; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end8962; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]8963; GFX7-NEXT:    v_mov_b32_e32 v0, s108964; GFX7-NEXT:    v_mov_b32_e32 v1, s118965; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]8966; GFX7-NEXT:    s_endpgm8967;8968; GFX8-LABEL: atomic_dec_i64_ret:8969; GFX8:       ; %bb.0: ; %entry8970; GFX8-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x248971; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x348972; GFX8-NEXT:    s_mov_b64 s[6:7], 08973; GFX8-NEXT:    s_waitcnt lgkmcnt(0)8974; GFX8-NEXT:    v_mov_b32_e32 v0, s88975; GFX8-NEXT:    v_mov_b32_e32 v1, s98976; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]8977; GFX8-NEXT:    v_mov_b32_e32 v4, s58978; GFX8-NEXT:    v_mov_b32_e32 v5, s48979; GFX8-NEXT:  .LBB120_1: ; %atomicrmw.start8980; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=18981; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8982; GFX8-NEXT:    v_mov_b32_e32 v9, v38983; GFX8-NEXT:    v_mov_b32_e32 v8, v28984; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]8985; GFX8-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[4:5], v[8:9]8986; GFX8-NEXT:    v_add_u32_e64 v2, s[2:3], -1, v88987; GFX8-NEXT:    v_addc_u32_e64 v3, s[2:3], -1, v9, s[2:3]8988; GFX8-NEXT:    s_or_b64 vcc, vcc, s[0:1]8989; GFX8-NEXT:    v_cndmask_b32_e32 v7, v3, v4, vcc8990; GFX8-NEXT:    v_cndmask_b32_e32 v6, v2, v5, vcc8991; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc8992; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)8993; GFX8-NEXT:    buffer_wbinvl1_vol8994; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]8995; GFX8-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]8996; GFX8-NEXT:    s_andn2_b64 exec, exec, s[6:7]8997; GFX8-NEXT:    s_cbranch_execnz .LBB120_18998; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end8999; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]9000; GFX8-NEXT:    v_mov_b32_e32 v0, s109001; GFX8-NEXT:    v_mov_b32_e32 v1, s119002; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]9003; GFX8-NEXT:    s_endpgm9004;9005; GFX12-LABEL: atomic_dec_i64_ret:9006; GFX12:       ; %bb.0: ; %entry9007; GFX12-NEXT:    s_clause 0x19008; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x249009; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x349010; GFX12-NEXT:    s_wait_kmcnt 0x09011; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s19012; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s59013; GFX12-NEXT:    flat_atomic_dec_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV9014; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x09015; GFX12-NEXT:    global_inv scope:SCOPE_DEV9016; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s39017; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]9018; GFX12-NEXT:    s_endpgm9019entry:9020  %tmp0 = atomicrmw volatile udec_wrap ptr %out, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !09021  store i64 %tmp0, ptr %out29022  ret void9023}9024 9025define amdgpu_kernel void @atomic_dec_i64_decr64(ptr %out, i64 %in, i64 %index) {9026; GFX7-LABEL: atomic_dec_i64_decr64:9027; GFX7:       ; %bb.0: ; %entry9028; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xd9029; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x99030; GFX7-NEXT:    s_waitcnt lgkmcnt(0)9031; GFX7-NEXT:    s_lshl_b64 s[0:1], s[0:1], 39032; GFX7-NEXT:    s_add_u32 s0, s4, s09033; GFX7-NEXT:    s_addc_u32 s1, s5, s19034; GFX7-NEXT:    v_mov_b32_e32 v5, s19035; GFX7-NEXT:    v_mov_b32_e32 v4, s09036; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]9037; GFX7-NEXT:    s_mov_b64 s[4:5], 09038; GFX7-NEXT:    v_mov_b32_e32 v6, s79039; GFX7-NEXT:    v_mov_b32_e32 v7, s69040; GFX7-NEXT:  .LBB121_1: ; %atomicrmw.start9041; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=19042; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)9043; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]9044; GFX7-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[6:7], v[2:3]9045; GFX7-NEXT:    v_add_i32_e64 v0, s[2:3], -1, v29046; GFX7-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v3, s[2:3]9047; GFX7-NEXT:    s_or_b64 vcc, vcc, s[0:1]9048; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc9049; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc9050; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc9051; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)9052; GFX7-NEXT:    buffer_wbinvl1_vol9053; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]9054; GFX7-NEXT:    v_mov_b32_e32 v3, v19055; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]9056; GFX7-NEXT:    v_mov_b32_e32 v2, v09057; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]9058; GFX7-NEXT:    s_cbranch_execnz .LBB121_19059; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end9060; GFX7-NEXT:    s_endpgm9061;9062; GFX8-LABEL: atomic_dec_i64_decr64:9063; GFX8:       ; %bb.0: ; %entry9064; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x349065; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x249066; GFX8-NEXT:    s_waitcnt lgkmcnt(0)9067; GFX8-NEXT:    s_lshl_b64 s[0:1], s[0:1], 39068; GFX8-NEXT:    s_add_u32 s0, s4, s09069; GFX8-NEXT:    s_addc_u32 s1, s5, s19070; GFX8-NEXT:    v_mov_b32_e32 v5, s19071; GFX8-NEXT:    v_mov_b32_e32 v4, s09072; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]9073; GFX8-NEXT:    s_mov_b64 s[4:5], 09074; GFX8-NEXT:    v_mov_b32_e32 v6, s79075; GFX8-NEXT:    v_mov_b32_e32 v7, s69076; GFX8-NEXT:  .LBB121_1: ; %atomicrmw.start9077; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=19078; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)9079; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]9080; GFX8-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[6:7], v[2:3]9081; GFX8-NEXT:    v_add_u32_e64 v0, s[2:3], -1, v29082; GFX8-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v3, s[2:3]9083; GFX8-NEXT:    s_or_b64 vcc, vcc, s[0:1]9084; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc9085; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc9086; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc9087; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)9088; GFX8-NEXT:    buffer_wbinvl1_vol9089; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]9090; GFX8-NEXT:    v_mov_b32_e32 v3, v19091; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]9092; GFX8-NEXT:    v_mov_b32_e32 v2, v09093; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]9094; GFX8-NEXT:    s_cbranch_execnz .LBB121_19095; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end9096; GFX8-NEXT:    s_endpgm9097;9098; GFX12-LABEL: atomic_dec_i64_decr64:9099; GFX12:       ; %bb.0: ; %entry9100; GFX12-NEXT:    s_clause 0x19101; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x349102; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x249103; GFX12-NEXT:    s_wait_kmcnt 0x09104; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 39105; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s39106; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]9107; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)9108; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s09109; GFX12-NEXT:    flat_atomic_dec_u64 v[2:3], v[0:1] scope:SCOPE_DEV9110; GFX12-NEXT:    s_wait_storecnt_dscnt 0x09111; GFX12-NEXT:    global_inv scope:SCOPE_DEV9112; GFX12-NEXT:    s_endpgm9113entry:9114  %ptr = getelementptr inbounds i64, ptr %out, i64 %index9115  %tmp0 = atomicrmw volatile udec_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !09116  ret void9117}9118 9119define amdgpu_kernel void @atomic_dec_i64_ret_decr64(ptr %out, ptr %out2, i64 %in, i64 %index) {9120; GFX7-LABEL: atomic_dec_i64_ret_decr64:9121; GFX7:       ; %bb.0: ; %entry9122; GFX7-NEXT:    s_load_dwordx8 s[4:11], s[4:5], 0x99123; GFX7-NEXT:    s_waitcnt lgkmcnt(0)9124; GFX7-NEXT:    s_lshl_b64 s[0:1], s[10:11], 39125; GFX7-NEXT:    s_add_u32 s0, s4, s09126; GFX7-NEXT:    s_addc_u32 s1, s5, s19127; GFX7-NEXT:    v_mov_b32_e32 v0, s09128; GFX7-NEXT:    v_mov_b32_e32 v1, s19129; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]9130; GFX7-NEXT:    s_mov_b64 s[4:5], 09131; GFX7-NEXT:    v_mov_b32_e32 v4, s99132; GFX7-NEXT:    v_mov_b32_e32 v5, s89133; GFX7-NEXT:  .LBB122_1: ; %atomicrmw.start9134; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=19135; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)9136; GFX7-NEXT:    v_mov_b32_e32 v9, v39137; GFX7-NEXT:    v_mov_b32_e32 v8, v29138; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]9139; GFX7-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[8:9], v[8:9]9140; GFX7-NEXT:    v_add_i32_e64 v2, s[2:3], -1, v89141; GFX7-NEXT:    v_addc_u32_e64 v3, s[2:3], -1, v9, s[2:3]9142; GFX7-NEXT:    s_or_b64 vcc, vcc, s[0:1]9143; GFX7-NEXT:    v_cndmask_b32_e32 v7, v3, v4, vcc9144; GFX7-NEXT:    v_cndmask_b32_e32 v6, v2, v5, vcc9145; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc9146; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)9147; GFX7-NEXT:    buffer_wbinvl1_vol9148; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]9149; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]9150; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]9151; GFX7-NEXT:    s_cbranch_execnz .LBB122_19152; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end9153; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]9154; GFX7-NEXT:    v_mov_b32_e32 v0, s69155; GFX7-NEXT:    v_mov_b32_e32 v1, s79156; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]9157; GFX7-NEXT:    s_endpgm9158;9159; GFX8-LABEL: atomic_dec_i64_ret_decr64:9160; GFX8:       ; %bb.0: ; %entry9161; GFX8-NEXT:    s_load_dwordx8 s[4:11], s[4:5], 0x249162; GFX8-NEXT:    s_waitcnt lgkmcnt(0)9163; GFX8-NEXT:    s_lshl_b64 s[0:1], s[10:11], 39164; GFX8-NEXT:    s_add_u32 s0, s4, s09165; GFX8-NEXT:    s_addc_u32 s1, s5, s19166; GFX8-NEXT:    v_mov_b32_e32 v0, s09167; GFX8-NEXT:    v_mov_b32_e32 v1, s19168; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]9169; GFX8-NEXT:    s_mov_b64 s[4:5], 09170; GFX8-NEXT:    v_mov_b32_e32 v4, s99171; GFX8-NEXT:    v_mov_b32_e32 v5, s89172; GFX8-NEXT:  .LBB122_1: ; %atomicrmw.start9173; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=19174; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)9175; GFX8-NEXT:    v_mov_b32_e32 v9, v39176; GFX8-NEXT:    v_mov_b32_e32 v8, v29177; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]9178; GFX8-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[8:9], v[8:9]9179; GFX8-NEXT:    v_add_u32_e64 v2, s[2:3], -1, v89180; GFX8-NEXT:    v_addc_u32_e64 v3, s[2:3], -1, v9, s[2:3]9181; GFX8-NEXT:    s_or_b64 vcc, vcc, s[0:1]9182; GFX8-NEXT:    v_cndmask_b32_e32 v7, v3, v4, vcc9183; GFX8-NEXT:    v_cndmask_b32_e32 v6, v2, v5, vcc9184; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc9185; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)9186; GFX8-NEXT:    buffer_wbinvl1_vol9187; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]9188; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]9189; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]9190; GFX8-NEXT:    s_cbranch_execnz .LBB122_19191; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end9192; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]9193; GFX8-NEXT:    v_mov_b32_e32 v0, s69194; GFX8-NEXT:    v_mov_b32_e32 v1, s79195; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]9196; GFX8-NEXT:    s_endpgm9197;9198; GFX12-LABEL: atomic_dec_i64_ret_decr64:9199; GFX12:       ; %bb.0: ; %entry9200; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x249201; GFX12-NEXT:    s_wait_kmcnt 0x09202; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 39203; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s59204; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]9205; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)9206; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s09207; GFX12-NEXT:    flat_atomic_dec_u64 v[0:1], v[2:3], v[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV9208; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x09209; GFX12-NEXT:    global_inv scope:SCOPE_DEV9210; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s39211; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]9212; GFX12-NEXT:    s_endpgm9213entry:9214  %ptr = getelementptr inbounds i64, ptr %out, i64 %index9215  %tmp0 = atomicrmw volatile udec_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst, !noalias.addrspace !09216  store i64 %tmp0, ptr %out29217  ret void9218}9219 9220!0 = !{i32 5, i32 6}9221