3642 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=bonaire -mattr=+flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s3; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=tonga < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI %s4; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s5; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX10 %s6; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11 %s7 8; FIXME: Merge with other test. DS offset folding doesn't work due to9; register bank copies, and no return optimization is missing.10 11@lds0 = internal addrspace(3) global [512 x i32] poison12@lds1 = internal addrspace(3) global [512 x i64] poison, align 813 14declare i32 @llvm.amdgcn.workitem.id.x() #015 16define amdgpu_kernel void @lds_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {17; CI-LABEL: lds_atomic_dec_ret_i32:18; CI: ; %bb.0:19; CI-NEXT: s_load_dword s2, s[8:9], 0x220; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x021; CI-NEXT: v_mov_b32_e32 v0, 4222; CI-NEXT: s_mov_b32 m0, -123; CI-NEXT: s_add_i32 s12, s12, s1724; CI-NEXT: s_waitcnt lgkmcnt(0)25; CI-NEXT: v_mov_b32_e32 v1, s226; CI-NEXT: ds_dec_rtn_u32 v2, v1, v027; CI-NEXT: s_waitcnt lgkmcnt(0)28; CI-NEXT: v_mov_b32_e32 v0, s029; CI-NEXT: s_mov_b32 flat_scratch_lo, s1330; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 831; CI-NEXT: v_mov_b32_e32 v1, s132; CI-NEXT: flat_store_dword v[0:1], v233; CI-NEXT: s_endpgm34;35; VI-LABEL: lds_atomic_dec_ret_i32:36; VI: ; %bb.0:37; VI-NEXT: s_load_dword s2, s[8:9], 0x838; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x039; VI-NEXT: v_mov_b32_e32 v0, 4240; VI-NEXT: s_mov_b32 m0, -141; VI-NEXT: s_add_i32 s12, s12, s1742; VI-NEXT: s_waitcnt lgkmcnt(0)43; VI-NEXT: v_mov_b32_e32 v1, s244; VI-NEXT: ds_dec_rtn_u32 v2, v1, v045; VI-NEXT: s_waitcnt lgkmcnt(0)46; VI-NEXT: v_mov_b32_e32 v0, s047; VI-NEXT: s_mov_b32 flat_scratch_lo, s1348; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 849; VI-NEXT: v_mov_b32_e32 v1, s150; VI-NEXT: flat_store_dword v[0:1], v251; VI-NEXT: s_endpgm52;53; GFX9-LABEL: lds_atomic_dec_ret_i32:54; GFX9: ; %bb.0:55; GFX9-NEXT: s_load_dword s2, s[8:9], 0x856; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x057; GFX9-NEXT: v_mov_b32_e32 v1, 4258; GFX9-NEXT: s_waitcnt lgkmcnt(0)59; GFX9-NEXT: v_mov_b32_e32 v0, s260; GFX9-NEXT: ds_dec_rtn_u32 v0, v0, v161; GFX9-NEXT: s_waitcnt lgkmcnt(0)62; GFX9-NEXT: v_mov_b32_e32 v1, 063; GFX9-NEXT: global_store_dword v1, v0, s[0:1]64; GFX9-NEXT: s_endpgm65;66; GFX10-LABEL: lds_atomic_dec_ret_i32:67; GFX10: ; %bb.0:68; GFX10-NEXT: s_load_dword s0, s[8:9], 0x869; GFX10-NEXT: v_mov_b32_e32 v1, 4270; GFX10-NEXT: s_waitcnt lgkmcnt(0)71; GFX10-NEXT: v_mov_b32_e32 v0, s072; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x073; GFX10-NEXT: s_waitcnt lgkmcnt(0)74; GFX10-NEXT: ds_dec_rtn_u32 v0, v0, v175; GFX10-NEXT: s_waitcnt lgkmcnt(0)76; GFX10-NEXT: buffer_gl0_inv77; GFX10-NEXT: v_mov_b32_e32 v1, 078; GFX10-NEXT: global_store_dword v1, v0, s[0:1]79; GFX10-NEXT: s_endpgm80;81; GFX11-LABEL: lds_atomic_dec_ret_i32:82; GFX11: ; %bb.0:83; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x884; GFX11-NEXT: s_waitcnt lgkmcnt(0)85; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s086; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x087; GFX11-NEXT: s_waitcnt lgkmcnt(0)88; GFX11-NEXT: ds_dec_rtn_u32 v0, v0, v189; GFX11-NEXT: s_waitcnt lgkmcnt(0)90; GFX11-NEXT: buffer_gl0_inv91; GFX11-NEXT: v_mov_b32_e32 v1, 092; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]93; GFX11-NEXT: s_endpgm94 %result = atomicrmw udec_wrap ptr addrspace(3) %ptr, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !195 store i32 %result, ptr addrspace(1) %out, align 496 ret void97}98 99define amdgpu_kernel void @lds_atomic_dec_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {100; CI-LABEL: lds_atomic_dec_ret_i32_offset:101; CI: ; %bb.0:102; CI-NEXT: s_load_dword s2, s[8:9], 0x2103; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0104; CI-NEXT: v_mov_b32_e32 v0, 42105; CI-NEXT: s_mov_b32 m0, -1106; CI-NEXT: s_add_i32 s12, s12, s17107; CI-NEXT: s_waitcnt lgkmcnt(0)108; CI-NEXT: v_mov_b32_e32 v1, s2109; CI-NEXT: ds_dec_rtn_u32 v2, v1, v0 offset:16110; CI-NEXT: s_waitcnt lgkmcnt(0)111; CI-NEXT: v_mov_b32_e32 v0, s0112; CI-NEXT: s_mov_b32 flat_scratch_lo, s13113; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8114; CI-NEXT: v_mov_b32_e32 v1, s1115; CI-NEXT: flat_store_dword v[0:1], v2116; CI-NEXT: s_endpgm117;118; VI-LABEL: lds_atomic_dec_ret_i32_offset:119; VI: ; %bb.0:120; VI-NEXT: s_load_dword s2, s[8:9], 0x8121; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0122; VI-NEXT: v_mov_b32_e32 v0, 42123; VI-NEXT: s_mov_b32 m0, -1124; VI-NEXT: s_add_i32 s12, s12, s17125; VI-NEXT: s_waitcnt lgkmcnt(0)126; VI-NEXT: v_mov_b32_e32 v1, s2127; VI-NEXT: ds_dec_rtn_u32 v2, v1, v0 offset:16128; VI-NEXT: s_waitcnt lgkmcnt(0)129; VI-NEXT: v_mov_b32_e32 v0, s0130; VI-NEXT: s_mov_b32 flat_scratch_lo, s13131; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8132; VI-NEXT: v_mov_b32_e32 v1, s1133; VI-NEXT: flat_store_dword v[0:1], v2134; VI-NEXT: s_endpgm135;136; GFX9-LABEL: lds_atomic_dec_ret_i32_offset:137; GFX9: ; %bb.0:138; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8139; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0140; GFX9-NEXT: v_mov_b32_e32 v0, 42141; GFX9-NEXT: s_waitcnt lgkmcnt(0)142; GFX9-NEXT: v_mov_b32_e32 v1, s2143; GFX9-NEXT: ds_dec_rtn_u32 v0, v1, v0 offset:16144; GFX9-NEXT: s_waitcnt lgkmcnt(0)145; GFX9-NEXT: v_mov_b32_e32 v1, 0146; GFX9-NEXT: global_store_dword v1, v0, s[0:1]147; GFX9-NEXT: s_endpgm148;149; GFX10-LABEL: lds_atomic_dec_ret_i32_offset:150; GFX10: ; %bb.0:151; GFX10-NEXT: s_load_dword s0, s[8:9], 0x8152; GFX10-NEXT: v_mov_b32_e32 v0, 42153; GFX10-NEXT: s_waitcnt lgkmcnt(0)154; GFX10-NEXT: v_mov_b32_e32 v1, s0155; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0156; GFX10-NEXT: s_waitcnt lgkmcnt(0)157; GFX10-NEXT: ds_dec_rtn_u32 v0, v1, v0 offset:16158; GFX10-NEXT: s_waitcnt lgkmcnt(0)159; GFX10-NEXT: buffer_gl0_inv160; GFX10-NEXT: v_mov_b32_e32 v1, 0161; GFX10-NEXT: global_store_dword v1, v0, s[0:1]162; GFX10-NEXT: s_endpgm163;164; GFX11-LABEL: lds_atomic_dec_ret_i32_offset:165; GFX11: ; %bb.0:166; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8167; GFX11-NEXT: s_waitcnt lgkmcnt(0)168; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, s0169; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0170; GFX11-NEXT: s_waitcnt lgkmcnt(0)171; GFX11-NEXT: ds_dec_rtn_u32 v0, v1, v0 offset:16172; GFX11-NEXT: s_waitcnt lgkmcnt(0)173; GFX11-NEXT: buffer_gl0_inv174; GFX11-NEXT: v_mov_b32_e32 v1, 0175; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]176; GFX11-NEXT: s_endpgm177 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4178 %result = atomicrmw udec_wrap ptr addrspace(3) %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1179 store i32 %result, ptr addrspace(1) %out, align 4180 ret void181}182 183define amdgpu_kernel void @lds_atomic_dec_noret_i32(ptr addrspace(3) %ptr) #1 {184; CI-LABEL: lds_atomic_dec_noret_i32:185; CI: ; %bb.0:186; CI-NEXT: s_load_dword s0, s[8:9], 0x0187; CI-NEXT: v_mov_b32_e32 v0, 42188; CI-NEXT: s_mov_b32 m0, -1189; CI-NEXT: s_waitcnt lgkmcnt(0)190; CI-NEXT: v_mov_b32_e32 v1, s0191; CI-NEXT: ds_dec_u32 v1, v0192; CI-NEXT: s_waitcnt lgkmcnt(0)193; CI-NEXT: s_endpgm194;195; VI-LABEL: lds_atomic_dec_noret_i32:196; VI: ; %bb.0:197; VI-NEXT: s_load_dword s0, s[8:9], 0x0198; VI-NEXT: v_mov_b32_e32 v0, 42199; VI-NEXT: s_mov_b32 m0, -1200; VI-NEXT: s_waitcnt lgkmcnt(0)201; VI-NEXT: v_mov_b32_e32 v1, s0202; VI-NEXT: ds_dec_u32 v1, v0203; VI-NEXT: s_waitcnt lgkmcnt(0)204; VI-NEXT: s_endpgm205;206; GFX9-LABEL: lds_atomic_dec_noret_i32:207; GFX9: ; %bb.0:208; GFX9-NEXT: s_load_dword s0, s[8:9], 0x0209; GFX9-NEXT: v_mov_b32_e32 v1, 42210; GFX9-NEXT: s_waitcnt lgkmcnt(0)211; GFX9-NEXT: v_mov_b32_e32 v0, s0212; GFX9-NEXT: ds_dec_u32 v0, v1213; GFX9-NEXT: s_waitcnt lgkmcnt(0)214; GFX9-NEXT: s_endpgm215;216; GFX10-LABEL: lds_atomic_dec_noret_i32:217; GFX10: ; %bb.0:218; GFX10-NEXT: s_load_dword s0, s[8:9], 0x0219; GFX10-NEXT: v_mov_b32_e32 v1, 42220; GFX10-NEXT: s_waitcnt lgkmcnt(0)221; GFX10-NEXT: v_mov_b32_e32 v0, s0222; GFX10-NEXT: ds_dec_u32 v0, v1223; GFX10-NEXT: s_waitcnt lgkmcnt(0)224; GFX10-NEXT: buffer_gl0_inv225; GFX10-NEXT: s_endpgm226;227; GFX11-LABEL: lds_atomic_dec_noret_i32:228; GFX11: ; %bb.0:229; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x0230; GFX11-NEXT: s_waitcnt lgkmcnt(0)231; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s0232; GFX11-NEXT: ds_dec_u32 v0, v1233; GFX11-NEXT: s_waitcnt lgkmcnt(0)234; GFX11-NEXT: buffer_gl0_inv235; GFX11-NEXT: s_endpgm236 %result = atomicrmw udec_wrap ptr addrspace(3) %ptr, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1237 ret void238}239 240define amdgpu_kernel void @lds_atomic_dec_noret_i32_offset(ptr addrspace(3) %ptr) #1 {241; CI-LABEL: lds_atomic_dec_noret_i32_offset:242; CI: ; %bb.0:243; CI-NEXT: s_load_dword s0, s[8:9], 0x0244; CI-NEXT: v_mov_b32_e32 v0, 42245; CI-NEXT: s_mov_b32 m0, -1246; CI-NEXT: s_waitcnt lgkmcnt(0)247; CI-NEXT: v_mov_b32_e32 v1, s0248; CI-NEXT: ds_dec_u32 v1, v0 offset:16249; CI-NEXT: s_waitcnt lgkmcnt(0)250; CI-NEXT: s_endpgm251;252; VI-LABEL: lds_atomic_dec_noret_i32_offset:253; VI: ; %bb.0:254; VI-NEXT: s_load_dword s0, s[8:9], 0x0255; VI-NEXT: v_mov_b32_e32 v0, 42256; VI-NEXT: s_mov_b32 m0, -1257; VI-NEXT: s_waitcnt lgkmcnt(0)258; VI-NEXT: v_mov_b32_e32 v1, s0259; VI-NEXT: ds_dec_u32 v1, v0 offset:16260; VI-NEXT: s_waitcnt lgkmcnt(0)261; VI-NEXT: s_endpgm262;263; GFX9-LABEL: lds_atomic_dec_noret_i32_offset:264; GFX9: ; %bb.0:265; GFX9-NEXT: s_load_dword s0, s[8:9], 0x0266; GFX9-NEXT: v_mov_b32_e32 v0, 42267; GFX9-NEXT: s_waitcnt lgkmcnt(0)268; GFX9-NEXT: v_mov_b32_e32 v1, s0269; GFX9-NEXT: ds_dec_u32 v1, v0 offset:16270; GFX9-NEXT: s_waitcnt lgkmcnt(0)271; GFX9-NEXT: s_endpgm272;273; GFX10-LABEL: lds_atomic_dec_noret_i32_offset:274; GFX10: ; %bb.0:275; GFX10-NEXT: s_load_dword s0, s[8:9], 0x0276; GFX10-NEXT: v_mov_b32_e32 v0, 42277; GFX10-NEXT: s_waitcnt lgkmcnt(0)278; GFX10-NEXT: v_mov_b32_e32 v1, s0279; GFX10-NEXT: ds_dec_u32 v1, v0 offset:16280; GFX10-NEXT: s_waitcnt lgkmcnt(0)281; GFX10-NEXT: buffer_gl0_inv282; GFX10-NEXT: s_endpgm283;284; GFX11-LABEL: lds_atomic_dec_noret_i32_offset:285; GFX11: ; %bb.0:286; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x0287; GFX11-NEXT: s_waitcnt lgkmcnt(0)288; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, s0289; GFX11-NEXT: ds_dec_u32 v1, v0 offset:16290; GFX11-NEXT: s_waitcnt lgkmcnt(0)291; GFX11-NEXT: buffer_gl0_inv292; GFX11-NEXT: s_endpgm293 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4294 %result = atomicrmw udec_wrap ptr addrspace(3) %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1295 ret void296}297 298define amdgpu_kernel void @global_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #1 {299; CI-LABEL: global_atomic_dec_ret_i32:300; CI: ; %bb.0:301; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0302; CI-NEXT: s_add_i32 s12, s12, s17303; CI-NEXT: s_mov_b32 flat_scratch_lo, s13304; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8305; CI-NEXT: v_mov_b32_e32 v2, 42306; CI-NEXT: s_waitcnt lgkmcnt(0)307; CI-NEXT: v_mov_b32_e32 v0, s2308; CI-NEXT: v_mov_b32_e32 v1, s3309; CI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc310; CI-NEXT: s_waitcnt vmcnt(0)311; CI-NEXT: buffer_wbinvl1_vol312; CI-NEXT: v_mov_b32_e32 v0, s0313; CI-NEXT: v_mov_b32_e32 v1, s1314; CI-NEXT: flat_store_dword v[0:1], v2315; CI-NEXT: s_endpgm316;317; VI-LABEL: global_atomic_dec_ret_i32:318; VI: ; %bb.0:319; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0320; VI-NEXT: s_add_i32 s12, s12, s17321; VI-NEXT: s_mov_b32 flat_scratch_lo, s13322; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8323; VI-NEXT: v_mov_b32_e32 v2, 42324; VI-NEXT: s_waitcnt lgkmcnt(0)325; VI-NEXT: v_mov_b32_e32 v0, s2326; VI-NEXT: v_mov_b32_e32 v1, s3327; VI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc328; VI-NEXT: s_waitcnt vmcnt(0)329; VI-NEXT: buffer_wbinvl1_vol330; VI-NEXT: v_mov_b32_e32 v0, s0331; VI-NEXT: v_mov_b32_e32 v1, s1332; VI-NEXT: flat_store_dword v[0:1], v2333; VI-NEXT: s_endpgm334;335; GFX9-LABEL: global_atomic_dec_ret_i32:336; GFX9: ; %bb.0:337; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0338; GFX9-NEXT: v_mov_b32_e32 v0, 42339; GFX9-NEXT: v_mov_b32_e32 v1, 0340; GFX9-NEXT: s_waitcnt lgkmcnt(0)341; GFX9-NEXT: global_atomic_dec v0, v1, v0, s[2:3] glc342; GFX9-NEXT: s_waitcnt vmcnt(0)343; GFX9-NEXT: buffer_wbinvl1_vol344; GFX9-NEXT: global_store_dword v1, v0, s[0:1]345; GFX9-NEXT: s_endpgm346;347; GFX10-LABEL: global_atomic_dec_ret_i32:348; GFX10: ; %bb.0:349; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0350; GFX10-NEXT: v_mov_b32_e32 v0, 42351; GFX10-NEXT: v_mov_b32_e32 v1, 0352; GFX10-NEXT: s_waitcnt lgkmcnt(0)353; GFX10-NEXT: global_atomic_dec v0, v1, v0, s[2:3] glc354; GFX10-NEXT: s_waitcnt vmcnt(0)355; GFX10-NEXT: buffer_gl1_inv356; GFX10-NEXT: buffer_gl0_inv357; GFX10-NEXT: global_store_dword v1, v0, s[0:1]358; GFX10-NEXT: s_endpgm359;360; GFX11-LABEL: global_atomic_dec_ret_i32:361; GFX11: ; %bb.0:362; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0363; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0364; GFX11-NEXT: s_waitcnt lgkmcnt(0)365; GFX11-NEXT: global_atomic_dec_u32 v0, v1, v0, s[2:3] glc366; GFX11-NEXT: s_waitcnt vmcnt(0)367; GFX11-NEXT: buffer_gl1_inv368; GFX11-NEXT: buffer_gl0_inv369; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]370; GFX11-NEXT: s_endpgm371 %result = atomicrmw udec_wrap ptr addrspace(1) %ptr, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1372 store i32 %result, ptr addrspace(1) %out, align 4373 ret void374}375 376define amdgpu_kernel void @global_atomic_dec_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #1 {377; CI-LABEL: global_atomic_dec_ret_i32_offset:378; CI: ; %bb.0:379; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0380; CI-NEXT: s_add_i32 s12, s12, s17381; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8382; CI-NEXT: s_mov_b32 flat_scratch_lo, s13383; CI-NEXT: v_mov_b32_e32 v2, 42384; CI-NEXT: s_waitcnt lgkmcnt(0)385; CI-NEXT: s_add_u32 s2, s2, 16386; CI-NEXT: s_addc_u32 s3, s3, 0387; CI-NEXT: v_mov_b32_e32 v0, s2388; CI-NEXT: v_mov_b32_e32 v1, s3389; CI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc390; CI-NEXT: s_waitcnt vmcnt(0)391; CI-NEXT: buffer_wbinvl1_vol392; CI-NEXT: v_mov_b32_e32 v0, s0393; CI-NEXT: v_mov_b32_e32 v1, s1394; CI-NEXT: flat_store_dword v[0:1], v2395; CI-NEXT: s_endpgm396;397; VI-LABEL: global_atomic_dec_ret_i32_offset:398; VI: ; %bb.0:399; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0400; VI-NEXT: s_add_i32 s12, s12, s17401; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8402; VI-NEXT: s_mov_b32 flat_scratch_lo, s13403; VI-NEXT: v_mov_b32_e32 v2, 42404; VI-NEXT: s_waitcnt lgkmcnt(0)405; VI-NEXT: s_add_u32 s2, s2, 16406; VI-NEXT: s_addc_u32 s3, s3, 0407; VI-NEXT: v_mov_b32_e32 v0, s2408; VI-NEXT: v_mov_b32_e32 v1, s3409; VI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc410; VI-NEXT: s_waitcnt vmcnt(0)411; VI-NEXT: buffer_wbinvl1_vol412; VI-NEXT: v_mov_b32_e32 v0, s0413; VI-NEXT: v_mov_b32_e32 v1, s1414; VI-NEXT: flat_store_dword v[0:1], v2415; VI-NEXT: s_endpgm416;417; GFX9-LABEL: global_atomic_dec_ret_i32_offset:418; GFX9: ; %bb.0:419; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0420; GFX9-NEXT: v_mov_b32_e32 v0, 42421; GFX9-NEXT: v_mov_b32_e32 v1, 0422; GFX9-NEXT: s_waitcnt lgkmcnt(0)423; GFX9-NEXT: global_atomic_dec v0, v1, v0, s[2:3] offset:16 glc424; GFX9-NEXT: s_waitcnt vmcnt(0)425; GFX9-NEXT: buffer_wbinvl1_vol426; GFX9-NEXT: global_store_dword v1, v0, s[0:1]427; GFX9-NEXT: s_endpgm428;429; GFX10-LABEL: global_atomic_dec_ret_i32_offset:430; GFX10: ; %bb.0:431; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0432; GFX10-NEXT: v_mov_b32_e32 v0, 42433; GFX10-NEXT: v_mov_b32_e32 v1, 0434; GFX10-NEXT: s_waitcnt lgkmcnt(0)435; GFX10-NEXT: global_atomic_dec v0, v1, v0, s[2:3] offset:16 glc436; GFX10-NEXT: s_waitcnt vmcnt(0)437; GFX10-NEXT: buffer_gl1_inv438; GFX10-NEXT: buffer_gl0_inv439; GFX10-NEXT: global_store_dword v1, v0, s[0:1]440; GFX10-NEXT: s_endpgm441;442; GFX11-LABEL: global_atomic_dec_ret_i32_offset:443; GFX11: ; %bb.0:444; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0445; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0446; GFX11-NEXT: s_waitcnt lgkmcnt(0)447; GFX11-NEXT: global_atomic_dec_u32 v0, v1, v0, s[2:3] offset:16 glc448; GFX11-NEXT: s_waitcnt vmcnt(0)449; GFX11-NEXT: buffer_gl1_inv450; GFX11-NEXT: buffer_gl0_inv451; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]452; GFX11-NEXT: s_endpgm453 %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4454 %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1455 store i32 %result, ptr addrspace(1) %out, align 4456 ret void457}458 459define amdgpu_kernel void @global_atomic_dec_ret_i32_offset_system(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #1 {460; CI-LABEL: global_atomic_dec_ret_i32_offset_system:461; CI: ; %bb.0:462; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0463; CI-NEXT: s_add_i32 s12, s12, s17464; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8465; CI-NEXT: s_mov_b32 flat_scratch_lo, s13466; CI-NEXT: v_mov_b32_e32 v2, 42467; CI-NEXT: s_waitcnt lgkmcnt(0)468; CI-NEXT: s_add_u32 s2, s2, 16469; CI-NEXT: s_addc_u32 s3, s3, 0470; CI-NEXT: v_mov_b32_e32 v0, s2471; CI-NEXT: v_mov_b32_e32 v1, s3472; CI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc473; CI-NEXT: s_waitcnt vmcnt(0)474; CI-NEXT: buffer_wbinvl1_vol475; CI-NEXT: v_mov_b32_e32 v0, s0476; CI-NEXT: v_mov_b32_e32 v1, s1477; CI-NEXT: flat_store_dword v[0:1], v2478; CI-NEXT: s_endpgm479;480; VI-LABEL: global_atomic_dec_ret_i32_offset_system:481; VI: ; %bb.0:482; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0483; VI-NEXT: s_add_i32 s12, s12, s17484; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8485; VI-NEXT: s_mov_b32 flat_scratch_lo, s13486; VI-NEXT: v_mov_b32_e32 v2, 42487; VI-NEXT: s_waitcnt lgkmcnt(0)488; VI-NEXT: s_add_u32 s2, s2, 16489; VI-NEXT: s_addc_u32 s3, s3, 0490; VI-NEXT: v_mov_b32_e32 v0, s2491; VI-NEXT: v_mov_b32_e32 v1, s3492; VI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc493; VI-NEXT: s_waitcnt vmcnt(0)494; VI-NEXT: buffer_wbinvl1_vol495; VI-NEXT: v_mov_b32_e32 v0, s0496; VI-NEXT: v_mov_b32_e32 v1, s1497; VI-NEXT: flat_store_dword v[0:1], v2498; VI-NEXT: s_endpgm499;500; GFX9-LABEL: global_atomic_dec_ret_i32_offset_system:501; GFX9: ; %bb.0:502; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0503; GFX9-NEXT: v_mov_b32_e32 v0, 42504; GFX9-NEXT: v_mov_b32_e32 v1, 0505; GFX9-NEXT: s_waitcnt lgkmcnt(0)506; GFX9-NEXT: global_atomic_dec v0, v1, v0, s[2:3] offset:16 glc507; GFX9-NEXT: s_waitcnt vmcnt(0)508; GFX9-NEXT: buffer_wbinvl1_vol509; GFX9-NEXT: global_store_dword v1, v0, s[0:1]510; GFX9-NEXT: s_endpgm511;512; GFX10-LABEL: global_atomic_dec_ret_i32_offset_system:513; GFX10: ; %bb.0:514; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0515; GFX10-NEXT: v_mov_b32_e32 v0, 42516; GFX10-NEXT: v_mov_b32_e32 v1, 0517; GFX10-NEXT: s_waitcnt lgkmcnt(0)518; GFX10-NEXT: global_atomic_dec v0, v1, v0, s[2:3] offset:16 glc519; GFX10-NEXT: s_waitcnt vmcnt(0)520; GFX10-NEXT: buffer_gl1_inv521; GFX10-NEXT: buffer_gl0_inv522; GFX10-NEXT: global_store_dword v1, v0, s[0:1]523; GFX10-NEXT: s_endpgm524;525; GFX11-LABEL: global_atomic_dec_ret_i32_offset_system:526; GFX11: ; %bb.0:527; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0528; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0529; GFX11-NEXT: s_waitcnt lgkmcnt(0)530; GFX11-NEXT: global_atomic_dec_u32 v0, v1, v0, s[2:3] offset:16 glc531; GFX11-NEXT: s_waitcnt vmcnt(0)532; GFX11-NEXT: buffer_gl1_inv533; GFX11-NEXT: buffer_gl0_inv534; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]535; GFX11-NEXT: s_endpgm536 %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4537 %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 42 seq_cst, align 4, !amdgpu.no.remote.memory !1538 store i32 %result, ptr addrspace(1) %out, align 4539 ret void540}541 542define amdgpu_kernel void @global_atomic_dec_noret_i32(ptr addrspace(1) %ptr) #1 {543; CI-LABEL: global_atomic_dec_noret_i32:544; CI: ; %bb.0:545; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0546; CI-NEXT: s_add_i32 s12, s12, s17547; CI-NEXT: s_mov_b32 flat_scratch_lo, s13548; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8549; CI-NEXT: v_mov_b32_e32 v2, 42550; CI-NEXT: s_waitcnt lgkmcnt(0)551; CI-NEXT: v_mov_b32_e32 v0, s0552; CI-NEXT: v_mov_b32_e32 v1, s1553; CI-NEXT: flat_atomic_dec v[0:1], v2554; CI-NEXT: s_waitcnt vmcnt(0)555; CI-NEXT: buffer_wbinvl1_vol556; CI-NEXT: s_endpgm557;558; VI-LABEL: global_atomic_dec_noret_i32:559; VI: ; %bb.0:560; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0561; VI-NEXT: s_add_i32 s12, s12, s17562; VI-NEXT: s_mov_b32 flat_scratch_lo, s13563; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8564; VI-NEXT: v_mov_b32_e32 v2, 42565; VI-NEXT: s_waitcnt lgkmcnt(0)566; VI-NEXT: v_mov_b32_e32 v0, s0567; VI-NEXT: v_mov_b32_e32 v1, s1568; VI-NEXT: flat_atomic_dec v[0:1], v2569; VI-NEXT: s_waitcnt vmcnt(0)570; VI-NEXT: buffer_wbinvl1_vol571; VI-NEXT: s_endpgm572;573; GFX9-LABEL: global_atomic_dec_noret_i32:574; GFX9: ; %bb.0:575; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0576; GFX9-NEXT: v_mov_b32_e32 v0, 42577; GFX9-NEXT: v_mov_b32_e32 v1, 0578; GFX9-NEXT: s_waitcnt lgkmcnt(0)579; GFX9-NEXT: global_atomic_dec v1, v0, s[0:1]580; GFX9-NEXT: s_waitcnt vmcnt(0)581; GFX9-NEXT: buffer_wbinvl1_vol582; GFX9-NEXT: s_endpgm583;584; GFX10-LABEL: global_atomic_dec_noret_i32:585; GFX10: ; %bb.0:586; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0587; GFX10-NEXT: v_mov_b32_e32 v0, 42588; GFX10-NEXT: v_mov_b32_e32 v1, 0589; GFX10-NEXT: s_waitcnt lgkmcnt(0)590; GFX10-NEXT: global_atomic_dec v1, v0, s[0:1]591; GFX10-NEXT: s_waitcnt_vscnt null, 0x0592; GFX10-NEXT: buffer_gl1_inv593; GFX10-NEXT: buffer_gl0_inv594; GFX10-NEXT: s_endpgm595;596; GFX11-LABEL: global_atomic_dec_noret_i32:597; GFX11: ; %bb.0:598; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0599; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0600; GFX11-NEXT: s_waitcnt lgkmcnt(0)601; GFX11-NEXT: global_atomic_dec_u32 v1, v0, s[0:1]602; GFX11-NEXT: s_waitcnt_vscnt null, 0x0603; GFX11-NEXT: buffer_gl1_inv604; GFX11-NEXT: buffer_gl0_inv605; GFX11-NEXT: s_endpgm606 %result = atomicrmw udec_wrap ptr addrspace(1) %ptr, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1607 ret void608}609 610define amdgpu_kernel void @global_atomic_dec_noret_i32_offset(ptr addrspace(1) %ptr) #1 {611; CI-LABEL: global_atomic_dec_noret_i32_offset:612; CI: ; %bb.0:613; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0614; CI-NEXT: s_add_i32 s12, s12, s17615; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8616; CI-NEXT: s_mov_b32 flat_scratch_lo, s13617; CI-NEXT: v_mov_b32_e32 v2, 42618; CI-NEXT: s_waitcnt lgkmcnt(0)619; CI-NEXT: s_add_u32 s0, s0, 16620; CI-NEXT: s_addc_u32 s1, s1, 0621; CI-NEXT: v_mov_b32_e32 v0, s0622; CI-NEXT: v_mov_b32_e32 v1, s1623; CI-NEXT: flat_atomic_dec v[0:1], v2624; CI-NEXT: s_waitcnt vmcnt(0)625; CI-NEXT: buffer_wbinvl1_vol626; CI-NEXT: s_endpgm627;628; VI-LABEL: global_atomic_dec_noret_i32_offset:629; VI: ; %bb.0:630; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0631; VI-NEXT: s_add_i32 s12, s12, s17632; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8633; VI-NEXT: s_mov_b32 flat_scratch_lo, s13634; VI-NEXT: v_mov_b32_e32 v2, 42635; VI-NEXT: s_waitcnt lgkmcnt(0)636; VI-NEXT: s_add_u32 s0, s0, 16637; VI-NEXT: s_addc_u32 s1, s1, 0638; VI-NEXT: v_mov_b32_e32 v0, s0639; VI-NEXT: v_mov_b32_e32 v1, s1640; VI-NEXT: flat_atomic_dec v[0:1], v2641; VI-NEXT: s_waitcnt vmcnt(0)642; VI-NEXT: buffer_wbinvl1_vol643; VI-NEXT: s_endpgm644;645; GFX9-LABEL: global_atomic_dec_noret_i32_offset:646; GFX9: ; %bb.0:647; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0648; GFX9-NEXT: v_mov_b32_e32 v0, 42649; GFX9-NEXT: v_mov_b32_e32 v1, 0650; GFX9-NEXT: s_waitcnt lgkmcnt(0)651; GFX9-NEXT: global_atomic_dec v1, v0, s[0:1] offset:16652; GFX9-NEXT: s_waitcnt vmcnt(0)653; GFX9-NEXT: buffer_wbinvl1_vol654; GFX9-NEXT: s_endpgm655;656; GFX10-LABEL: global_atomic_dec_noret_i32_offset:657; GFX10: ; %bb.0:658; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0659; GFX10-NEXT: v_mov_b32_e32 v0, 42660; GFX10-NEXT: v_mov_b32_e32 v1, 0661; GFX10-NEXT: s_waitcnt lgkmcnt(0)662; GFX10-NEXT: global_atomic_dec v1, v0, s[0:1] offset:16663; GFX10-NEXT: s_waitcnt_vscnt null, 0x0664; GFX10-NEXT: buffer_gl1_inv665; GFX10-NEXT: buffer_gl0_inv666; GFX10-NEXT: s_endpgm667;668; GFX11-LABEL: global_atomic_dec_noret_i32_offset:669; GFX11: ; %bb.0:670; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0671; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0672; GFX11-NEXT: s_waitcnt lgkmcnt(0)673; GFX11-NEXT: global_atomic_dec_u32 v1, v0, s[0:1] offset:16674; GFX11-NEXT: s_waitcnt_vscnt null, 0x0675; GFX11-NEXT: buffer_gl1_inv676; GFX11-NEXT: buffer_gl0_inv677; GFX11-NEXT: s_endpgm678 %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4679 %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1680 ret void681}682 683define amdgpu_kernel void @global_atomic_dec_noret_i32_offset_system(ptr addrspace(1) %ptr) #1 {684; CI-LABEL: global_atomic_dec_noret_i32_offset_system:685; CI: ; %bb.0:686; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0687; CI-NEXT: s_add_i32 s12, s12, s17688; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8689; CI-NEXT: s_mov_b32 flat_scratch_lo, s13690; CI-NEXT: v_mov_b32_e32 v2, 42691; CI-NEXT: s_waitcnt lgkmcnt(0)692; CI-NEXT: s_add_u32 s0, s0, 16693; CI-NEXT: s_addc_u32 s1, s1, 0694; CI-NEXT: v_mov_b32_e32 v0, s0695; CI-NEXT: v_mov_b32_e32 v1, s1696; CI-NEXT: flat_atomic_dec v[0:1], v2697; CI-NEXT: s_waitcnt vmcnt(0)698; CI-NEXT: buffer_wbinvl1_vol699; CI-NEXT: s_endpgm700;701; VI-LABEL: global_atomic_dec_noret_i32_offset_system:702; VI: ; %bb.0:703; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0704; VI-NEXT: s_add_i32 s12, s12, s17705; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8706; VI-NEXT: s_mov_b32 flat_scratch_lo, s13707; VI-NEXT: v_mov_b32_e32 v2, 42708; VI-NEXT: s_waitcnt lgkmcnt(0)709; VI-NEXT: s_add_u32 s0, s0, 16710; VI-NEXT: s_addc_u32 s1, s1, 0711; VI-NEXT: v_mov_b32_e32 v0, s0712; VI-NEXT: v_mov_b32_e32 v1, s1713; VI-NEXT: flat_atomic_dec v[0:1], v2714; VI-NEXT: s_waitcnt vmcnt(0)715; VI-NEXT: buffer_wbinvl1_vol716; VI-NEXT: s_endpgm717;718; GFX9-LABEL: global_atomic_dec_noret_i32_offset_system:719; GFX9: ; %bb.0:720; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0721; GFX9-NEXT: v_mov_b32_e32 v0, 42722; GFX9-NEXT: v_mov_b32_e32 v1, 0723; GFX9-NEXT: s_waitcnt lgkmcnt(0)724; GFX9-NEXT: global_atomic_dec v1, v0, s[0:1] offset:16725; GFX9-NEXT: s_waitcnt vmcnt(0)726; GFX9-NEXT: buffer_wbinvl1_vol727; GFX9-NEXT: s_endpgm728;729; GFX10-LABEL: global_atomic_dec_noret_i32_offset_system:730; GFX10: ; %bb.0:731; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0732; GFX10-NEXT: v_mov_b32_e32 v0, 42733; GFX10-NEXT: v_mov_b32_e32 v1, 0734; GFX10-NEXT: s_waitcnt lgkmcnt(0)735; GFX10-NEXT: global_atomic_dec v1, v0, s[0:1] offset:16736; GFX10-NEXT: s_waitcnt_vscnt null, 0x0737; GFX10-NEXT: buffer_gl1_inv738; GFX10-NEXT: buffer_gl0_inv739; GFX10-NEXT: s_endpgm740;741; GFX11-LABEL: global_atomic_dec_noret_i32_offset_system:742; GFX11: ; %bb.0:743; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0744; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0745; GFX11-NEXT: s_waitcnt lgkmcnt(0)746; GFX11-NEXT: global_atomic_dec_u32 v1, v0, s[0:1] offset:16747; GFX11-NEXT: s_waitcnt_vscnt null, 0x0748; GFX11-NEXT: buffer_gl1_inv749; GFX11-NEXT: buffer_gl0_inv750; GFX11-NEXT: s_endpgm751 %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4752 %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 42 seq_cst, align 4, !amdgpu.no.remote.memory !1753 ret void754}755 756define amdgpu_kernel void @global_atomic_dec_ret_i32_offset_addr64(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #1 {757; CI-LABEL: global_atomic_dec_ret_i32_offset_addr64:758; CI: ; %bb.0:759; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0760; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v0761; CI-NEXT: s_add_i32 s12, s12, s17762; CI-NEXT: s_mov_b32 flat_scratch_lo, s13763; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8764; CI-NEXT: s_waitcnt lgkmcnt(0)765; CI-NEXT: v_mov_b32_e32 v0, s2766; CI-NEXT: v_mov_b32_e32 v1, s3767; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2768; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc769; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v0770; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc771; CI-NEXT: v_mov_b32_e32 v3, 42772; CI-NEXT: flat_atomic_dec v3, v[0:1], v3 glc773; CI-NEXT: s_waitcnt vmcnt(0)774; CI-NEXT: buffer_wbinvl1_vol775; CI-NEXT: v_mov_b32_e32 v0, s0776; CI-NEXT: v_mov_b32_e32 v1, s1777; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2778; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc779; CI-NEXT: flat_store_dword v[0:1], v3780; CI-NEXT: s_endpgm781;782; VI-LABEL: global_atomic_dec_ret_i32_offset_addr64:783; VI: ; %bb.0:784; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0785; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0786; VI-NEXT: s_add_i32 s12, s12, s17787; VI-NEXT: s_mov_b32 flat_scratch_lo, s13788; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8789; VI-NEXT: s_waitcnt lgkmcnt(0)790; VI-NEXT: v_mov_b32_e32 v0, s2791; VI-NEXT: v_mov_b32_e32 v1, s3792; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2793; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc794; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v0795; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc796; VI-NEXT: v_mov_b32_e32 v3, 42797; VI-NEXT: flat_atomic_dec v3, v[0:1], v3 glc798; VI-NEXT: s_waitcnt vmcnt(0)799; VI-NEXT: buffer_wbinvl1_vol800; VI-NEXT: v_mov_b32_e32 v0, s0801; VI-NEXT: v_mov_b32_e32 v1, s1802; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2803; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc804; VI-NEXT: flat_store_dword v[0:1], v3805; VI-NEXT: s_endpgm806;807; GFX9-LABEL: global_atomic_dec_ret_i32_offset_addr64:808; GFX9: ; %bb.0:809; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0810; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0811; GFX9-NEXT: v_mov_b32_e32 v1, 42812; GFX9-NEXT: s_waitcnt lgkmcnt(0)813; GFX9-NEXT: global_atomic_dec v1, v0, v1, s[2:3] offset:20 glc814; GFX9-NEXT: s_waitcnt vmcnt(0)815; GFX9-NEXT: buffer_wbinvl1_vol816; GFX9-NEXT: global_store_dword v0, v1, s[0:1]817; GFX9-NEXT: s_endpgm818;819; GFX10-LABEL: global_atomic_dec_ret_i32_offset_addr64:820; GFX10: ; %bb.0:821; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0822; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0823; GFX10-NEXT: v_mov_b32_e32 v1, 42824; GFX10-NEXT: s_waitcnt lgkmcnt(0)825; GFX10-NEXT: global_atomic_dec v1, v0, v1, s[2:3] offset:20 glc826; GFX10-NEXT: s_waitcnt vmcnt(0)827; GFX10-NEXT: buffer_gl1_inv828; GFX10-NEXT: buffer_gl0_inv829; GFX10-NEXT: global_store_dword v0, v1, s[0:1]830; GFX10-NEXT: s_endpgm831;832; GFX11-LABEL: global_atomic_dec_ret_i32_offset_addr64:833; GFX11: ; %bb.0:834; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0835; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_and_b32 v0, 0x3ff, v0836; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)837; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0838; GFX11-NEXT: s_waitcnt lgkmcnt(0)839; GFX11-NEXT: global_atomic_dec_u32 v1, v0, v1, s[2:3] offset:20 glc840; GFX11-NEXT: s_waitcnt vmcnt(0)841; GFX11-NEXT: buffer_gl1_inv842; GFX11-NEXT: buffer_gl0_inv843; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]844; GFX11-NEXT: s_endpgm845 %id = call i32 @llvm.amdgcn.workitem.id.x()846 %gep.tid = getelementptr i32, ptr addrspace(1) %ptr, i32 %id847 %out.gep = getelementptr i32, ptr addrspace(1) %out, i32 %id848 %gep = getelementptr i32, ptr addrspace(1) %gep.tid, i32 5849 %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1850 store i32 %result, ptr addrspace(1) %out.gep, align 4851 ret void852}853 854define amdgpu_kernel void @global_atomic_dec_noret_i32_offset_addr64(ptr addrspace(1) %ptr) #1 {855; CI-LABEL: global_atomic_dec_noret_i32_offset_addr64:856; CI: ; %bb.0:857; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0858; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v0859; CI-NEXT: s_add_i32 s12, s12, s17860; CI-NEXT: s_mov_b32 flat_scratch_lo, s13861; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8862; CI-NEXT: s_waitcnt lgkmcnt(0)863; CI-NEXT: v_mov_b32_e32 v0, s0864; CI-NEXT: v_mov_b32_e32 v1, s1865; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2866; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc867; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v0868; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc869; CI-NEXT: v_mov_b32_e32 v2, 42870; CI-NEXT: flat_atomic_dec v[0:1], v2871; CI-NEXT: s_waitcnt vmcnt(0)872; CI-NEXT: buffer_wbinvl1_vol873; CI-NEXT: s_endpgm874;875; VI-LABEL: global_atomic_dec_noret_i32_offset_addr64:876; VI: ; %bb.0:877; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0878; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0879; VI-NEXT: s_add_i32 s12, s12, s17880; VI-NEXT: s_mov_b32 flat_scratch_lo, s13881; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8882; VI-NEXT: s_waitcnt lgkmcnt(0)883; VI-NEXT: v_mov_b32_e32 v0, s0884; VI-NEXT: v_mov_b32_e32 v1, s1885; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2886; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc887; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v0888; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc889; VI-NEXT: v_mov_b32_e32 v2, 42890; VI-NEXT: flat_atomic_dec v[0:1], v2891; VI-NEXT: s_waitcnt vmcnt(0)892; VI-NEXT: buffer_wbinvl1_vol893; VI-NEXT: s_endpgm894;895; GFX9-LABEL: global_atomic_dec_noret_i32_offset_addr64:896; GFX9: ; %bb.0:897; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0898; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0899; GFX9-NEXT: v_mov_b32_e32 v1, 42900; GFX9-NEXT: s_waitcnt lgkmcnt(0)901; GFX9-NEXT: global_atomic_dec v0, v1, s[0:1] offset:20902; GFX9-NEXT: s_waitcnt vmcnt(0)903; GFX9-NEXT: buffer_wbinvl1_vol904; GFX9-NEXT: s_endpgm905;906; GFX10-LABEL: global_atomic_dec_noret_i32_offset_addr64:907; GFX10: ; %bb.0:908; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0909; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0910; GFX10-NEXT: v_mov_b32_e32 v1, 42911; GFX10-NEXT: s_waitcnt lgkmcnt(0)912; GFX10-NEXT: global_atomic_dec v0, v1, s[0:1] offset:20913; GFX10-NEXT: s_waitcnt_vscnt null, 0x0914; GFX10-NEXT: buffer_gl1_inv915; GFX10-NEXT: buffer_gl0_inv916; GFX10-NEXT: s_endpgm917;918; GFX11-LABEL: global_atomic_dec_noret_i32_offset_addr64:919; GFX11: ; %bb.0:920; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0921; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_and_b32 v0, 0x3ff, v0922; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)923; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0924; GFX11-NEXT: s_waitcnt lgkmcnt(0)925; GFX11-NEXT: global_atomic_dec_u32 v0, v1, s[0:1] offset:20926; GFX11-NEXT: s_waitcnt_vscnt null, 0x0927; GFX11-NEXT: buffer_gl1_inv928; GFX11-NEXT: buffer_gl0_inv929; GFX11-NEXT: s_endpgm930 %id = call i32 @llvm.amdgcn.workitem.id.x()931 %gep.tid = getelementptr i32, ptr addrspace(1) %ptr, i32 %id932 %gep = getelementptr i32, ptr addrspace(1) %gep.tid, i32 5933 %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1934 ret void935}936 937define amdgpu_kernel void @flat_atomic_dec_ret_i32(ptr %out, ptr %ptr) #1 {938; CI-LABEL: flat_atomic_dec_ret_i32:939; CI: ; %bb.0:940; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0941; CI-NEXT: s_add_i32 s12, s12, s17942; CI-NEXT: s_mov_b32 flat_scratch_lo, s13943; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8944; CI-NEXT: v_mov_b32_e32 v2, 42945; CI-NEXT: s_waitcnt lgkmcnt(0)946; CI-NEXT: v_mov_b32_e32 v0, s2947; CI-NEXT: v_mov_b32_e32 v1, s3948; CI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc949; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)950; CI-NEXT: buffer_wbinvl1_vol951; CI-NEXT: v_mov_b32_e32 v0, s0952; CI-NEXT: v_mov_b32_e32 v1, s1953; CI-NEXT: flat_store_dword v[0:1], v2954; CI-NEXT: s_endpgm955;956; VI-LABEL: flat_atomic_dec_ret_i32:957; VI: ; %bb.0:958; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0959; VI-NEXT: s_add_i32 s12, s12, s17960; VI-NEXT: s_mov_b32 flat_scratch_lo, s13961; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8962; VI-NEXT: v_mov_b32_e32 v2, 42963; VI-NEXT: s_waitcnt lgkmcnt(0)964; VI-NEXT: v_mov_b32_e32 v0, s2965; VI-NEXT: v_mov_b32_e32 v1, s3966; VI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc967; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)968; VI-NEXT: buffer_wbinvl1_vol969; VI-NEXT: v_mov_b32_e32 v0, s0970; VI-NEXT: v_mov_b32_e32 v1, s1971; VI-NEXT: flat_store_dword v[0:1], v2972; VI-NEXT: s_endpgm973;974; GFX9-LABEL: flat_atomic_dec_ret_i32:975; GFX9: ; %bb.0:976; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0977; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s17978; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 0979; GFX9-NEXT: v_mov_b32_e32 v2, 42980; GFX9-NEXT: s_waitcnt lgkmcnt(0)981; GFX9-NEXT: v_mov_b32_e32 v0, s2982; GFX9-NEXT: v_mov_b32_e32 v1, s3983; GFX9-NEXT: flat_atomic_dec v2, v[0:1], v2 glc984; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)985; GFX9-NEXT: buffer_wbinvl1_vol986; GFX9-NEXT: v_mov_b32_e32 v0, s0987; GFX9-NEXT: v_mov_b32_e32 v1, s1988; GFX9-NEXT: flat_store_dword v[0:1], v2989; GFX9-NEXT: s_endpgm990;991; GFX10-LABEL: flat_atomic_dec_ret_i32:992; GFX10: ; %bb.0:993; GFX10-NEXT: s_add_u32 s12, s12, s17994; GFX10-NEXT: s_addc_u32 s13, s13, 0995; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12996; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13997; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0998; GFX10-NEXT: v_mov_b32_e32 v2, 42999; GFX10-NEXT: s_waitcnt lgkmcnt(0)1000; GFX10-NEXT: v_mov_b32_e32 v0, s21001; GFX10-NEXT: v_mov_b32_e32 v1, s31002; GFX10-NEXT: flat_atomic_dec v2, v[0:1], v2 glc1003; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1004; GFX10-NEXT: buffer_gl1_inv1005; GFX10-NEXT: buffer_gl0_inv1006; GFX10-NEXT: v_mov_b32_e32 v0, s01007; GFX10-NEXT: v_mov_b32_e32 v1, s11008; GFX10-NEXT: flat_store_dword v[0:1], v21009; GFX10-NEXT: s_endpgm1010;1011; GFX11-LABEL: flat_atomic_dec_ret_i32:1012; GFX11: ; %bb.0:1013; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x01014; GFX11-NEXT: v_mov_b32_e32 v2, 421015; GFX11-NEXT: s_waitcnt lgkmcnt(0)1016; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s31017; GFX11-NEXT: flat_atomic_dec_u32 v2, v[0:1], v2 glc1018; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1019; GFX11-NEXT: buffer_gl1_inv1020; GFX11-NEXT: buffer_gl0_inv1021; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11022; GFX11-NEXT: flat_store_b32 v[0:1], v21023; GFX11-NEXT: s_endpgm1024 %result = atomicrmw udec_wrap ptr %ptr, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !11025 store i32 %result, ptr %out, align 41026 ret void1027}1028 1029define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset(ptr %out, ptr %ptr) #1 {1030; CI-LABEL: flat_atomic_dec_ret_i32_offset:1031; CI: ; %bb.0:1032; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01033; CI-NEXT: s_add_i32 s12, s12, s171034; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81035; CI-NEXT: s_mov_b32 flat_scratch_lo, s131036; CI-NEXT: v_mov_b32_e32 v2, 421037; CI-NEXT: s_waitcnt lgkmcnt(0)1038; CI-NEXT: s_add_u32 s2, s2, 161039; CI-NEXT: s_addc_u32 s3, s3, 01040; CI-NEXT: v_mov_b32_e32 v0, s21041; CI-NEXT: v_mov_b32_e32 v1, s31042; CI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc1043; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1044; CI-NEXT: buffer_wbinvl1_vol1045; CI-NEXT: v_mov_b32_e32 v0, s01046; CI-NEXT: v_mov_b32_e32 v1, s11047; CI-NEXT: flat_store_dword v[0:1], v21048; CI-NEXT: s_endpgm1049;1050; VI-LABEL: flat_atomic_dec_ret_i32_offset:1051; VI: ; %bb.0:1052; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01053; VI-NEXT: s_add_i32 s12, s12, s171054; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81055; VI-NEXT: s_mov_b32 flat_scratch_lo, s131056; VI-NEXT: v_mov_b32_e32 v2, 421057; VI-NEXT: s_waitcnt lgkmcnt(0)1058; VI-NEXT: s_add_u32 s2, s2, 161059; VI-NEXT: s_addc_u32 s3, s3, 01060; VI-NEXT: v_mov_b32_e32 v0, s21061; VI-NEXT: v_mov_b32_e32 v1, s31062; VI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc1063; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1064; VI-NEXT: buffer_wbinvl1_vol1065; VI-NEXT: v_mov_b32_e32 v0, s01066; VI-NEXT: v_mov_b32_e32 v1, s11067; VI-NEXT: flat_store_dword v[0:1], v21068; VI-NEXT: s_endpgm1069;1070; GFX9-LABEL: flat_atomic_dec_ret_i32_offset:1071; GFX9: ; %bb.0:1072; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01073; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s171074; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 01075; GFX9-NEXT: v_mov_b32_e32 v2, 421076; GFX9-NEXT: s_waitcnt lgkmcnt(0)1077; GFX9-NEXT: v_mov_b32_e32 v0, s21078; GFX9-NEXT: v_mov_b32_e32 v1, s31079; GFX9-NEXT: flat_atomic_dec v2, v[0:1], v2 offset:16 glc1080; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1081; GFX9-NEXT: buffer_wbinvl1_vol1082; GFX9-NEXT: v_mov_b32_e32 v0, s01083; GFX9-NEXT: v_mov_b32_e32 v1, s11084; GFX9-NEXT: flat_store_dword v[0:1], v21085; GFX9-NEXT: s_endpgm1086;1087; GFX10-LABEL: flat_atomic_dec_ret_i32_offset:1088; GFX10: ; %bb.0:1089; GFX10-NEXT: s_add_u32 s12, s12, s171090; GFX10-NEXT: s_addc_u32 s13, s13, 01091; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s121092; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s131093; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01094; GFX10-NEXT: v_mov_b32_e32 v2, 421095; GFX10-NEXT: s_waitcnt lgkmcnt(0)1096; GFX10-NEXT: s_add_u32 s2, s2, 161097; GFX10-NEXT: s_addc_u32 s3, s3, 01098; GFX10-NEXT: v_mov_b32_e32 v0, s21099; GFX10-NEXT: v_mov_b32_e32 v1, s31100; GFX10-NEXT: flat_atomic_dec v2, v[0:1], v2 glc1101; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1102; GFX10-NEXT: buffer_gl1_inv1103; GFX10-NEXT: buffer_gl0_inv1104; GFX10-NEXT: v_mov_b32_e32 v0, s01105; GFX10-NEXT: v_mov_b32_e32 v1, s11106; GFX10-NEXT: flat_store_dword v[0:1], v21107; GFX10-NEXT: s_endpgm1108;1109; GFX11-LABEL: flat_atomic_dec_ret_i32_offset:1110; GFX11: ; %bb.0:1111; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x01112; GFX11-NEXT: v_mov_b32_e32 v2, 421113; GFX11-NEXT: s_waitcnt lgkmcnt(0)1114; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s31115; GFX11-NEXT: flat_atomic_dec_u32 v2, v[0:1], v2 offset:16 glc1116; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1117; GFX11-NEXT: buffer_gl1_inv1118; GFX11-NEXT: buffer_gl0_inv1119; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11120; GFX11-NEXT: flat_store_b32 v[0:1], v21121; GFX11-NEXT: s_endpgm1122 %gep = getelementptr inbounds i32, ptr %ptr, i32 41123 %result = atomicrmw udec_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !11124 store i32 %result, ptr %out, align 41125 ret void1126}1127 1128define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_system(ptr %out, ptr %ptr) #1 {1129; CI-LABEL: flat_atomic_dec_ret_i32_offset_system:1130; CI: ; %bb.0:1131; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01132; CI-NEXT: s_add_i32 s12, s12, s171133; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81134; CI-NEXT: s_mov_b32 flat_scratch_lo, s131135; CI-NEXT: v_mov_b32_e32 v2, 421136; CI-NEXT: s_waitcnt lgkmcnt(0)1137; CI-NEXT: s_add_u32 s2, s2, 161138; CI-NEXT: s_addc_u32 s3, s3, 01139; CI-NEXT: v_mov_b32_e32 v0, s21140; CI-NEXT: v_mov_b32_e32 v1, s31141; CI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc1142; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1143; CI-NEXT: buffer_wbinvl1_vol1144; CI-NEXT: v_mov_b32_e32 v0, s01145; CI-NEXT: v_mov_b32_e32 v1, s11146; CI-NEXT: flat_store_dword v[0:1], v21147; CI-NEXT: s_endpgm1148;1149; VI-LABEL: flat_atomic_dec_ret_i32_offset_system:1150; VI: ; %bb.0:1151; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01152; VI-NEXT: s_add_i32 s12, s12, s171153; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81154; VI-NEXT: s_mov_b32 flat_scratch_lo, s131155; VI-NEXT: v_mov_b32_e32 v2, 421156; VI-NEXT: s_waitcnt lgkmcnt(0)1157; VI-NEXT: s_add_u32 s2, s2, 161158; VI-NEXT: s_addc_u32 s3, s3, 01159; VI-NEXT: v_mov_b32_e32 v0, s21160; VI-NEXT: v_mov_b32_e32 v1, s31161; VI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc1162; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1163; VI-NEXT: buffer_wbinvl1_vol1164; VI-NEXT: v_mov_b32_e32 v0, s01165; VI-NEXT: v_mov_b32_e32 v1, s11166; VI-NEXT: flat_store_dword v[0:1], v21167; VI-NEXT: s_endpgm1168;1169; GFX9-LABEL: flat_atomic_dec_ret_i32_offset_system:1170; GFX9: ; %bb.0:1171; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01172; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s171173; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 01174; GFX9-NEXT: v_mov_b32_e32 v2, 421175; GFX9-NEXT: s_waitcnt lgkmcnt(0)1176; GFX9-NEXT: v_mov_b32_e32 v0, s21177; GFX9-NEXT: v_mov_b32_e32 v1, s31178; GFX9-NEXT: flat_atomic_dec v2, v[0:1], v2 offset:16 glc1179; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1180; GFX9-NEXT: buffer_wbinvl1_vol1181; GFX9-NEXT: v_mov_b32_e32 v0, s01182; GFX9-NEXT: v_mov_b32_e32 v1, s11183; GFX9-NEXT: flat_store_dword v[0:1], v21184; GFX9-NEXT: s_endpgm1185;1186; GFX10-LABEL: flat_atomic_dec_ret_i32_offset_system:1187; GFX10: ; %bb.0:1188; GFX10-NEXT: s_add_u32 s12, s12, s171189; GFX10-NEXT: s_addc_u32 s13, s13, 01190; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s121191; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s131192; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01193; GFX10-NEXT: v_mov_b32_e32 v2, 421194; GFX10-NEXT: s_waitcnt lgkmcnt(0)1195; GFX10-NEXT: s_add_u32 s2, s2, 161196; GFX10-NEXT: s_addc_u32 s3, s3, 01197; GFX10-NEXT: v_mov_b32_e32 v0, s21198; GFX10-NEXT: v_mov_b32_e32 v1, s31199; GFX10-NEXT: flat_atomic_dec v2, v[0:1], v2 glc1200; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1201; GFX10-NEXT: buffer_gl1_inv1202; GFX10-NEXT: buffer_gl0_inv1203; GFX10-NEXT: v_mov_b32_e32 v0, s01204; GFX10-NEXT: v_mov_b32_e32 v1, s11205; GFX10-NEXT: flat_store_dword v[0:1], v21206; GFX10-NEXT: s_endpgm1207;1208; GFX11-LABEL: flat_atomic_dec_ret_i32_offset_system:1209; GFX11: ; %bb.0:1210; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x01211; GFX11-NEXT: v_mov_b32_e32 v2, 421212; GFX11-NEXT: s_waitcnt lgkmcnt(0)1213; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s31214; GFX11-NEXT: flat_atomic_dec_u32 v2, v[0:1], v2 offset:16 glc1215; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1216; GFX11-NEXT: buffer_gl1_inv1217; GFX11-NEXT: buffer_gl0_inv1218; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11219; GFX11-NEXT: flat_store_b32 v[0:1], v21220; GFX11-NEXT: s_endpgm1221 %gep = getelementptr inbounds i32, ptr %ptr, i32 41222 %result = atomicrmw udec_wrap ptr %gep, i32 42 seq_cst, align 4, !amdgpu.no.remote.memory !11223 store i32 %result, ptr %out, align 41224 ret void1225}1226 1227define amdgpu_kernel void @flat_atomic_dec_noret_i32(ptr %ptr) #1 {1228; CI-LABEL: flat_atomic_dec_noret_i32:1229; CI: ; %bb.0:1230; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01231; CI-NEXT: s_add_i32 s12, s12, s171232; CI-NEXT: s_mov_b32 flat_scratch_lo, s131233; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81234; CI-NEXT: v_mov_b32_e32 v2, 421235; CI-NEXT: s_waitcnt lgkmcnt(0)1236; CI-NEXT: v_mov_b32_e32 v0, s01237; CI-NEXT: v_mov_b32_e32 v1, s11238; CI-NEXT: flat_atomic_dec v[0:1], v21239; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1240; CI-NEXT: buffer_wbinvl1_vol1241; CI-NEXT: s_endpgm1242;1243; VI-LABEL: flat_atomic_dec_noret_i32:1244; VI: ; %bb.0:1245; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01246; VI-NEXT: s_add_i32 s12, s12, s171247; VI-NEXT: s_mov_b32 flat_scratch_lo, s131248; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81249; VI-NEXT: v_mov_b32_e32 v2, 421250; VI-NEXT: s_waitcnt lgkmcnt(0)1251; VI-NEXT: v_mov_b32_e32 v0, s01252; VI-NEXT: v_mov_b32_e32 v1, s11253; VI-NEXT: flat_atomic_dec v[0:1], v21254; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1255; VI-NEXT: buffer_wbinvl1_vol1256; VI-NEXT: s_endpgm1257;1258; GFX9-LABEL: flat_atomic_dec_noret_i32:1259; GFX9: ; %bb.0:1260; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01261; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s171262; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 01263; GFX9-NEXT: v_mov_b32_e32 v2, 421264; GFX9-NEXT: s_waitcnt lgkmcnt(0)1265; GFX9-NEXT: v_mov_b32_e32 v0, s01266; GFX9-NEXT: v_mov_b32_e32 v1, s11267; GFX9-NEXT: flat_atomic_dec v[0:1], v21268; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1269; GFX9-NEXT: buffer_wbinvl1_vol1270; GFX9-NEXT: s_endpgm1271;1272; GFX10-LABEL: flat_atomic_dec_noret_i32:1273; GFX10: ; %bb.0:1274; GFX10-NEXT: s_add_u32 s12, s12, s171275; GFX10-NEXT: s_addc_u32 s13, s13, 01276; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s121277; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s131278; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01279; GFX10-NEXT: v_mov_b32_e32 v2, 421280; GFX10-NEXT: s_waitcnt lgkmcnt(0)1281; GFX10-NEXT: v_mov_b32_e32 v0, s01282; GFX10-NEXT: v_mov_b32_e32 v1, s11283; GFX10-NEXT: flat_atomic_dec v[0:1], v21284; GFX10-NEXT: s_waitcnt lgkmcnt(0)1285; GFX10-NEXT: s_waitcnt_vscnt null, 0x01286; GFX10-NEXT: buffer_gl1_inv1287; GFX10-NEXT: buffer_gl0_inv1288; GFX10-NEXT: s_endpgm1289;1290; GFX11-LABEL: flat_atomic_dec_noret_i32:1291; GFX11: ; %bb.0:1292; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x01293; GFX11-NEXT: v_mov_b32_e32 v2, 421294; GFX11-NEXT: s_waitcnt lgkmcnt(0)1295; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11296; GFX11-NEXT: flat_atomic_dec_u32 v[0:1], v21297; GFX11-NEXT: s_waitcnt lgkmcnt(0)1298; GFX11-NEXT: s_waitcnt_vscnt null, 0x01299; GFX11-NEXT: buffer_gl1_inv1300; GFX11-NEXT: buffer_gl0_inv1301; GFX11-NEXT: s_endpgm1302 %result = atomicrmw udec_wrap ptr %ptr, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !11303 ret void1304}1305 1306define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset(ptr %ptr) #1 {1307; CI-LABEL: flat_atomic_dec_noret_i32_offset:1308; CI: ; %bb.0:1309; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01310; CI-NEXT: s_add_i32 s12, s12, s171311; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81312; CI-NEXT: s_mov_b32 flat_scratch_lo, s131313; CI-NEXT: v_mov_b32_e32 v2, 421314; CI-NEXT: s_waitcnt lgkmcnt(0)1315; CI-NEXT: s_add_u32 s0, s0, 161316; CI-NEXT: s_addc_u32 s1, s1, 01317; CI-NEXT: v_mov_b32_e32 v0, s01318; CI-NEXT: v_mov_b32_e32 v1, s11319; CI-NEXT: flat_atomic_dec v[0:1], v21320; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1321; CI-NEXT: buffer_wbinvl1_vol1322; CI-NEXT: s_endpgm1323;1324; VI-LABEL: flat_atomic_dec_noret_i32_offset:1325; VI: ; %bb.0:1326; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01327; VI-NEXT: s_add_i32 s12, s12, s171328; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81329; VI-NEXT: s_mov_b32 flat_scratch_lo, s131330; VI-NEXT: v_mov_b32_e32 v2, 421331; VI-NEXT: s_waitcnt lgkmcnt(0)1332; VI-NEXT: s_add_u32 s0, s0, 161333; VI-NEXT: s_addc_u32 s1, s1, 01334; VI-NEXT: v_mov_b32_e32 v0, s01335; VI-NEXT: v_mov_b32_e32 v1, s11336; VI-NEXT: flat_atomic_dec v[0:1], v21337; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1338; VI-NEXT: buffer_wbinvl1_vol1339; VI-NEXT: s_endpgm1340;1341; GFX9-LABEL: flat_atomic_dec_noret_i32_offset:1342; GFX9: ; %bb.0:1343; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01344; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s171345; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 01346; GFX9-NEXT: v_mov_b32_e32 v2, 421347; GFX9-NEXT: s_waitcnt lgkmcnt(0)1348; GFX9-NEXT: v_mov_b32_e32 v0, s01349; GFX9-NEXT: v_mov_b32_e32 v1, s11350; GFX9-NEXT: flat_atomic_dec v[0:1], v2 offset:161351; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1352; GFX9-NEXT: buffer_wbinvl1_vol1353; GFX9-NEXT: s_endpgm1354;1355; GFX10-LABEL: flat_atomic_dec_noret_i32_offset:1356; GFX10: ; %bb.0:1357; GFX10-NEXT: s_add_u32 s12, s12, s171358; GFX10-NEXT: s_addc_u32 s13, s13, 01359; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s121360; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s131361; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01362; GFX10-NEXT: v_mov_b32_e32 v2, 421363; GFX10-NEXT: s_waitcnt lgkmcnt(0)1364; GFX10-NEXT: s_add_u32 s0, s0, 161365; GFX10-NEXT: s_addc_u32 s1, s1, 01366; GFX10-NEXT: v_mov_b32_e32 v0, s01367; GFX10-NEXT: v_mov_b32_e32 v1, s11368; GFX10-NEXT: flat_atomic_dec v[0:1], v21369; GFX10-NEXT: s_waitcnt lgkmcnt(0)1370; GFX10-NEXT: s_waitcnt_vscnt null, 0x01371; GFX10-NEXT: buffer_gl1_inv1372; GFX10-NEXT: buffer_gl0_inv1373; GFX10-NEXT: s_endpgm1374;1375; GFX11-LABEL: flat_atomic_dec_noret_i32_offset:1376; GFX11: ; %bb.0:1377; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x01378; GFX11-NEXT: v_mov_b32_e32 v2, 421379; GFX11-NEXT: s_waitcnt lgkmcnt(0)1380; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11381; GFX11-NEXT: flat_atomic_dec_u32 v[0:1], v2 offset:161382; GFX11-NEXT: s_waitcnt lgkmcnt(0)1383; GFX11-NEXT: s_waitcnt_vscnt null, 0x01384; GFX11-NEXT: buffer_gl1_inv1385; GFX11-NEXT: buffer_gl0_inv1386; GFX11-NEXT: s_endpgm1387 %gep = getelementptr inbounds i32, ptr %ptr, i32 41388 %result = atomicrmw udec_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !11389 ret void1390}1391 1392define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_system(ptr %ptr) #1 {1393; CI-LABEL: flat_atomic_dec_noret_i32_offset_system:1394; CI: ; %bb.0:1395; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01396; CI-NEXT: s_add_i32 s12, s12, s171397; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81398; CI-NEXT: s_mov_b32 flat_scratch_lo, s131399; CI-NEXT: v_mov_b32_e32 v2, 421400; CI-NEXT: s_waitcnt lgkmcnt(0)1401; CI-NEXT: s_add_u32 s0, s0, 161402; CI-NEXT: s_addc_u32 s1, s1, 01403; CI-NEXT: v_mov_b32_e32 v0, s01404; CI-NEXT: v_mov_b32_e32 v1, s11405; CI-NEXT: flat_atomic_dec v[0:1], v21406; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1407; CI-NEXT: buffer_wbinvl1_vol1408; CI-NEXT: s_endpgm1409;1410; VI-LABEL: flat_atomic_dec_noret_i32_offset_system:1411; VI: ; %bb.0:1412; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01413; VI-NEXT: s_add_i32 s12, s12, s171414; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81415; VI-NEXT: s_mov_b32 flat_scratch_lo, s131416; VI-NEXT: v_mov_b32_e32 v2, 421417; VI-NEXT: s_waitcnt lgkmcnt(0)1418; VI-NEXT: s_add_u32 s0, s0, 161419; VI-NEXT: s_addc_u32 s1, s1, 01420; VI-NEXT: v_mov_b32_e32 v0, s01421; VI-NEXT: v_mov_b32_e32 v1, s11422; VI-NEXT: flat_atomic_dec v[0:1], v21423; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1424; VI-NEXT: buffer_wbinvl1_vol1425; VI-NEXT: s_endpgm1426;1427; GFX9-LABEL: flat_atomic_dec_noret_i32_offset_system:1428; GFX9: ; %bb.0:1429; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01430; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s171431; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 01432; GFX9-NEXT: v_mov_b32_e32 v2, 421433; GFX9-NEXT: s_waitcnt lgkmcnt(0)1434; GFX9-NEXT: v_mov_b32_e32 v0, s01435; GFX9-NEXT: v_mov_b32_e32 v1, s11436; GFX9-NEXT: flat_atomic_dec v[0:1], v2 offset:161437; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1438; GFX9-NEXT: buffer_wbinvl1_vol1439; GFX9-NEXT: s_endpgm1440;1441; GFX10-LABEL: flat_atomic_dec_noret_i32_offset_system:1442; GFX10: ; %bb.0:1443; GFX10-NEXT: s_add_u32 s12, s12, s171444; GFX10-NEXT: s_addc_u32 s13, s13, 01445; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s121446; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s131447; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01448; GFX10-NEXT: v_mov_b32_e32 v2, 421449; GFX10-NEXT: s_waitcnt lgkmcnt(0)1450; GFX10-NEXT: s_add_u32 s0, s0, 161451; GFX10-NEXT: s_addc_u32 s1, s1, 01452; GFX10-NEXT: v_mov_b32_e32 v0, s01453; GFX10-NEXT: v_mov_b32_e32 v1, s11454; GFX10-NEXT: flat_atomic_dec v[0:1], v21455; GFX10-NEXT: s_waitcnt lgkmcnt(0)1456; GFX10-NEXT: s_waitcnt_vscnt null, 0x01457; GFX10-NEXT: buffer_gl1_inv1458; GFX10-NEXT: buffer_gl0_inv1459; GFX10-NEXT: s_endpgm1460;1461; GFX11-LABEL: flat_atomic_dec_noret_i32_offset_system:1462; GFX11: ; %bb.0:1463; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x01464; GFX11-NEXT: v_mov_b32_e32 v2, 421465; GFX11-NEXT: s_waitcnt lgkmcnt(0)1466; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11467; GFX11-NEXT: flat_atomic_dec_u32 v[0:1], v2 offset:161468; GFX11-NEXT: s_waitcnt lgkmcnt(0)1469; GFX11-NEXT: s_waitcnt_vscnt null, 0x01470; GFX11-NEXT: buffer_gl1_inv1471; GFX11-NEXT: buffer_gl0_inv1472; GFX11-NEXT: s_endpgm1473 %gep = getelementptr inbounds i32, ptr %ptr, i32 41474 %result = atomicrmw udec_wrap ptr %gep, i32 42 seq_cst, align 4, !amdgpu.no.remote.memory !11475 ret void1476}1477 1478define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_addr64(ptr %out, ptr %ptr) #1 {1479; CI-LABEL: flat_atomic_dec_ret_i32_offset_addr64:1480; CI: ; %bb.0:1481; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01482; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v01483; CI-NEXT: s_add_i32 s12, s12, s171484; CI-NEXT: s_mov_b32 flat_scratch_lo, s131485; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81486; CI-NEXT: s_waitcnt lgkmcnt(0)1487; CI-NEXT: v_mov_b32_e32 v0, s21488; CI-NEXT: v_mov_b32_e32 v1, s31489; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v21490; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1491; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v01492; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1493; CI-NEXT: v_mov_b32_e32 v3, 421494; CI-NEXT: flat_atomic_dec v3, v[0:1], v3 glc1495; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1496; CI-NEXT: buffer_wbinvl1_vol1497; CI-NEXT: v_mov_b32_e32 v0, s01498; CI-NEXT: v_mov_b32_e32 v1, s11499; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v21500; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1501; CI-NEXT: flat_store_dword v[0:1], v31502; CI-NEXT: s_endpgm1503;1504; VI-LABEL: flat_atomic_dec_ret_i32_offset_addr64:1505; VI: ; %bb.0:1506; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01507; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v01508; VI-NEXT: s_add_i32 s12, s12, s171509; VI-NEXT: s_mov_b32 flat_scratch_lo, s131510; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81511; VI-NEXT: s_waitcnt lgkmcnt(0)1512; VI-NEXT: v_mov_b32_e32 v0, s21513; VI-NEXT: v_mov_b32_e32 v1, s31514; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v21515; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1516; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v01517; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1518; VI-NEXT: v_mov_b32_e32 v3, 421519; VI-NEXT: flat_atomic_dec v3, v[0:1], v3 glc1520; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1521; VI-NEXT: buffer_wbinvl1_vol1522; VI-NEXT: v_mov_b32_e32 v0, s01523; VI-NEXT: v_mov_b32_e32 v1, s11524; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v21525; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1526; VI-NEXT: flat_store_dword v[0:1], v31527; VI-NEXT: s_endpgm1528;1529; GFX9-LABEL: flat_atomic_dec_ret_i32_offset_addr64:1530; GFX9: ; %bb.0:1531; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01532; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v01533; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s171534; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 01535; GFX9-NEXT: v_mov_b32_e32 v3, 421536; GFX9-NEXT: s_waitcnt lgkmcnt(0)1537; GFX9-NEXT: v_mov_b32_e32 v0, s21538; GFX9-NEXT: v_mov_b32_e32 v1, s31539; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v21540; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc1541; GFX9-NEXT: flat_atomic_dec v3, v[0:1], v3 offset:20 glc1542; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1543; GFX9-NEXT: buffer_wbinvl1_vol1544; GFX9-NEXT: v_mov_b32_e32 v0, s01545; GFX9-NEXT: v_mov_b32_e32 v1, s11546; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v21547; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc1548; GFX9-NEXT: flat_store_dword v[0:1], v31549; GFX9-NEXT: s_endpgm1550;1551; GFX10-LABEL: flat_atomic_dec_ret_i32_offset_addr64:1552; GFX10: ; %bb.0:1553; GFX10-NEXT: s_add_u32 s12, s12, s171554; GFX10-NEXT: s_addc_u32 s13, s13, 01555; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s121556; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s131557; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01558; GFX10-NEXT: v_lshlrev_b32_e32 v2, 2, v01559; GFX10-NEXT: v_mov_b32_e32 v3, 421560; GFX10-NEXT: s_waitcnt lgkmcnt(0)1561; GFX10-NEXT: v_mov_b32_e32 v0, s21562; GFX10-NEXT: v_mov_b32_e32 v1, s31563; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v21564; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo1565; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, 201566; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo1567; GFX10-NEXT: flat_atomic_dec v3, v[0:1], v3 glc1568; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1569; GFX10-NEXT: buffer_gl1_inv1570; GFX10-NEXT: buffer_gl0_inv1571; GFX10-NEXT: v_mov_b32_e32 v0, s01572; GFX10-NEXT: v_mov_b32_e32 v1, s11573; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v21574; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo1575; GFX10-NEXT: flat_store_dword v[0:1], v31576; GFX10-NEXT: s_endpgm1577;1578; GFX11-LABEL: flat_atomic_dec_ret_i32_offset_addr64:1579; GFX11: ; %bb.0:1580; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x01581; GFX11-NEXT: v_dual_mov_b32 v3, 42 :: v_dual_and_b32 v0, 0x3ff, v01582; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)1583; GFX11-NEXT: v_lshlrev_b32_e32 v2, 2, v01584; GFX11-NEXT: s_waitcnt lgkmcnt(0)1585; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s31586; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v21587; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)1588; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo1589; GFX11-NEXT: flat_atomic_dec_u32 v3, v[0:1], v3 offset:20 glc1590; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1591; GFX11-NEXT: buffer_gl1_inv1592; GFX11-NEXT: buffer_gl0_inv1593; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11594; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1595; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v21596; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo1597; GFX11-NEXT: flat_store_b32 v[0:1], v31598; GFX11-NEXT: s_endpgm1599 %id = call i32 @llvm.amdgcn.workitem.id.x()1600 %gep.tid = getelementptr i32, ptr %ptr, i32 %id1601 %out.gep = getelementptr i32, ptr %out, i32 %id1602 %gep = getelementptr inbounds i32, ptr %gep.tid, i32 51603 %result = atomicrmw udec_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !11604 store i32 %result, ptr %out.gep, align 41605 ret void1606}1607 1608define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_addr64(ptr %ptr) #1 {1609; CI-LABEL: flat_atomic_dec_noret_i32_offset_addr64:1610; CI: ; %bb.0:1611; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01612; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v01613; CI-NEXT: s_add_i32 s12, s12, s171614; CI-NEXT: s_mov_b32 flat_scratch_lo, s131615; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81616; CI-NEXT: s_waitcnt lgkmcnt(0)1617; CI-NEXT: v_mov_b32_e32 v0, s01618; CI-NEXT: v_mov_b32_e32 v1, s11619; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v21620; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1621; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v01622; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1623; CI-NEXT: v_mov_b32_e32 v2, 421624; CI-NEXT: flat_atomic_dec v[0:1], v21625; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1626; CI-NEXT: buffer_wbinvl1_vol1627; CI-NEXT: s_endpgm1628;1629; VI-LABEL: flat_atomic_dec_noret_i32_offset_addr64:1630; VI: ; %bb.0:1631; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01632; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v01633; VI-NEXT: s_add_i32 s12, s12, s171634; VI-NEXT: s_mov_b32 flat_scratch_lo, s131635; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81636; VI-NEXT: s_waitcnt lgkmcnt(0)1637; VI-NEXT: v_mov_b32_e32 v0, s01638; VI-NEXT: v_mov_b32_e32 v1, s11639; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v21640; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1641; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v01642; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1643; VI-NEXT: v_mov_b32_e32 v2, 421644; VI-NEXT: flat_atomic_dec v[0:1], v21645; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1646; VI-NEXT: buffer_wbinvl1_vol1647; VI-NEXT: s_endpgm1648;1649; GFX9-LABEL: flat_atomic_dec_noret_i32_offset_addr64:1650; GFX9: ; %bb.0:1651; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01652; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v01653; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s171654; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 01655; GFX9-NEXT: s_waitcnt lgkmcnt(0)1656; GFX9-NEXT: v_mov_b32_e32 v0, s01657; GFX9-NEXT: v_mov_b32_e32 v1, s11658; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v21659; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc1660; GFX9-NEXT: v_mov_b32_e32 v2, 421661; GFX9-NEXT: flat_atomic_dec v[0:1], v2 offset:201662; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1663; GFX9-NEXT: buffer_wbinvl1_vol1664; GFX9-NEXT: s_endpgm1665;1666; GFX10-LABEL: flat_atomic_dec_noret_i32_offset_addr64:1667; GFX10: ; %bb.0:1668; GFX10-NEXT: s_add_u32 s12, s12, s171669; GFX10-NEXT: s_addc_u32 s13, s13, 01670; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s121671; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s131672; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01673; GFX10-NEXT: v_lshlrev_b32_e32 v2, 2, v01674; GFX10-NEXT: s_waitcnt lgkmcnt(0)1675; GFX10-NEXT: v_mov_b32_e32 v0, s01676; GFX10-NEXT: v_mov_b32_e32 v1, s11677; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v21678; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo1679; GFX10-NEXT: v_mov_b32_e32 v2, 421680; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, 201681; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo1682; GFX10-NEXT: flat_atomic_dec v[0:1], v21683; GFX10-NEXT: s_waitcnt lgkmcnt(0)1684; GFX10-NEXT: s_waitcnt_vscnt null, 0x01685; GFX10-NEXT: buffer_gl1_inv1686; GFX10-NEXT: buffer_gl0_inv1687; GFX10-NEXT: s_endpgm1688;1689; GFX11-LABEL: flat_atomic_dec_noret_i32_offset_addr64:1690; GFX11: ; %bb.0:1691; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x01692; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v01693; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)1694; GFX11-NEXT: v_lshlrev_b32_e32 v2, 2, v01695; GFX11-NEXT: s_waitcnt lgkmcnt(0)1696; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11697; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v21698; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)1699; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo1700; GFX11-NEXT: v_mov_b32_e32 v2, 421701; GFX11-NEXT: flat_atomic_dec_u32 v[0:1], v2 offset:201702; GFX11-NEXT: s_waitcnt lgkmcnt(0)1703; GFX11-NEXT: s_waitcnt_vscnt null, 0x01704; GFX11-NEXT: buffer_gl1_inv1705; GFX11-NEXT: buffer_gl0_inv1706; GFX11-NEXT: s_endpgm1707 %id = call i32 @llvm.amdgcn.workitem.id.x()1708 %gep.tid = getelementptr i32, ptr %ptr, i32 %id1709 %gep = getelementptr inbounds i32, ptr %gep.tid, i32 51710 %result = atomicrmw udec_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !11711 ret void1712}1713 1714define amdgpu_kernel void @flat_atomic_dec_ret_i64(ptr %out, ptr %ptr) #1 {1715; CI-LABEL: flat_atomic_dec_ret_i64:1716; CI: ; %bb.0:1717; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01718; CI-NEXT: s_add_i32 s12, s12, s171719; CI-NEXT: v_mov_b32_e32 v0, 421720; CI-NEXT: s_mov_b32 flat_scratch_lo, s131721; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81722; CI-NEXT: s_waitcnt lgkmcnt(0)1723; CI-NEXT: v_mov_b32_e32 v2, s21724; CI-NEXT: v_mov_b32_e32 v1, 01725; CI-NEXT: v_mov_b32_e32 v3, s31726; CI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc1727; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1728; CI-NEXT: buffer_wbinvl1_vol1729; CI-NEXT: v_mov_b32_e32 v3, s11730; CI-NEXT: v_mov_b32_e32 v2, s01731; CI-NEXT: s_add_u32 s0, s0, 41732; CI-NEXT: s_addc_u32 s1, s1, 01733; CI-NEXT: v_mov_b32_e32 v5, s11734; CI-NEXT: v_mov_b32_e32 v4, s01735; CI-NEXT: flat_store_dword v[2:3], v01736; CI-NEXT: flat_store_dword v[4:5], v11737; CI-NEXT: s_endpgm1738;1739; VI-LABEL: flat_atomic_dec_ret_i64:1740; VI: ; %bb.0:1741; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01742; VI-NEXT: s_add_i32 s12, s12, s171743; VI-NEXT: v_mov_b32_e32 v0, 421744; VI-NEXT: s_mov_b32 flat_scratch_lo, s131745; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81746; VI-NEXT: s_waitcnt lgkmcnt(0)1747; VI-NEXT: v_mov_b32_e32 v2, s21748; VI-NEXT: v_mov_b32_e32 v1, 01749; VI-NEXT: v_mov_b32_e32 v3, s31750; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc1751; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1752; VI-NEXT: buffer_wbinvl1_vol1753; VI-NEXT: v_mov_b32_e32 v3, s11754; VI-NEXT: v_mov_b32_e32 v2, s01755; VI-NEXT: s_add_u32 s0, s0, 41756; VI-NEXT: s_addc_u32 s1, s1, 01757; VI-NEXT: v_mov_b32_e32 v5, s11758; VI-NEXT: v_mov_b32_e32 v4, s01759; VI-NEXT: flat_store_dword v[2:3], v01760; VI-NEXT: flat_store_dword v[4:5], v11761; VI-NEXT: s_endpgm1762;1763; GFX9-LABEL: flat_atomic_dec_ret_i64:1764; GFX9: ; %bb.0:1765; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01766; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s171767; GFX9-NEXT: v_mov_b32_e32 v0, 421768; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 01769; GFX9-NEXT: v_mov_b32_e32 v1, 01770; GFX9-NEXT: s_waitcnt lgkmcnt(0)1771; GFX9-NEXT: v_mov_b32_e32 v2, s21772; GFX9-NEXT: v_mov_b32_e32 v3, s31773; GFX9-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc1774; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1775; GFX9-NEXT: buffer_wbinvl1_vol1776; GFX9-NEXT: v_mov_b32_e32 v3, s11777; GFX9-NEXT: v_mov_b32_e32 v2, s01778; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1779; GFX9-NEXT: s_endpgm1780;1781; GFX10-LABEL: flat_atomic_dec_ret_i64:1782; GFX10: ; %bb.0:1783; GFX10-NEXT: s_add_u32 s12, s12, s171784; GFX10-NEXT: s_addc_u32 s13, s13, 01785; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s121786; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s131787; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01788; GFX10-NEXT: v_mov_b32_e32 v0, 421789; GFX10-NEXT: v_mov_b32_e32 v1, 01790; GFX10-NEXT: s_waitcnt lgkmcnt(0)1791; GFX10-NEXT: v_mov_b32_e32 v2, s21792; GFX10-NEXT: v_mov_b32_e32 v3, s31793; GFX10-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc1794; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1795; GFX10-NEXT: buffer_gl1_inv1796; GFX10-NEXT: buffer_gl0_inv1797; GFX10-NEXT: v_mov_b32_e32 v3, s11798; GFX10-NEXT: v_mov_b32_e32 v2, s01799; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1800; GFX10-NEXT: s_endpgm1801;1802; GFX11-LABEL: flat_atomic_dec_ret_i64:1803; GFX11: ; %bb.0:1804; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x01805; GFX11-NEXT: v_mov_b32_e32 v0, 421806; GFX11-NEXT: s_waitcnt lgkmcnt(0)1807; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s21808; GFX11-NEXT: v_mov_b32_e32 v3, s31809; GFX11-NEXT: flat_atomic_dec_u64 v[0:1], v[2:3], v[0:1] glc1810; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1811; GFX11-NEXT: buffer_gl1_inv1812; GFX11-NEXT: buffer_gl0_inv1813; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s01814; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]1815; GFX11-NEXT: s_endpgm1816 %result = atomicrmw udec_wrap ptr %ptr, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !11817 store i64 %result, ptr %out, align 41818 ret void1819}1820 1821define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset(ptr %out, ptr %ptr) #1 {1822; CI-LABEL: flat_atomic_dec_ret_i64_offset:1823; CI: ; %bb.0:1824; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01825; CI-NEXT: s_add_i32 s12, s12, s171826; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81827; CI-NEXT: v_mov_b32_e32 v0, 421828; CI-NEXT: s_mov_b32 flat_scratch_lo, s131829; CI-NEXT: s_waitcnt lgkmcnt(0)1830; CI-NEXT: s_add_u32 s2, s2, 321831; CI-NEXT: s_addc_u32 s3, s3, 01832; CI-NEXT: v_mov_b32_e32 v2, s21833; CI-NEXT: v_mov_b32_e32 v1, 01834; CI-NEXT: v_mov_b32_e32 v3, s31835; CI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc1836; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1837; CI-NEXT: buffer_wbinvl1_vol1838; CI-NEXT: v_mov_b32_e32 v3, s11839; CI-NEXT: v_mov_b32_e32 v2, s01840; CI-NEXT: s_add_u32 s0, s0, 41841; CI-NEXT: s_addc_u32 s1, s1, 01842; CI-NEXT: v_mov_b32_e32 v5, s11843; CI-NEXT: v_mov_b32_e32 v4, s01844; CI-NEXT: flat_store_dword v[2:3], v01845; CI-NEXT: flat_store_dword v[4:5], v11846; CI-NEXT: s_endpgm1847;1848; VI-LABEL: flat_atomic_dec_ret_i64_offset:1849; VI: ; %bb.0:1850; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01851; VI-NEXT: s_add_i32 s12, s12, s171852; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81853; VI-NEXT: v_mov_b32_e32 v0, 421854; VI-NEXT: s_mov_b32 flat_scratch_lo, s131855; VI-NEXT: s_waitcnt lgkmcnt(0)1856; VI-NEXT: s_add_u32 s2, s2, 321857; VI-NEXT: s_addc_u32 s3, s3, 01858; VI-NEXT: v_mov_b32_e32 v2, s21859; VI-NEXT: v_mov_b32_e32 v1, 01860; VI-NEXT: v_mov_b32_e32 v3, s31861; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc1862; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1863; VI-NEXT: buffer_wbinvl1_vol1864; VI-NEXT: v_mov_b32_e32 v3, s11865; VI-NEXT: v_mov_b32_e32 v2, s01866; VI-NEXT: s_add_u32 s0, s0, 41867; VI-NEXT: s_addc_u32 s1, s1, 01868; VI-NEXT: v_mov_b32_e32 v5, s11869; VI-NEXT: v_mov_b32_e32 v4, s01870; VI-NEXT: flat_store_dword v[2:3], v01871; VI-NEXT: flat_store_dword v[4:5], v11872; VI-NEXT: s_endpgm1873;1874; GFX9-LABEL: flat_atomic_dec_ret_i64_offset:1875; GFX9: ; %bb.0:1876; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01877; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s171878; GFX9-NEXT: v_mov_b32_e32 v0, 421879; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 01880; GFX9-NEXT: v_mov_b32_e32 v1, 01881; GFX9-NEXT: s_waitcnt lgkmcnt(0)1882; GFX9-NEXT: v_mov_b32_e32 v2, s21883; GFX9-NEXT: v_mov_b32_e32 v3, s31884; GFX9-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] offset:32 glc1885; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1886; GFX9-NEXT: buffer_wbinvl1_vol1887; GFX9-NEXT: v_mov_b32_e32 v3, s11888; GFX9-NEXT: v_mov_b32_e32 v2, s01889; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1890; GFX9-NEXT: s_endpgm1891;1892; GFX10-LABEL: flat_atomic_dec_ret_i64_offset:1893; GFX10: ; %bb.0:1894; GFX10-NEXT: s_add_u32 s12, s12, s171895; GFX10-NEXT: s_addc_u32 s13, s13, 01896; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s121897; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s131898; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01899; GFX10-NEXT: v_mov_b32_e32 v0, 421900; GFX10-NEXT: v_mov_b32_e32 v1, 01901; GFX10-NEXT: s_waitcnt lgkmcnt(0)1902; GFX10-NEXT: s_add_u32 s2, s2, 321903; GFX10-NEXT: s_addc_u32 s3, s3, 01904; GFX10-NEXT: v_mov_b32_e32 v2, s21905; GFX10-NEXT: v_mov_b32_e32 v3, s31906; GFX10-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc1907; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1908; GFX10-NEXT: buffer_gl1_inv1909; GFX10-NEXT: buffer_gl0_inv1910; GFX10-NEXT: v_mov_b32_e32 v3, s11911; GFX10-NEXT: v_mov_b32_e32 v2, s01912; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1913; GFX10-NEXT: s_endpgm1914;1915; GFX11-LABEL: flat_atomic_dec_ret_i64_offset:1916; GFX11: ; %bb.0:1917; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x01918; GFX11-NEXT: v_mov_b32_e32 v0, 421919; GFX11-NEXT: s_waitcnt lgkmcnt(0)1920; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s21921; GFX11-NEXT: v_mov_b32_e32 v3, s31922; GFX11-NEXT: flat_atomic_dec_u64 v[0:1], v[2:3], v[0:1] offset:32 glc1923; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1924; GFX11-NEXT: buffer_gl1_inv1925; GFX11-NEXT: buffer_gl0_inv1926; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s01927; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]1928; GFX11-NEXT: s_endpgm1929 %gep = getelementptr inbounds i64, ptr %ptr, i32 41930 %result = atomicrmw udec_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !11931 store i64 %result, ptr %out, align 41932 ret void1933}1934 1935define amdgpu_kernel void @flat_atomic_dec_noret_i64(ptr %ptr) #1 {1936; CI-LABEL: flat_atomic_dec_noret_i64:1937; CI: ; %bb.0:1938; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01939; CI-NEXT: s_add_i32 s12, s12, s171940; CI-NEXT: v_mov_b32_e32 v0, 421941; CI-NEXT: s_mov_b32 flat_scratch_lo, s131942; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81943; CI-NEXT: s_waitcnt lgkmcnt(0)1944; CI-NEXT: v_mov_b32_e32 v3, s11945; CI-NEXT: v_mov_b32_e32 v1, 01946; CI-NEXT: v_mov_b32_e32 v2, s01947; CI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]1948; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1949; CI-NEXT: buffer_wbinvl1_vol1950; CI-NEXT: s_endpgm1951;1952; VI-LABEL: flat_atomic_dec_noret_i64:1953; VI: ; %bb.0:1954; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01955; VI-NEXT: s_add_i32 s12, s12, s171956; VI-NEXT: v_mov_b32_e32 v0, 421957; VI-NEXT: s_mov_b32 flat_scratch_lo, s131958; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81959; VI-NEXT: s_waitcnt lgkmcnt(0)1960; VI-NEXT: v_mov_b32_e32 v3, s11961; VI-NEXT: v_mov_b32_e32 v1, 01962; VI-NEXT: v_mov_b32_e32 v2, s01963; VI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]1964; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1965; VI-NEXT: buffer_wbinvl1_vol1966; VI-NEXT: s_endpgm1967;1968; GFX9-LABEL: flat_atomic_dec_noret_i64:1969; GFX9: ; %bb.0:1970; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01971; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s171972; GFX9-NEXT: v_mov_b32_e32 v0, 421973; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 01974; GFX9-NEXT: v_mov_b32_e32 v1, 01975; GFX9-NEXT: s_waitcnt lgkmcnt(0)1976; GFX9-NEXT: v_mov_b32_e32 v3, s11977; GFX9-NEXT: v_mov_b32_e32 v2, s01978; GFX9-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]1979; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1980; GFX9-NEXT: buffer_wbinvl1_vol1981; GFX9-NEXT: s_endpgm1982;1983; GFX10-LABEL: flat_atomic_dec_noret_i64:1984; GFX10: ; %bb.0:1985; GFX10-NEXT: s_add_u32 s12, s12, s171986; GFX10-NEXT: s_addc_u32 s13, s13, 01987; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s121988; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s131989; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01990; GFX10-NEXT: v_mov_b32_e32 v0, 421991; GFX10-NEXT: v_mov_b32_e32 v1, 01992; GFX10-NEXT: s_waitcnt lgkmcnt(0)1993; GFX10-NEXT: v_mov_b32_e32 v3, s11994; GFX10-NEXT: v_mov_b32_e32 v2, s01995; GFX10-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]1996; GFX10-NEXT: s_waitcnt lgkmcnt(0)1997; GFX10-NEXT: s_waitcnt_vscnt null, 0x01998; GFX10-NEXT: buffer_gl1_inv1999; GFX10-NEXT: buffer_gl0_inv2000; GFX10-NEXT: s_endpgm2001;2002; GFX11-LABEL: flat_atomic_dec_noret_i64:2003; GFX11: ; %bb.0:2004; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x02005; GFX11-NEXT: v_mov_b32_e32 v0, 422006; GFX11-NEXT: v_mov_b32_e32 v1, 02007; GFX11-NEXT: s_waitcnt lgkmcnt(0)2008; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s02009; GFX11-NEXT: flat_atomic_dec_u64 v[2:3], v[0:1]2010; GFX11-NEXT: s_waitcnt lgkmcnt(0)2011; GFX11-NEXT: s_waitcnt_vscnt null, 0x02012; GFX11-NEXT: buffer_gl1_inv2013; GFX11-NEXT: buffer_gl0_inv2014; GFX11-NEXT: s_endpgm2015 %result = atomicrmw udec_wrap ptr %ptr, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !12016 ret void2017}2018 2019define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset(ptr %ptr) #1 {2020; CI-LABEL: flat_atomic_dec_noret_i64_offset:2021; CI: ; %bb.0:2022; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02023; CI-NEXT: s_add_i32 s12, s12, s172024; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82025; CI-NEXT: v_mov_b32_e32 v0, 422026; CI-NEXT: s_mov_b32 flat_scratch_lo, s132027; CI-NEXT: s_waitcnt lgkmcnt(0)2028; CI-NEXT: s_add_u32 s0, s0, 322029; CI-NEXT: s_addc_u32 s1, s1, 02030; CI-NEXT: v_mov_b32_e32 v3, s12031; CI-NEXT: v_mov_b32_e32 v1, 02032; CI-NEXT: v_mov_b32_e32 v2, s02033; CI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]2034; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2035; CI-NEXT: buffer_wbinvl1_vol2036; CI-NEXT: s_endpgm2037;2038; VI-LABEL: flat_atomic_dec_noret_i64_offset:2039; VI: ; %bb.0:2040; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02041; VI-NEXT: s_add_i32 s12, s12, s172042; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82043; VI-NEXT: v_mov_b32_e32 v0, 422044; VI-NEXT: s_mov_b32 flat_scratch_lo, s132045; VI-NEXT: s_waitcnt lgkmcnt(0)2046; VI-NEXT: s_add_u32 s0, s0, 322047; VI-NEXT: s_addc_u32 s1, s1, 02048; VI-NEXT: v_mov_b32_e32 v3, s12049; VI-NEXT: v_mov_b32_e32 v1, 02050; VI-NEXT: v_mov_b32_e32 v2, s02051; VI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]2052; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2053; VI-NEXT: buffer_wbinvl1_vol2054; VI-NEXT: s_endpgm2055;2056; GFX9-LABEL: flat_atomic_dec_noret_i64_offset:2057; GFX9: ; %bb.0:2058; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02059; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s172060; GFX9-NEXT: v_mov_b32_e32 v0, 422061; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 02062; GFX9-NEXT: v_mov_b32_e32 v1, 02063; GFX9-NEXT: s_waitcnt lgkmcnt(0)2064; GFX9-NEXT: v_mov_b32_e32 v3, s12065; GFX9-NEXT: v_mov_b32_e32 v2, s02066; GFX9-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1] offset:322067; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2068; GFX9-NEXT: buffer_wbinvl1_vol2069; GFX9-NEXT: s_endpgm2070;2071; GFX10-LABEL: flat_atomic_dec_noret_i64_offset:2072; GFX10: ; %bb.0:2073; GFX10-NEXT: s_add_u32 s12, s12, s172074; GFX10-NEXT: s_addc_u32 s13, s13, 02075; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s122076; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s132077; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02078; GFX10-NEXT: v_mov_b32_e32 v0, 422079; GFX10-NEXT: v_mov_b32_e32 v1, 02080; GFX10-NEXT: s_waitcnt lgkmcnt(0)2081; GFX10-NEXT: s_add_u32 s0, s0, 322082; GFX10-NEXT: s_addc_u32 s1, s1, 02083; GFX10-NEXT: v_mov_b32_e32 v3, s12084; GFX10-NEXT: v_mov_b32_e32 v2, s02085; GFX10-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]2086; GFX10-NEXT: s_waitcnt lgkmcnt(0)2087; GFX10-NEXT: s_waitcnt_vscnt null, 0x02088; GFX10-NEXT: buffer_gl1_inv2089; GFX10-NEXT: buffer_gl0_inv2090; GFX10-NEXT: s_endpgm2091;2092; GFX11-LABEL: flat_atomic_dec_noret_i64_offset:2093; GFX11: ; %bb.0:2094; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x02095; GFX11-NEXT: v_mov_b32_e32 v0, 422096; GFX11-NEXT: v_mov_b32_e32 v1, 02097; GFX11-NEXT: s_waitcnt lgkmcnt(0)2098; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s02099; GFX11-NEXT: flat_atomic_dec_u64 v[2:3], v[0:1] offset:322100; GFX11-NEXT: s_waitcnt lgkmcnt(0)2101; GFX11-NEXT: s_waitcnt_vscnt null, 0x02102; GFX11-NEXT: buffer_gl1_inv2103; GFX11-NEXT: buffer_gl0_inv2104; GFX11-NEXT: s_endpgm2105 %gep = getelementptr inbounds i64, ptr %ptr, i32 42106 %result = atomicrmw udec_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !12107 ret void2108}2109 2110define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_system(ptr %ptr) #1 {2111; CI-LABEL: flat_atomic_dec_noret_i64_offset_system:2112; CI: ; %bb.0:2113; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02114; CI-NEXT: s_add_i32 s12, s12, s172115; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82116; CI-NEXT: v_mov_b32_e32 v0, 422117; CI-NEXT: s_mov_b32 flat_scratch_lo, s132118; CI-NEXT: s_waitcnt lgkmcnt(0)2119; CI-NEXT: s_add_u32 s0, s0, 322120; CI-NEXT: s_addc_u32 s1, s1, 02121; CI-NEXT: v_mov_b32_e32 v3, s12122; CI-NEXT: v_mov_b32_e32 v1, 02123; CI-NEXT: v_mov_b32_e32 v2, s02124; CI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]2125; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2126; CI-NEXT: buffer_wbinvl1_vol2127; CI-NEXT: s_endpgm2128;2129; VI-LABEL: flat_atomic_dec_noret_i64_offset_system:2130; VI: ; %bb.0:2131; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02132; VI-NEXT: s_add_i32 s12, s12, s172133; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82134; VI-NEXT: v_mov_b32_e32 v0, 422135; VI-NEXT: s_mov_b32 flat_scratch_lo, s132136; VI-NEXT: s_waitcnt lgkmcnt(0)2137; VI-NEXT: s_add_u32 s0, s0, 322138; VI-NEXT: s_addc_u32 s1, s1, 02139; VI-NEXT: v_mov_b32_e32 v3, s12140; VI-NEXT: v_mov_b32_e32 v1, 02141; VI-NEXT: v_mov_b32_e32 v2, s02142; VI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]2143; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2144; VI-NEXT: buffer_wbinvl1_vol2145; VI-NEXT: s_endpgm2146;2147; GFX9-LABEL: flat_atomic_dec_noret_i64_offset_system:2148; GFX9: ; %bb.0:2149; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02150; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s172151; GFX9-NEXT: v_mov_b32_e32 v0, 422152; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 02153; GFX9-NEXT: v_mov_b32_e32 v1, 02154; GFX9-NEXT: s_waitcnt lgkmcnt(0)2155; GFX9-NEXT: v_mov_b32_e32 v3, s12156; GFX9-NEXT: v_mov_b32_e32 v2, s02157; GFX9-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1] offset:322158; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2159; GFX9-NEXT: buffer_wbinvl1_vol2160; GFX9-NEXT: s_endpgm2161;2162; GFX10-LABEL: flat_atomic_dec_noret_i64_offset_system:2163; GFX10: ; %bb.0:2164; GFX10-NEXT: s_add_u32 s12, s12, s172165; GFX10-NEXT: s_addc_u32 s13, s13, 02166; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s122167; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s132168; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02169; GFX10-NEXT: v_mov_b32_e32 v0, 422170; GFX10-NEXT: v_mov_b32_e32 v1, 02171; GFX10-NEXT: s_waitcnt lgkmcnt(0)2172; GFX10-NEXT: s_add_u32 s0, s0, 322173; GFX10-NEXT: s_addc_u32 s1, s1, 02174; GFX10-NEXT: v_mov_b32_e32 v3, s12175; GFX10-NEXT: v_mov_b32_e32 v2, s02176; GFX10-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]2177; GFX10-NEXT: s_waitcnt lgkmcnt(0)2178; GFX10-NEXT: s_waitcnt_vscnt null, 0x02179; GFX10-NEXT: buffer_gl1_inv2180; GFX10-NEXT: buffer_gl0_inv2181; GFX10-NEXT: s_endpgm2182;2183; GFX11-LABEL: flat_atomic_dec_noret_i64_offset_system:2184; GFX11: ; %bb.0:2185; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x02186; GFX11-NEXT: v_mov_b32_e32 v0, 422187; GFX11-NEXT: v_mov_b32_e32 v1, 02188; GFX11-NEXT: s_waitcnt lgkmcnt(0)2189; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s02190; GFX11-NEXT: flat_atomic_dec_u64 v[2:3], v[0:1] offset:322191; GFX11-NEXT: s_waitcnt lgkmcnt(0)2192; GFX11-NEXT: s_waitcnt_vscnt null, 0x02193; GFX11-NEXT: buffer_gl1_inv2194; GFX11-NEXT: buffer_gl0_inv2195; GFX11-NEXT: s_endpgm2196 %gep = getelementptr inbounds i64, ptr %ptr, i32 42197 %result = atomicrmw udec_wrap ptr %gep, i64 42 seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !12198 ret void2199}2200 2201define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset_addr64(ptr %out, ptr %ptr) #1 {2202; CI-LABEL: flat_atomic_dec_ret_i64_offset_addr64:2203; CI: ; %bb.0:2204; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02205; CI-NEXT: v_lshlrev_b32_e32 v4, 3, v02206; CI-NEXT: s_add_i32 s12, s12, s172207; CI-NEXT: s_mov_b32 flat_scratch_lo, s132208; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82209; CI-NEXT: s_waitcnt lgkmcnt(0)2210; CI-NEXT: v_mov_b32_e32 v0, s22211; CI-NEXT: v_mov_b32_e32 v1, s32212; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v42213; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc2214; CI-NEXT: v_mov_b32_e32 v0, 422215; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v22216; CI-NEXT: v_mov_b32_e32 v1, 02217; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc2218; CI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc2219; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2220; CI-NEXT: buffer_wbinvl1_vol2221; CI-NEXT: v_mov_b32_e32 v3, s12222; CI-NEXT: v_mov_b32_e32 v2, s02223; CI-NEXT: v_add_i32_e32 v2, vcc, v2, v42224; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc2225; CI-NEXT: v_add_i32_e32 v4, vcc, 4, v22226; CI-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc2227; CI-NEXT: flat_store_dword v[2:3], v02228; CI-NEXT: flat_store_dword v[4:5], v12229; CI-NEXT: s_endpgm2230;2231; VI-LABEL: flat_atomic_dec_ret_i64_offset_addr64:2232; VI: ; %bb.0:2233; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02234; VI-NEXT: v_lshlrev_b32_e32 v4, 3, v02235; VI-NEXT: s_add_i32 s12, s12, s172236; VI-NEXT: s_mov_b32 flat_scratch_lo, s132237; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82238; VI-NEXT: s_waitcnt lgkmcnt(0)2239; VI-NEXT: v_mov_b32_e32 v0, s22240; VI-NEXT: v_mov_b32_e32 v1, s32241; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v42242; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc2243; VI-NEXT: v_mov_b32_e32 v0, 422244; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v22245; VI-NEXT: v_mov_b32_e32 v1, 02246; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc2247; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc2248; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2249; VI-NEXT: buffer_wbinvl1_vol2250; VI-NEXT: v_mov_b32_e32 v3, s12251; VI-NEXT: v_mov_b32_e32 v2, s02252; VI-NEXT: v_add_u32_e32 v2, vcc, v2, v42253; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc2254; VI-NEXT: v_add_u32_e32 v4, vcc, 4, v22255; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc2256; VI-NEXT: flat_store_dword v[2:3], v02257; VI-NEXT: flat_store_dword v[4:5], v12258; VI-NEXT: s_endpgm2259;2260; GFX9-LABEL: flat_atomic_dec_ret_i64_offset_addr64:2261; GFX9: ; %bb.0:2262; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02263; GFX9-NEXT: v_lshlrev_b32_e32 v5, 3, v02264; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s172265; GFX9-NEXT: v_mov_b32_e32 v1, 422266; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 02267; GFX9-NEXT: s_waitcnt lgkmcnt(0)2268; GFX9-NEXT: v_mov_b32_e32 v4, s32269; GFX9-NEXT: v_mov_b32_e32 v3, s22270; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, v3, v52271; GFX9-NEXT: v_mov_b32_e32 v2, 02272; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc2273; GFX9-NEXT: flat_atomic_dec_x2 v[0:1], v[3:4], v[1:2] offset:40 glc2274; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2275; GFX9-NEXT: buffer_wbinvl1_vol2276; GFX9-NEXT: v_mov_b32_e32 v3, s12277; GFX9-NEXT: v_mov_b32_e32 v2, s02278; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v52279; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc2280; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]2281; GFX9-NEXT: s_endpgm2282;2283; GFX10-LABEL: flat_atomic_dec_ret_i64_offset_addr64:2284; GFX10: ; %bb.0:2285; GFX10-NEXT: s_add_u32 s12, s12, s172286; GFX10-NEXT: s_addc_u32 s13, s13, 02287; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s122288; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s132289; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02290; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v02291; GFX10-NEXT: s_waitcnt lgkmcnt(0)2292; GFX10-NEXT: v_mov_b32_e32 v0, s22293; GFX10-NEXT: v_mov_b32_e32 v1, s32294; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v0, v42295; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo2296; GFX10-NEXT: v_mov_b32_e32 v0, 422297; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, 402298; GFX10-NEXT: v_mov_b32_e32 v1, 02299; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo2300; GFX10-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc2301; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2302; GFX10-NEXT: buffer_gl1_inv2303; GFX10-NEXT: buffer_gl0_inv2304; GFX10-NEXT: v_mov_b32_e32 v3, s12305; GFX10-NEXT: v_mov_b32_e32 v2, s02306; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v42307; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo2308; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]2309; GFX10-NEXT: s_endpgm2310;2311; GFX11-LABEL: flat_atomic_dec_ret_i64_offset_addr64:2312; GFX11: ; %bb.0:2313; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x02314; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v02315; GFX11-NEXT: v_mov_b32_e32 v2, 422316; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)2317; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_lshlrev_b32 v4, 3, v02318; GFX11-NEXT: s_waitcnt lgkmcnt(0)2319; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s32320; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v42321; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)2322; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo2323; GFX11-NEXT: flat_atomic_dec_u64 v[0:1], v[0:1], v[2:3] offset:40 glc2324; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2325; GFX11-NEXT: buffer_gl1_inv2326; GFX11-NEXT: buffer_gl0_inv2327; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s02328; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2329; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, v2, v42330; GFX11-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo2331; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]2332; GFX11-NEXT: s_endpgm2333 %id = call i32 @llvm.amdgcn.workitem.id.x()2334 %gep.tid = getelementptr i64, ptr %ptr, i32 %id2335 %out.gep = getelementptr i64, ptr %out, i32 %id2336 %gep = getelementptr inbounds i64, ptr %gep.tid, i32 52337 %result = atomicrmw udec_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !12338 store i64 %result, ptr %out.gep, align 42339 ret void2340}2341 2342define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_addr64(ptr %ptr) #1 {2343; CI-LABEL: flat_atomic_dec_noret_i64_offset_addr64:2344; CI: ; %bb.0:2345; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02346; CI-NEXT: v_lshlrev_b32_e32 v2, 3, v02347; CI-NEXT: s_add_i32 s12, s12, s172348; CI-NEXT: s_mov_b32 flat_scratch_lo, s132349; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82350; CI-NEXT: s_waitcnt lgkmcnt(0)2351; CI-NEXT: v_mov_b32_e32 v0, s02352; CI-NEXT: v_mov_b32_e32 v1, s12353; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v22354; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc2355; CI-NEXT: v_mov_b32_e32 v0, 422356; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v22357; CI-NEXT: v_mov_b32_e32 v1, 02358; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc2359; CI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]2360; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2361; CI-NEXT: buffer_wbinvl1_vol2362; CI-NEXT: s_endpgm2363;2364; VI-LABEL: flat_atomic_dec_noret_i64_offset_addr64:2365; VI: ; %bb.0:2366; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02367; VI-NEXT: v_lshlrev_b32_e32 v2, 3, v02368; VI-NEXT: s_add_i32 s12, s12, s172369; VI-NEXT: s_mov_b32 flat_scratch_lo, s132370; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82371; VI-NEXT: s_waitcnt lgkmcnt(0)2372; VI-NEXT: v_mov_b32_e32 v0, s02373; VI-NEXT: v_mov_b32_e32 v1, s12374; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v22375; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc2376; VI-NEXT: v_mov_b32_e32 v0, 422377; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v22378; VI-NEXT: v_mov_b32_e32 v1, 02379; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc2380; VI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]2381; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2382; VI-NEXT: buffer_wbinvl1_vol2383; VI-NEXT: s_endpgm2384;2385; GFX9-LABEL: flat_atomic_dec_noret_i64_offset_addr64:2386; GFX9: ; %bb.0:2387; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02388; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v02389; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s172390; GFX9-NEXT: v_mov_b32_e32 v1, 422391; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 02392; GFX9-NEXT: s_waitcnt lgkmcnt(0)2393; GFX9-NEXT: v_mov_b32_e32 v4, s12394; GFX9-NEXT: v_mov_b32_e32 v3, s02395; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, v3, v02396; GFX9-NEXT: v_mov_b32_e32 v2, 02397; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc2398; GFX9-NEXT: flat_atomic_dec_x2 v[3:4], v[1:2] offset:402399; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2400; GFX9-NEXT: buffer_wbinvl1_vol2401; GFX9-NEXT: s_endpgm2402;2403; GFX10-LABEL: flat_atomic_dec_noret_i64_offset_addr64:2404; GFX10: ; %bb.0:2405; GFX10-NEXT: s_add_u32 s12, s12, s172406; GFX10-NEXT: s_addc_u32 s13, s13, 02407; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s122408; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s132409; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02410; GFX10-NEXT: v_lshlrev_b32_e32 v2, 3, v02411; GFX10-NEXT: s_waitcnt lgkmcnt(0)2412; GFX10-NEXT: v_mov_b32_e32 v0, s02413; GFX10-NEXT: v_mov_b32_e32 v1, s12414; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v0, v22415; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo2416; GFX10-NEXT: v_mov_b32_e32 v0, 422417; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, 402418; GFX10-NEXT: v_mov_b32_e32 v1, 02419; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo2420; GFX10-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]2421; GFX10-NEXT: s_waitcnt lgkmcnt(0)2422; GFX10-NEXT: s_waitcnt_vscnt null, 0x02423; GFX10-NEXT: buffer_gl1_inv2424; GFX10-NEXT: buffer_gl0_inv2425; GFX10-NEXT: s_endpgm2426;2427; GFX11-LABEL: flat_atomic_dec_noret_i64_offset_addr64:2428; GFX11: ; %bb.0:2429; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x02430; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v02431; GFX11-NEXT: v_mov_b32_e32 v2, 422432; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)2433; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_lshlrev_b32 v4, 3, v02434; GFX11-NEXT: s_waitcnt lgkmcnt(0)2435; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12436; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v42437; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)2438; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo2439; GFX11-NEXT: flat_atomic_dec_u64 v[0:1], v[2:3] offset:402440; GFX11-NEXT: s_waitcnt lgkmcnt(0)2441; GFX11-NEXT: s_waitcnt_vscnt null, 0x02442; GFX11-NEXT: buffer_gl1_inv2443; GFX11-NEXT: buffer_gl0_inv2444; GFX11-NEXT: s_endpgm2445 %id = call i32 @llvm.amdgcn.workitem.id.x()2446 %gep.tid = getelementptr i64, ptr %ptr, i32 %id2447 %gep = getelementptr inbounds i64, ptr %gep.tid, i32 52448 %result = atomicrmw udec_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !12449 ret void2450}2451 2452define amdgpu_kernel void @atomic_dec_shl_base_lds_0(ptr addrspace(1) %out, ptr addrspace(1) %add_use) #1 {2453; CI-LABEL: atomic_dec_shl_base_lds_0:2454; CI: ; %bb.0:2455; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02456; CI-NEXT: v_lshlrev_b32_e32 v1, 2, v02457; CI-NEXT: v_mov_b32_e32 v2, 92458; CI-NEXT: s_mov_b32 m0, -12459; CI-NEXT: s_waitcnt lgkmcnt(0)2460; CI-NEXT: ds_dec_rtn_u32 v2, v1, v2 offset:82461; CI-NEXT: s_waitcnt lgkmcnt(0)2462; CI-NEXT: s_add_i32 s12, s12, s172463; CI-NEXT: v_add_i32_e32 v3, vcc, 2, v02464; CI-NEXT: v_mov_b32_e32 v0, s22465; CI-NEXT: s_mov_b32 flat_scratch_lo, s132466; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82467; CI-NEXT: v_mov_b32_e32 v1, s32468; CI-NEXT: flat_store_dword v[0:1], v32469; CI-NEXT: v_mov_b32_e32 v0, s02470; CI-NEXT: v_mov_b32_e32 v1, s12471; CI-NEXT: flat_store_dword v[0:1], v22472; CI-NEXT: s_endpgm2473;2474; VI-LABEL: atomic_dec_shl_base_lds_0:2475; VI: ; %bb.0:2476; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02477; VI-NEXT: v_lshlrev_b32_e32 v1, 2, v02478; VI-NEXT: v_mov_b32_e32 v2, 92479; VI-NEXT: s_mov_b32 m0, -12480; VI-NEXT: s_waitcnt lgkmcnt(0)2481; VI-NEXT: ds_dec_rtn_u32 v2, v1, v2 offset:82482; VI-NEXT: s_waitcnt lgkmcnt(0)2483; VI-NEXT: s_add_i32 s12, s12, s172484; VI-NEXT: v_add_u32_e32 v3, vcc, 2, v02485; VI-NEXT: v_mov_b32_e32 v0, s22486; VI-NEXT: s_mov_b32 flat_scratch_lo, s132487; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82488; VI-NEXT: v_mov_b32_e32 v1, s32489; VI-NEXT: flat_store_dword v[0:1], v32490; VI-NEXT: v_mov_b32_e32 v0, s02491; VI-NEXT: v_mov_b32_e32 v1, s12492; VI-NEXT: flat_store_dword v[0:1], v22493; VI-NEXT: s_endpgm2494;2495; GFX9-LABEL: atomic_dec_shl_base_lds_0:2496; GFX9: ; %bb.0:2497; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02498; GFX9-NEXT: v_lshlrev_b32_e32 v1, 2, v02499; GFX9-NEXT: v_mov_b32_e32 v2, 92500; GFX9-NEXT: s_waitcnt lgkmcnt(0)2501; GFX9-NEXT: ds_dec_rtn_u32 v1, v1, v2 offset:82502; GFX9-NEXT: s_waitcnt lgkmcnt(0)2503; GFX9-NEXT: v_add_u32_e32 v0, 2, v02504; GFX9-NEXT: v_mov_b32_e32 v2, 02505; GFX9-NEXT: global_store_dword v2, v0, s[2:3]2506; GFX9-NEXT: global_store_dword v2, v1, s[0:1]2507; GFX9-NEXT: s_endpgm2508;2509; GFX10-LABEL: atomic_dec_shl_base_lds_0:2510; GFX10: ; %bb.0:2511; GFX10-NEXT: v_lshlrev_b32_e32 v1, 2, v02512; GFX10-NEXT: v_mov_b32_e32 v2, 92513; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02514; GFX10-NEXT: v_add_nc_u32_e32 v0, 2, v02515; GFX10-NEXT: s_waitcnt lgkmcnt(0)2516; GFX10-NEXT: ds_dec_rtn_u32 v1, v1, v2 offset:82517; GFX10-NEXT: s_waitcnt lgkmcnt(0)2518; GFX10-NEXT: buffer_gl0_inv2519; GFX10-NEXT: v_mov_b32_e32 v2, 02520; GFX10-NEXT: global_store_dword v2, v0, s[2:3]2521; GFX10-NEXT: global_store_dword v2, v1, s[0:1]2522; GFX10-NEXT: s_endpgm2523;2524; GFX11-LABEL: atomic_dec_shl_base_lds_0:2525; GFX11: ; %bb.0:2526; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v02527; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x02528; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)2529; GFX11-NEXT: v_dual_mov_b32 v2, 9 :: v_dual_lshlrev_b32 v1, 2, v02530; GFX11-NEXT: v_add_nc_u32_e32 v0, 2, v02531; GFX11-NEXT: s_waitcnt lgkmcnt(0)2532; GFX11-NEXT: ds_dec_rtn_u32 v1, v1, v2 offset:82533; GFX11-NEXT: s_waitcnt lgkmcnt(0)2534; GFX11-NEXT: buffer_gl0_inv2535; GFX11-NEXT: v_mov_b32_e32 v2, 02536; GFX11-NEXT: s_clause 0x12537; GFX11-NEXT: global_store_b32 v2, v0, s[2:3]2538; GFX11-NEXT: global_store_b32 v2, v1, s[0:1]2539; GFX11-NEXT: s_endpgm2540 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #22541 %idx.0 = add nsw i32 %tid.x, 22542 %arrayidx0 = getelementptr inbounds [512 x i32], ptr addrspace(3) @lds0, i32 0, i32 %idx.02543 %result = atomicrmw udec_wrap ptr addrspace(3) %arrayidx0, i32 9 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !12544 store i32 %idx.0, ptr addrspace(1) %add_use, align 42545 store i32 %result, ptr addrspace(1) %out, align 42546 ret void2547}2548 2549define amdgpu_kernel void @lds_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {2550; CI-LABEL: lds_atomic_dec_ret_i64:2551; CI: ; %bb.0:2552; CI-NEXT: s_load_dword s2, s[8:9], 0x22553; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02554; CI-NEXT: v_mov_b32_e32 v0, 422555; CI-NEXT: v_mov_b32_e32 v1, 02556; CI-NEXT: s_mov_b32 m0, -12557; CI-NEXT: s_waitcnt lgkmcnt(0)2558; CI-NEXT: v_mov_b32_e32 v2, s22559; CI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]2560; CI-NEXT: s_waitcnt lgkmcnt(0)2561; CI-NEXT: s_add_i32 s12, s12, s172562; CI-NEXT: v_mov_b32_e32 v3, s12563; CI-NEXT: s_mov_b32 flat_scratch_lo, s132564; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82565; CI-NEXT: v_mov_b32_e32 v2, s02566; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]2567; CI-NEXT: s_endpgm2568;2569; VI-LABEL: lds_atomic_dec_ret_i64:2570; VI: ; %bb.0:2571; VI-NEXT: s_load_dword s2, s[8:9], 0x82572; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02573; VI-NEXT: v_mov_b32_e32 v0, 422574; VI-NEXT: v_mov_b32_e32 v1, 02575; VI-NEXT: s_mov_b32 m0, -12576; VI-NEXT: s_waitcnt lgkmcnt(0)2577; VI-NEXT: v_mov_b32_e32 v2, s22578; VI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]2579; VI-NEXT: s_waitcnt lgkmcnt(0)2580; VI-NEXT: s_add_i32 s12, s12, s172581; VI-NEXT: v_mov_b32_e32 v3, s12582; VI-NEXT: s_mov_b32 flat_scratch_lo, s132583; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82584; VI-NEXT: v_mov_b32_e32 v2, s02585; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]2586; VI-NEXT: s_endpgm2587;2588; GFX9-LABEL: lds_atomic_dec_ret_i64:2589; GFX9: ; %bb.0:2590; GFX9-NEXT: s_load_dword s2, s[8:9], 0x82591; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02592; GFX9-NEXT: v_mov_b32_e32 v0, 422593; GFX9-NEXT: v_mov_b32_e32 v1, 02594; GFX9-NEXT: s_waitcnt lgkmcnt(0)2595; GFX9-NEXT: v_mov_b32_e32 v2, s22596; GFX9-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]2597; GFX9-NEXT: s_waitcnt lgkmcnt(0)2598; GFX9-NEXT: v_mov_b32_e32 v2, 02599; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]2600; GFX9-NEXT: s_endpgm2601;2602; GFX10-LABEL: lds_atomic_dec_ret_i64:2603; GFX10: ; %bb.0:2604; GFX10-NEXT: s_load_dword s0, s[8:9], 0x82605; GFX10-NEXT: v_mov_b32_e32 v0, 422606; GFX10-NEXT: v_mov_b32_e32 v1, 02607; GFX10-NEXT: s_waitcnt lgkmcnt(0)2608; GFX10-NEXT: v_mov_b32_e32 v2, s02609; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02610; GFX10-NEXT: s_waitcnt lgkmcnt(0)2611; GFX10-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]2612; GFX10-NEXT: s_waitcnt lgkmcnt(0)2613; GFX10-NEXT: buffer_gl0_inv2614; GFX10-NEXT: v_mov_b32_e32 v2, 02615; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]2616; GFX10-NEXT: s_endpgm2617;2618; GFX11-LABEL: lds_atomic_dec_ret_i64:2619; GFX11: ; %bb.0:2620; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x82621; GFX11-NEXT: v_mov_b32_e32 v0, 422622; GFX11-NEXT: s_waitcnt lgkmcnt(0)2623; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s02624; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x02625; GFX11-NEXT: s_waitcnt lgkmcnt(0)2626; GFX11-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]2627; GFX11-NEXT: s_waitcnt lgkmcnt(0)2628; GFX11-NEXT: buffer_gl0_inv2629; GFX11-NEXT: v_mov_b32_e32 v2, 02630; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]2631; GFX11-NEXT: s_endpgm2632 %result = atomicrmw udec_wrap ptr addrspace(3) %ptr, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !12633 store i64 %result, ptr addrspace(1) %out, align 42634 ret void2635}2636 2637define amdgpu_kernel void @lds_atomic_dec_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {2638; CI-LABEL: lds_atomic_dec_ret_i64_offset:2639; CI: ; %bb.0:2640; CI-NEXT: s_load_dword s2, s[8:9], 0x22641; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02642; CI-NEXT: v_mov_b32_e32 v0, 422643; CI-NEXT: v_mov_b32_e32 v1, 02644; CI-NEXT: s_mov_b32 m0, -12645; CI-NEXT: s_waitcnt lgkmcnt(0)2646; CI-NEXT: v_mov_b32_e32 v2, s22647; CI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:322648; CI-NEXT: s_waitcnt lgkmcnt(0)2649; CI-NEXT: s_add_i32 s12, s12, s172650; CI-NEXT: v_mov_b32_e32 v3, s12651; CI-NEXT: s_mov_b32 flat_scratch_lo, s132652; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82653; CI-NEXT: v_mov_b32_e32 v2, s02654; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]2655; CI-NEXT: s_endpgm2656;2657; VI-LABEL: lds_atomic_dec_ret_i64_offset:2658; VI: ; %bb.0:2659; VI-NEXT: s_load_dword s2, s[8:9], 0x82660; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02661; VI-NEXT: v_mov_b32_e32 v0, 422662; VI-NEXT: v_mov_b32_e32 v1, 02663; VI-NEXT: s_mov_b32 m0, -12664; VI-NEXT: s_waitcnt lgkmcnt(0)2665; VI-NEXT: v_mov_b32_e32 v2, s22666; VI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:322667; VI-NEXT: s_waitcnt lgkmcnt(0)2668; VI-NEXT: s_add_i32 s12, s12, s172669; VI-NEXT: v_mov_b32_e32 v3, s12670; VI-NEXT: s_mov_b32 flat_scratch_lo, s132671; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82672; VI-NEXT: v_mov_b32_e32 v2, s02673; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]2674; VI-NEXT: s_endpgm2675;2676; GFX9-LABEL: lds_atomic_dec_ret_i64_offset:2677; GFX9: ; %bb.0:2678; GFX9-NEXT: s_load_dword s2, s[8:9], 0x82679; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02680; GFX9-NEXT: v_mov_b32_e32 v0, 422681; GFX9-NEXT: v_mov_b32_e32 v1, 02682; GFX9-NEXT: s_waitcnt lgkmcnt(0)2683; GFX9-NEXT: v_mov_b32_e32 v2, s22684; GFX9-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:322685; GFX9-NEXT: s_waitcnt lgkmcnt(0)2686; GFX9-NEXT: v_mov_b32_e32 v2, 02687; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]2688; GFX9-NEXT: s_endpgm2689;2690; GFX10-LABEL: lds_atomic_dec_ret_i64_offset:2691; GFX10: ; %bb.0:2692; GFX10-NEXT: s_load_dword s0, s[8:9], 0x82693; GFX10-NEXT: v_mov_b32_e32 v0, 422694; GFX10-NEXT: v_mov_b32_e32 v1, 02695; GFX10-NEXT: s_waitcnt lgkmcnt(0)2696; GFX10-NEXT: v_mov_b32_e32 v2, s02697; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02698; GFX10-NEXT: s_waitcnt lgkmcnt(0)2699; GFX10-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:322700; GFX10-NEXT: s_waitcnt lgkmcnt(0)2701; GFX10-NEXT: buffer_gl0_inv2702; GFX10-NEXT: v_mov_b32_e32 v2, 02703; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]2704; GFX10-NEXT: s_endpgm2705;2706; GFX11-LABEL: lds_atomic_dec_ret_i64_offset:2707; GFX11: ; %bb.0:2708; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x82709; GFX11-NEXT: v_mov_b32_e32 v0, 422710; GFX11-NEXT: s_waitcnt lgkmcnt(0)2711; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s02712; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x02713; GFX11-NEXT: s_waitcnt lgkmcnt(0)2714; GFX11-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:322715; GFX11-NEXT: s_waitcnt lgkmcnt(0)2716; GFX11-NEXT: buffer_gl0_inv2717; GFX11-NEXT: v_mov_b32_e32 v2, 02718; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]2719; GFX11-NEXT: s_endpgm2720 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 42721 %result = atomicrmw udec_wrap ptr addrspace(3) %gep, i64 42 syncscope("agent") seq_cst, align 82722 store i64 %result, ptr addrspace(1) %out, align 42723 ret void2724}2725 2726define amdgpu_kernel void @lds_atomic_dec_noret_i64(ptr addrspace(3) %ptr) #1 {2727; CI-LABEL: lds_atomic_dec_noret_i64:2728; CI: ; %bb.0:2729; CI-NEXT: s_load_dword s0, s[8:9], 0x02730; CI-NEXT: v_mov_b32_e32 v0, 422731; CI-NEXT: v_mov_b32_e32 v1, 02732; CI-NEXT: s_mov_b32 m0, -12733; CI-NEXT: s_waitcnt lgkmcnt(0)2734; CI-NEXT: v_mov_b32_e32 v2, s02735; CI-NEXT: ds_dec_u64 v2, v[0:1]2736; CI-NEXT: s_waitcnt lgkmcnt(0)2737; CI-NEXT: s_endpgm2738;2739; VI-LABEL: lds_atomic_dec_noret_i64:2740; VI: ; %bb.0:2741; VI-NEXT: s_load_dword s0, s[8:9], 0x02742; VI-NEXT: v_mov_b32_e32 v0, 422743; VI-NEXT: v_mov_b32_e32 v1, 02744; VI-NEXT: s_mov_b32 m0, -12745; VI-NEXT: s_waitcnt lgkmcnt(0)2746; VI-NEXT: v_mov_b32_e32 v2, s02747; VI-NEXT: ds_dec_u64 v2, v[0:1]2748; VI-NEXT: s_waitcnt lgkmcnt(0)2749; VI-NEXT: s_endpgm2750;2751; GFX9-LABEL: lds_atomic_dec_noret_i64:2752; GFX9: ; %bb.0:2753; GFX9-NEXT: s_load_dword s0, s[8:9], 0x02754; GFX9-NEXT: v_mov_b32_e32 v0, 422755; GFX9-NEXT: v_mov_b32_e32 v1, 02756; GFX9-NEXT: s_waitcnt lgkmcnt(0)2757; GFX9-NEXT: v_mov_b32_e32 v2, s02758; GFX9-NEXT: ds_dec_u64 v2, v[0:1]2759; GFX9-NEXT: s_waitcnt lgkmcnt(0)2760; GFX9-NEXT: s_endpgm2761;2762; GFX10-LABEL: lds_atomic_dec_noret_i64:2763; GFX10: ; %bb.0:2764; GFX10-NEXT: s_load_dword s0, s[8:9], 0x02765; GFX10-NEXT: v_mov_b32_e32 v0, 422766; GFX10-NEXT: v_mov_b32_e32 v1, 02767; GFX10-NEXT: s_waitcnt lgkmcnt(0)2768; GFX10-NEXT: v_mov_b32_e32 v2, s02769; GFX10-NEXT: ds_dec_u64 v2, v[0:1]2770; GFX10-NEXT: s_waitcnt lgkmcnt(0)2771; GFX10-NEXT: buffer_gl0_inv2772; GFX10-NEXT: s_endpgm2773;2774; GFX11-LABEL: lds_atomic_dec_noret_i64:2775; GFX11: ; %bb.0:2776; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x02777; GFX11-NEXT: v_mov_b32_e32 v0, 422778; GFX11-NEXT: s_waitcnt lgkmcnt(0)2779; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s02780; GFX11-NEXT: ds_dec_u64 v2, v[0:1]2781; GFX11-NEXT: s_waitcnt lgkmcnt(0)2782; GFX11-NEXT: buffer_gl0_inv2783; GFX11-NEXT: s_endpgm2784 %result = atomicrmw udec_wrap ptr addrspace(3) %ptr, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !12785 ret void2786}2787 2788define amdgpu_kernel void @lds_atomic_dec_noret_i64_offset(ptr addrspace(3) %ptr) #1 {2789; CI-LABEL: lds_atomic_dec_noret_i64_offset:2790; CI: ; %bb.0:2791; CI-NEXT: s_load_dword s0, s[8:9], 0x02792; CI-NEXT: v_mov_b32_e32 v0, 422793; CI-NEXT: v_mov_b32_e32 v1, 02794; CI-NEXT: s_mov_b32 m0, -12795; CI-NEXT: s_waitcnt lgkmcnt(0)2796; CI-NEXT: v_mov_b32_e32 v2, s02797; CI-NEXT: ds_dec_u64 v2, v[0:1] offset:322798; CI-NEXT: s_waitcnt lgkmcnt(0)2799; CI-NEXT: s_endpgm2800;2801; VI-LABEL: lds_atomic_dec_noret_i64_offset:2802; VI: ; %bb.0:2803; VI-NEXT: s_load_dword s0, s[8:9], 0x02804; VI-NEXT: v_mov_b32_e32 v0, 422805; VI-NEXT: v_mov_b32_e32 v1, 02806; VI-NEXT: s_mov_b32 m0, -12807; VI-NEXT: s_waitcnt lgkmcnt(0)2808; VI-NEXT: v_mov_b32_e32 v2, s02809; VI-NEXT: ds_dec_u64 v2, v[0:1] offset:322810; VI-NEXT: s_waitcnt lgkmcnt(0)2811; VI-NEXT: s_endpgm2812;2813; GFX9-LABEL: lds_atomic_dec_noret_i64_offset:2814; GFX9: ; %bb.0:2815; GFX9-NEXT: s_load_dword s0, s[8:9], 0x02816; GFX9-NEXT: v_mov_b32_e32 v0, 422817; GFX9-NEXT: v_mov_b32_e32 v1, 02818; GFX9-NEXT: s_waitcnt lgkmcnt(0)2819; GFX9-NEXT: v_mov_b32_e32 v2, s02820; GFX9-NEXT: ds_dec_u64 v2, v[0:1] offset:322821; GFX9-NEXT: s_waitcnt lgkmcnt(0)2822; GFX9-NEXT: s_endpgm2823;2824; GFX10-LABEL: lds_atomic_dec_noret_i64_offset:2825; GFX10: ; %bb.0:2826; GFX10-NEXT: s_load_dword s0, s[8:9], 0x02827; GFX10-NEXT: v_mov_b32_e32 v0, 422828; GFX10-NEXT: v_mov_b32_e32 v1, 02829; GFX10-NEXT: s_waitcnt lgkmcnt(0)2830; GFX10-NEXT: v_mov_b32_e32 v2, s02831; GFX10-NEXT: ds_dec_u64 v2, v[0:1] offset:322832; GFX10-NEXT: s_waitcnt lgkmcnt(0)2833; GFX10-NEXT: buffer_gl0_inv2834; GFX10-NEXT: s_endpgm2835;2836; GFX11-LABEL: lds_atomic_dec_noret_i64_offset:2837; GFX11: ; %bb.0:2838; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x02839; GFX11-NEXT: v_mov_b32_e32 v0, 422840; GFX11-NEXT: s_waitcnt lgkmcnt(0)2841; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s02842; GFX11-NEXT: ds_dec_u64 v2, v[0:1] offset:322843; GFX11-NEXT: s_waitcnt lgkmcnt(0)2844; GFX11-NEXT: buffer_gl0_inv2845; GFX11-NEXT: s_endpgm2846 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 42847 %result = atomicrmw udec_wrap ptr addrspace(3) %gep, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !12848 ret void2849}2850 2851define amdgpu_kernel void @global_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #1 {2852; CI-LABEL: global_atomic_dec_ret_i64:2853; CI: ; %bb.0:2854; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02855; CI-NEXT: s_add_i32 s12, s12, s172856; CI-NEXT: v_mov_b32_e32 v0, 422857; CI-NEXT: s_mov_b32 flat_scratch_lo, s132858; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82859; CI-NEXT: s_waitcnt lgkmcnt(0)2860; CI-NEXT: v_mov_b32_e32 v2, s22861; CI-NEXT: v_mov_b32_e32 v1, 02862; CI-NEXT: v_mov_b32_e32 v3, s32863; CI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc2864; CI-NEXT: s_waitcnt vmcnt(0)2865; CI-NEXT: buffer_wbinvl1_vol2866; CI-NEXT: v_mov_b32_e32 v3, s12867; CI-NEXT: v_mov_b32_e32 v2, s02868; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]2869; CI-NEXT: s_endpgm2870;2871; VI-LABEL: global_atomic_dec_ret_i64:2872; VI: ; %bb.0:2873; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02874; VI-NEXT: s_add_i32 s12, s12, s172875; VI-NEXT: v_mov_b32_e32 v0, 422876; VI-NEXT: s_mov_b32 flat_scratch_lo, s132877; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82878; VI-NEXT: s_waitcnt lgkmcnt(0)2879; VI-NEXT: v_mov_b32_e32 v2, s22880; VI-NEXT: v_mov_b32_e32 v1, 02881; VI-NEXT: v_mov_b32_e32 v3, s32882; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc2883; VI-NEXT: s_waitcnt vmcnt(0)2884; VI-NEXT: buffer_wbinvl1_vol2885; VI-NEXT: v_mov_b32_e32 v3, s12886; VI-NEXT: v_mov_b32_e32 v2, s02887; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]2888; VI-NEXT: s_endpgm2889;2890; GFX9-LABEL: global_atomic_dec_ret_i64:2891; GFX9: ; %bb.0:2892; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02893; GFX9-NEXT: v_mov_b32_e32 v0, 422894; GFX9-NEXT: v_mov_b32_e32 v1, 02895; GFX9-NEXT: v_mov_b32_e32 v2, 02896; GFX9-NEXT: s_waitcnt lgkmcnt(0)2897; GFX9-NEXT: global_atomic_dec_x2 v[0:1], v2, v[0:1], s[2:3] glc2898; GFX9-NEXT: s_waitcnt vmcnt(0)2899; GFX9-NEXT: buffer_wbinvl1_vol2900; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]2901; GFX9-NEXT: s_endpgm2902;2903; GFX10-LABEL: global_atomic_dec_ret_i64:2904; GFX10: ; %bb.0:2905; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02906; GFX10-NEXT: v_mov_b32_e32 v0, 422907; GFX10-NEXT: v_mov_b32_e32 v1, 02908; GFX10-NEXT: v_mov_b32_e32 v2, 02909; GFX10-NEXT: s_waitcnt lgkmcnt(0)2910; GFX10-NEXT: global_atomic_dec_x2 v[0:1], v2, v[0:1], s[2:3] glc2911; GFX10-NEXT: s_waitcnt vmcnt(0)2912; GFX10-NEXT: buffer_gl1_inv2913; GFX10-NEXT: buffer_gl0_inv2914; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]2915; GFX10-NEXT: s_endpgm2916;2917; GFX11-LABEL: global_atomic_dec_ret_i64:2918; GFX11: ; %bb.0:2919; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x02920; GFX11-NEXT: v_mov_b32_e32 v0, 422921; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 02922; GFX11-NEXT: s_waitcnt lgkmcnt(0)2923; GFX11-NEXT: global_atomic_dec_u64 v[0:1], v2, v[0:1], s[2:3] glc2924; GFX11-NEXT: s_waitcnt vmcnt(0)2925; GFX11-NEXT: buffer_gl1_inv2926; GFX11-NEXT: buffer_gl0_inv2927; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]2928; GFX11-NEXT: s_endpgm2929 %result = atomicrmw udec_wrap ptr addrspace(1) %ptr, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !12930 store i64 %result, ptr addrspace(1) %out, align 42931 ret void2932}2933 2934define amdgpu_kernel void @global_atomic_dec_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #1 {2935; CI-LABEL: global_atomic_dec_ret_i64_offset:2936; CI: ; %bb.0:2937; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02938; CI-NEXT: s_add_i32 s12, s12, s172939; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82940; CI-NEXT: v_mov_b32_e32 v0, 422941; CI-NEXT: s_mov_b32 flat_scratch_lo, s132942; CI-NEXT: s_waitcnt lgkmcnt(0)2943; CI-NEXT: s_add_u32 s2, s2, 322944; CI-NEXT: s_addc_u32 s3, s3, 02945; CI-NEXT: v_mov_b32_e32 v2, s22946; CI-NEXT: v_mov_b32_e32 v1, 02947; CI-NEXT: v_mov_b32_e32 v3, s32948; CI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc2949; CI-NEXT: s_waitcnt vmcnt(0)2950; CI-NEXT: buffer_wbinvl1_vol2951; CI-NEXT: v_mov_b32_e32 v3, s12952; CI-NEXT: v_mov_b32_e32 v2, s02953; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]2954; CI-NEXT: s_endpgm2955;2956; VI-LABEL: global_atomic_dec_ret_i64_offset:2957; VI: ; %bb.0:2958; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02959; VI-NEXT: s_add_i32 s12, s12, s172960; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82961; VI-NEXT: v_mov_b32_e32 v0, 422962; VI-NEXT: s_mov_b32 flat_scratch_lo, s132963; VI-NEXT: s_waitcnt lgkmcnt(0)2964; VI-NEXT: s_add_u32 s2, s2, 322965; VI-NEXT: s_addc_u32 s3, s3, 02966; VI-NEXT: v_mov_b32_e32 v2, s22967; VI-NEXT: v_mov_b32_e32 v1, 02968; VI-NEXT: v_mov_b32_e32 v3, s32969; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc2970; VI-NEXT: s_waitcnt vmcnt(0)2971; VI-NEXT: buffer_wbinvl1_vol2972; VI-NEXT: v_mov_b32_e32 v3, s12973; VI-NEXT: v_mov_b32_e32 v2, s02974; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]2975; VI-NEXT: s_endpgm2976;2977; GFX9-LABEL: global_atomic_dec_ret_i64_offset:2978; GFX9: ; %bb.0:2979; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02980; GFX9-NEXT: v_mov_b32_e32 v0, 422981; GFX9-NEXT: v_mov_b32_e32 v1, 02982; GFX9-NEXT: v_mov_b32_e32 v2, 02983; GFX9-NEXT: s_waitcnt lgkmcnt(0)2984; GFX9-NEXT: global_atomic_dec_x2 v[0:1], v2, v[0:1], s[2:3] offset:32 glc2985; GFX9-NEXT: s_waitcnt vmcnt(0)2986; GFX9-NEXT: buffer_wbinvl1_vol2987; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]2988; GFX9-NEXT: s_endpgm2989;2990; GFX10-LABEL: global_atomic_dec_ret_i64_offset:2991; GFX10: ; %bb.0:2992; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02993; GFX10-NEXT: v_mov_b32_e32 v0, 422994; GFX10-NEXT: v_mov_b32_e32 v1, 02995; GFX10-NEXT: v_mov_b32_e32 v2, 02996; GFX10-NEXT: s_waitcnt lgkmcnt(0)2997; GFX10-NEXT: global_atomic_dec_x2 v[0:1], v2, v[0:1], s[2:3] offset:32 glc2998; GFX10-NEXT: s_waitcnt vmcnt(0)2999; GFX10-NEXT: buffer_gl1_inv3000; GFX10-NEXT: buffer_gl0_inv3001; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]3002; GFX10-NEXT: s_endpgm3003;3004; GFX11-LABEL: global_atomic_dec_ret_i64_offset:3005; GFX11: ; %bb.0:3006; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x03007; GFX11-NEXT: v_mov_b32_e32 v0, 423008; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 03009; GFX11-NEXT: s_waitcnt lgkmcnt(0)3010; GFX11-NEXT: global_atomic_dec_u64 v[0:1], v2, v[0:1], s[2:3] offset:32 glc3011; GFX11-NEXT: s_waitcnt vmcnt(0)3012; GFX11-NEXT: buffer_gl1_inv3013; GFX11-NEXT: buffer_gl0_inv3014; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]3015; GFX11-NEXT: s_endpgm3016 %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 43017 %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !13018 store i64 %result, ptr addrspace(1) %out, align 43019 ret void3020}3021 3022define amdgpu_kernel void @global_atomic_dec_ret_i64_offset_system(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #1 {3023; CI-LABEL: global_atomic_dec_ret_i64_offset_system:3024; CI: ; %bb.0:3025; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03026; CI-NEXT: s_add_i32 s12, s12, s173027; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83028; CI-NEXT: v_mov_b32_e32 v0, 423029; CI-NEXT: s_mov_b32 flat_scratch_lo, s133030; CI-NEXT: s_waitcnt lgkmcnt(0)3031; CI-NEXT: s_add_u32 s2, s2, 323032; CI-NEXT: s_addc_u32 s3, s3, 03033; CI-NEXT: v_mov_b32_e32 v2, s23034; CI-NEXT: v_mov_b32_e32 v1, 03035; CI-NEXT: v_mov_b32_e32 v3, s33036; CI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc3037; CI-NEXT: s_waitcnt vmcnt(0)3038; CI-NEXT: buffer_wbinvl1_vol3039; CI-NEXT: v_mov_b32_e32 v3, s13040; CI-NEXT: v_mov_b32_e32 v2, s03041; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]3042; CI-NEXT: s_endpgm3043;3044; VI-LABEL: global_atomic_dec_ret_i64_offset_system:3045; VI: ; %bb.0:3046; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03047; VI-NEXT: s_add_i32 s12, s12, s173048; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83049; VI-NEXT: v_mov_b32_e32 v0, 423050; VI-NEXT: s_mov_b32 flat_scratch_lo, s133051; VI-NEXT: s_waitcnt lgkmcnt(0)3052; VI-NEXT: s_add_u32 s2, s2, 323053; VI-NEXT: s_addc_u32 s3, s3, 03054; VI-NEXT: v_mov_b32_e32 v2, s23055; VI-NEXT: v_mov_b32_e32 v1, 03056; VI-NEXT: v_mov_b32_e32 v3, s33057; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc3058; VI-NEXT: s_waitcnt vmcnt(0)3059; VI-NEXT: buffer_wbinvl1_vol3060; VI-NEXT: v_mov_b32_e32 v3, s13061; VI-NEXT: v_mov_b32_e32 v2, s03062; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]3063; VI-NEXT: s_endpgm3064;3065; GFX9-LABEL: global_atomic_dec_ret_i64_offset_system:3066; GFX9: ; %bb.0:3067; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03068; GFX9-NEXT: v_mov_b32_e32 v0, 423069; GFX9-NEXT: v_mov_b32_e32 v1, 03070; GFX9-NEXT: v_mov_b32_e32 v2, 03071; GFX9-NEXT: s_waitcnt lgkmcnt(0)3072; GFX9-NEXT: global_atomic_dec_x2 v[0:1], v2, v[0:1], s[2:3] offset:32 glc3073; GFX9-NEXT: s_waitcnt vmcnt(0)3074; GFX9-NEXT: buffer_wbinvl1_vol3075; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]3076; GFX9-NEXT: s_endpgm3077;3078; GFX10-LABEL: global_atomic_dec_ret_i64_offset_system:3079; GFX10: ; %bb.0:3080; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03081; GFX10-NEXT: v_mov_b32_e32 v0, 423082; GFX10-NEXT: v_mov_b32_e32 v1, 03083; GFX10-NEXT: v_mov_b32_e32 v2, 03084; GFX10-NEXT: s_waitcnt lgkmcnt(0)3085; GFX10-NEXT: global_atomic_dec_x2 v[0:1], v2, v[0:1], s[2:3] offset:32 glc3086; GFX10-NEXT: s_waitcnt vmcnt(0)3087; GFX10-NEXT: buffer_gl1_inv3088; GFX10-NEXT: buffer_gl0_inv3089; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]3090; GFX10-NEXT: s_endpgm3091;3092; GFX11-LABEL: global_atomic_dec_ret_i64_offset_system:3093; GFX11: ; %bb.0:3094; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x03095; GFX11-NEXT: v_mov_b32_e32 v0, 423096; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 03097; GFX11-NEXT: s_waitcnt lgkmcnt(0)3098; GFX11-NEXT: global_atomic_dec_u64 v[0:1], v2, v[0:1], s[2:3] offset:32 glc3099; GFX11-NEXT: s_waitcnt vmcnt(0)3100; GFX11-NEXT: buffer_gl1_inv3101; GFX11-NEXT: buffer_gl0_inv3102; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]3103; GFX11-NEXT: s_endpgm3104 %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 43105 %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 42 seq_cst, align 8, !amdgpu.no.remote.memory !13106 store i64 %result, ptr addrspace(1) %out, align 43107 ret void3108}3109 3110define amdgpu_kernel void @global_atomic_dec_noret_i64(ptr addrspace(1) %ptr) #1 {3111; CI-LABEL: global_atomic_dec_noret_i64:3112; CI: ; %bb.0:3113; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03114; CI-NEXT: s_add_i32 s12, s12, s173115; CI-NEXT: v_mov_b32_e32 v0, 423116; CI-NEXT: s_mov_b32 flat_scratch_lo, s133117; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83118; CI-NEXT: s_waitcnt lgkmcnt(0)3119; CI-NEXT: v_mov_b32_e32 v3, s13120; CI-NEXT: v_mov_b32_e32 v1, 03121; CI-NEXT: v_mov_b32_e32 v2, s03122; CI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]3123; CI-NEXT: s_waitcnt vmcnt(0)3124; CI-NEXT: buffer_wbinvl1_vol3125; CI-NEXT: s_endpgm3126;3127; VI-LABEL: global_atomic_dec_noret_i64:3128; VI: ; %bb.0:3129; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03130; VI-NEXT: s_add_i32 s12, s12, s173131; VI-NEXT: v_mov_b32_e32 v0, 423132; VI-NEXT: s_mov_b32 flat_scratch_lo, s133133; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83134; VI-NEXT: s_waitcnt lgkmcnt(0)3135; VI-NEXT: v_mov_b32_e32 v3, s13136; VI-NEXT: v_mov_b32_e32 v1, 03137; VI-NEXT: v_mov_b32_e32 v2, s03138; VI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]3139; VI-NEXT: s_waitcnt vmcnt(0)3140; VI-NEXT: buffer_wbinvl1_vol3141; VI-NEXT: s_endpgm3142;3143; GFX9-LABEL: global_atomic_dec_noret_i64:3144; GFX9: ; %bb.0:3145; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03146; GFX9-NEXT: v_mov_b32_e32 v0, 423147; GFX9-NEXT: v_mov_b32_e32 v1, 03148; GFX9-NEXT: v_mov_b32_e32 v2, 03149; GFX9-NEXT: s_waitcnt lgkmcnt(0)3150; GFX9-NEXT: global_atomic_dec_x2 v2, v[0:1], s[0:1]3151; GFX9-NEXT: s_waitcnt vmcnt(0)3152; GFX9-NEXT: buffer_wbinvl1_vol3153; GFX9-NEXT: s_endpgm3154;3155; GFX10-LABEL: global_atomic_dec_noret_i64:3156; GFX10: ; %bb.0:3157; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03158; GFX10-NEXT: v_mov_b32_e32 v0, 423159; GFX10-NEXT: v_mov_b32_e32 v1, 03160; GFX10-NEXT: v_mov_b32_e32 v2, 03161; GFX10-NEXT: s_waitcnt lgkmcnt(0)3162; GFX10-NEXT: global_atomic_dec_x2 v2, v[0:1], s[0:1]3163; GFX10-NEXT: s_waitcnt_vscnt null, 0x03164; GFX10-NEXT: buffer_gl1_inv3165; GFX10-NEXT: buffer_gl0_inv3166; GFX10-NEXT: s_endpgm3167;3168; GFX11-LABEL: global_atomic_dec_noret_i64:3169; GFX11: ; %bb.0:3170; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x03171; GFX11-NEXT: v_mov_b32_e32 v0, 423172; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 03173; GFX11-NEXT: s_waitcnt lgkmcnt(0)3174; GFX11-NEXT: global_atomic_dec_u64 v2, v[0:1], s[0:1]3175; GFX11-NEXT: s_waitcnt_vscnt null, 0x03176; GFX11-NEXT: buffer_gl1_inv3177; GFX11-NEXT: buffer_gl0_inv3178; GFX11-NEXT: s_endpgm3179 %result = atomicrmw udec_wrap ptr addrspace(1) %ptr, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !13180 ret void3181}3182 3183define amdgpu_kernel void @global_atomic_dec_noret_i64_offset(ptr addrspace(1) %ptr) #1 {3184; CI-LABEL: global_atomic_dec_noret_i64_offset:3185; CI: ; %bb.0:3186; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03187; CI-NEXT: s_add_i32 s12, s12, s173188; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83189; CI-NEXT: v_mov_b32_e32 v0, 423190; CI-NEXT: s_mov_b32 flat_scratch_lo, s133191; CI-NEXT: s_waitcnt lgkmcnt(0)3192; CI-NEXT: s_add_u32 s0, s0, 323193; CI-NEXT: s_addc_u32 s1, s1, 03194; CI-NEXT: v_mov_b32_e32 v3, s13195; CI-NEXT: v_mov_b32_e32 v1, 03196; CI-NEXT: v_mov_b32_e32 v2, s03197; CI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]3198; CI-NEXT: s_waitcnt vmcnt(0)3199; CI-NEXT: buffer_wbinvl1_vol3200; CI-NEXT: s_endpgm3201;3202; VI-LABEL: global_atomic_dec_noret_i64_offset:3203; VI: ; %bb.0:3204; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03205; VI-NEXT: s_add_i32 s12, s12, s173206; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83207; VI-NEXT: v_mov_b32_e32 v0, 423208; VI-NEXT: s_mov_b32 flat_scratch_lo, s133209; VI-NEXT: s_waitcnt lgkmcnt(0)3210; VI-NEXT: s_add_u32 s0, s0, 323211; VI-NEXT: s_addc_u32 s1, s1, 03212; VI-NEXT: v_mov_b32_e32 v3, s13213; VI-NEXT: v_mov_b32_e32 v1, 03214; VI-NEXT: v_mov_b32_e32 v2, s03215; VI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]3216; VI-NEXT: s_waitcnt vmcnt(0)3217; VI-NEXT: buffer_wbinvl1_vol3218; VI-NEXT: s_endpgm3219;3220; GFX9-LABEL: global_atomic_dec_noret_i64_offset:3221; GFX9: ; %bb.0:3222; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03223; GFX9-NEXT: v_mov_b32_e32 v0, 423224; GFX9-NEXT: v_mov_b32_e32 v1, 03225; GFX9-NEXT: v_mov_b32_e32 v2, 03226; GFX9-NEXT: s_waitcnt lgkmcnt(0)3227; GFX9-NEXT: global_atomic_dec_x2 v2, v[0:1], s[0:1] offset:323228; GFX9-NEXT: s_waitcnt vmcnt(0)3229; GFX9-NEXT: buffer_wbinvl1_vol3230; GFX9-NEXT: s_endpgm3231;3232; GFX10-LABEL: global_atomic_dec_noret_i64_offset:3233; GFX10: ; %bb.0:3234; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03235; GFX10-NEXT: v_mov_b32_e32 v0, 423236; GFX10-NEXT: v_mov_b32_e32 v1, 03237; GFX10-NEXT: v_mov_b32_e32 v2, 03238; GFX10-NEXT: s_waitcnt lgkmcnt(0)3239; GFX10-NEXT: global_atomic_dec_x2 v2, v[0:1], s[0:1] offset:323240; GFX10-NEXT: s_waitcnt_vscnt null, 0x03241; GFX10-NEXT: buffer_gl1_inv3242; GFX10-NEXT: buffer_gl0_inv3243; GFX10-NEXT: s_endpgm3244;3245; GFX11-LABEL: global_atomic_dec_noret_i64_offset:3246; GFX11: ; %bb.0:3247; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x03248; GFX11-NEXT: v_mov_b32_e32 v0, 423249; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 03250; GFX11-NEXT: s_waitcnt lgkmcnt(0)3251; GFX11-NEXT: global_atomic_dec_u64 v2, v[0:1], s[0:1] offset:323252; GFX11-NEXT: s_waitcnt_vscnt null, 0x03253; GFX11-NEXT: buffer_gl1_inv3254; GFX11-NEXT: buffer_gl0_inv3255; GFX11-NEXT: s_endpgm3256 %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 43257 %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !13258 ret void3259}3260 3261define amdgpu_kernel void @global_atomic_dec_noret_i64_offset_system(ptr addrspace(1) %ptr) #1 {3262; CI-LABEL: global_atomic_dec_noret_i64_offset_system:3263; CI: ; %bb.0:3264; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03265; CI-NEXT: s_add_i32 s12, s12, s173266; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83267; CI-NEXT: v_mov_b32_e32 v0, 423268; CI-NEXT: s_mov_b32 flat_scratch_lo, s133269; CI-NEXT: s_waitcnt lgkmcnt(0)3270; CI-NEXT: s_add_u32 s0, s0, 323271; CI-NEXT: s_addc_u32 s1, s1, 03272; CI-NEXT: v_mov_b32_e32 v3, s13273; CI-NEXT: v_mov_b32_e32 v1, 03274; CI-NEXT: v_mov_b32_e32 v2, s03275; CI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]3276; CI-NEXT: s_waitcnt vmcnt(0)3277; CI-NEXT: buffer_wbinvl1_vol3278; CI-NEXT: s_endpgm3279;3280; VI-LABEL: global_atomic_dec_noret_i64_offset_system:3281; VI: ; %bb.0:3282; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03283; VI-NEXT: s_add_i32 s12, s12, s173284; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83285; VI-NEXT: v_mov_b32_e32 v0, 423286; VI-NEXT: s_mov_b32 flat_scratch_lo, s133287; VI-NEXT: s_waitcnt lgkmcnt(0)3288; VI-NEXT: s_add_u32 s0, s0, 323289; VI-NEXT: s_addc_u32 s1, s1, 03290; VI-NEXT: v_mov_b32_e32 v3, s13291; VI-NEXT: v_mov_b32_e32 v1, 03292; VI-NEXT: v_mov_b32_e32 v2, s03293; VI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]3294; VI-NEXT: s_waitcnt vmcnt(0)3295; VI-NEXT: buffer_wbinvl1_vol3296; VI-NEXT: s_endpgm3297;3298; GFX9-LABEL: global_atomic_dec_noret_i64_offset_system:3299; GFX9: ; %bb.0:3300; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03301; GFX9-NEXT: v_mov_b32_e32 v0, 423302; GFX9-NEXT: v_mov_b32_e32 v1, 03303; GFX9-NEXT: v_mov_b32_e32 v2, 03304; GFX9-NEXT: s_waitcnt lgkmcnt(0)3305; GFX9-NEXT: global_atomic_dec_x2 v2, v[0:1], s[0:1] offset:323306; GFX9-NEXT: s_waitcnt vmcnt(0)3307; GFX9-NEXT: buffer_wbinvl1_vol3308; GFX9-NEXT: s_endpgm3309;3310; GFX10-LABEL: global_atomic_dec_noret_i64_offset_system:3311; GFX10: ; %bb.0:3312; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03313; GFX10-NEXT: v_mov_b32_e32 v0, 423314; GFX10-NEXT: v_mov_b32_e32 v1, 03315; GFX10-NEXT: v_mov_b32_e32 v2, 03316; GFX10-NEXT: s_waitcnt lgkmcnt(0)3317; GFX10-NEXT: global_atomic_dec_x2 v2, v[0:1], s[0:1] offset:323318; GFX10-NEXT: s_waitcnt_vscnt null, 0x03319; GFX10-NEXT: buffer_gl1_inv3320; GFX10-NEXT: buffer_gl0_inv3321; GFX10-NEXT: s_endpgm3322;3323; GFX11-LABEL: global_atomic_dec_noret_i64_offset_system:3324; GFX11: ; %bb.0:3325; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x03326; GFX11-NEXT: v_mov_b32_e32 v0, 423327; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 03328; GFX11-NEXT: s_waitcnt lgkmcnt(0)3329; GFX11-NEXT: global_atomic_dec_u64 v2, v[0:1], s[0:1] offset:323330; GFX11-NEXT: s_waitcnt_vscnt null, 0x03331; GFX11-NEXT: buffer_gl1_inv3332; GFX11-NEXT: buffer_gl0_inv3333; GFX11-NEXT: s_endpgm3334 %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 43335 %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 42 seq_cst, align 8, !amdgpu.no.remote.memory !13336 ret void3337}3338 3339define amdgpu_kernel void @global_atomic_dec_ret_i64_offset_addr64(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #1 {3340; CI-LABEL: global_atomic_dec_ret_i64_offset_addr64:3341; CI: ; %bb.0:3342; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03343; CI-NEXT: v_lshlrev_b32_e32 v4, 3, v03344; CI-NEXT: s_add_i32 s12, s12, s173345; CI-NEXT: s_mov_b32 flat_scratch_lo, s133346; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83347; CI-NEXT: s_waitcnt lgkmcnt(0)3348; CI-NEXT: v_mov_b32_e32 v0, s23349; CI-NEXT: v_mov_b32_e32 v1, s33350; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v43351; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc3352; CI-NEXT: v_mov_b32_e32 v0, 423353; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v23354; CI-NEXT: v_mov_b32_e32 v1, 03355; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc3356; CI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc3357; CI-NEXT: s_waitcnt vmcnt(0)3358; CI-NEXT: buffer_wbinvl1_vol3359; CI-NEXT: v_mov_b32_e32 v3, s13360; CI-NEXT: v_mov_b32_e32 v2, s03361; CI-NEXT: v_add_i32_e32 v2, vcc, v2, v43362; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc3363; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]3364; CI-NEXT: s_endpgm3365;3366; VI-LABEL: global_atomic_dec_ret_i64_offset_addr64:3367; VI: ; %bb.0:3368; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03369; VI-NEXT: v_lshlrev_b32_e32 v4, 3, v03370; VI-NEXT: s_add_i32 s12, s12, s173371; VI-NEXT: s_mov_b32 flat_scratch_lo, s133372; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83373; VI-NEXT: s_waitcnt lgkmcnt(0)3374; VI-NEXT: v_mov_b32_e32 v0, s23375; VI-NEXT: v_mov_b32_e32 v1, s33376; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v43377; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc3378; VI-NEXT: v_mov_b32_e32 v0, 423379; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v23380; VI-NEXT: v_mov_b32_e32 v1, 03381; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc3382; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc3383; VI-NEXT: s_waitcnt vmcnt(0)3384; VI-NEXT: buffer_wbinvl1_vol3385; VI-NEXT: v_mov_b32_e32 v3, s13386; VI-NEXT: v_mov_b32_e32 v2, s03387; VI-NEXT: v_add_u32_e32 v2, vcc, v2, v43388; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc3389; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]3390; VI-NEXT: s_endpgm3391;3392; GFX9-LABEL: global_atomic_dec_ret_i64_offset_addr64:3393; GFX9: ; %bb.0:3394; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03395; GFX9-NEXT: v_mov_b32_e32 v1, 423396; GFX9-NEXT: v_mov_b32_e32 v2, 03397; GFX9-NEXT: v_lshlrev_b32_e32 v3, 3, v03398; GFX9-NEXT: s_waitcnt lgkmcnt(0)3399; GFX9-NEXT: global_atomic_dec_x2 v[0:1], v3, v[1:2], s[2:3] offset:40 glc3400; GFX9-NEXT: s_waitcnt vmcnt(0)3401; GFX9-NEXT: buffer_wbinvl1_vol3402; GFX9-NEXT: global_store_dwordx2 v3, v[0:1], s[0:1]3403; GFX9-NEXT: s_endpgm3404;3405; GFX10-LABEL: global_atomic_dec_ret_i64_offset_addr64:3406; GFX10: ; %bb.0:3407; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03408; GFX10-NEXT: v_mov_b32_e32 v1, 423409; GFX10-NEXT: v_mov_b32_e32 v2, 03410; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v03411; GFX10-NEXT: s_waitcnt lgkmcnt(0)3412; GFX10-NEXT: global_atomic_dec_x2 v[0:1], v3, v[1:2], s[2:3] offset:40 glc3413; GFX10-NEXT: s_waitcnt vmcnt(0)3414; GFX10-NEXT: buffer_gl1_inv3415; GFX10-NEXT: buffer_gl0_inv3416; GFX10-NEXT: global_store_dwordx2 v3, v[0:1], s[0:1]3417; GFX10-NEXT: s_endpgm3418;3419; GFX11-LABEL: global_atomic_dec_ret_i64_offset_addr64:3420; GFX11: ; %bb.0:3421; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x03422; GFX11-NEXT: v_and_b32_e32 v2, 0x3ff, v03423; GFX11-NEXT: v_mov_b32_e32 v0, 423424; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)3425; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v2, 3, v23426; GFX11-NEXT: s_waitcnt lgkmcnt(0)3427; GFX11-NEXT: global_atomic_dec_u64 v[0:1], v2, v[0:1], s[2:3] offset:40 glc3428; GFX11-NEXT: s_waitcnt vmcnt(0)3429; GFX11-NEXT: buffer_gl1_inv3430; GFX11-NEXT: buffer_gl0_inv3431; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]3432; GFX11-NEXT: s_endpgm3433 %id = call i32 @llvm.amdgcn.workitem.id.x()3434 %gep.tid = getelementptr i64, ptr addrspace(1) %ptr, i32 %id3435 %out.gep = getelementptr i64, ptr addrspace(1) %out, i32 %id3436 %gep = getelementptr i64, ptr addrspace(1) %gep.tid, i32 53437 %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !13438 store i64 %result, ptr addrspace(1) %out.gep, align 43439 ret void3440}3441 3442define amdgpu_kernel void @global_atomic_dec_noret_i64_offset_addr64(ptr addrspace(1) %ptr) #1 {3443; CI-LABEL: global_atomic_dec_noret_i64_offset_addr64:3444; CI: ; %bb.0:3445; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03446; CI-NEXT: v_lshlrev_b32_e32 v2, 3, v03447; CI-NEXT: s_add_i32 s12, s12, s173448; CI-NEXT: s_mov_b32 flat_scratch_lo, s133449; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83450; CI-NEXT: s_waitcnt lgkmcnt(0)3451; CI-NEXT: v_mov_b32_e32 v0, s03452; CI-NEXT: v_mov_b32_e32 v1, s13453; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v23454; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc3455; CI-NEXT: v_mov_b32_e32 v0, 423456; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v23457; CI-NEXT: v_mov_b32_e32 v1, 03458; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc3459; CI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]3460; CI-NEXT: s_waitcnt vmcnt(0)3461; CI-NEXT: buffer_wbinvl1_vol3462; CI-NEXT: s_endpgm3463;3464; VI-LABEL: global_atomic_dec_noret_i64_offset_addr64:3465; VI: ; %bb.0:3466; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03467; VI-NEXT: v_lshlrev_b32_e32 v2, 3, v03468; VI-NEXT: s_add_i32 s12, s12, s173469; VI-NEXT: s_mov_b32 flat_scratch_lo, s133470; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83471; VI-NEXT: s_waitcnt lgkmcnt(0)3472; VI-NEXT: v_mov_b32_e32 v0, s03473; VI-NEXT: v_mov_b32_e32 v1, s13474; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v23475; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc3476; VI-NEXT: v_mov_b32_e32 v0, 423477; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v23478; VI-NEXT: v_mov_b32_e32 v1, 03479; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc3480; VI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]3481; VI-NEXT: s_waitcnt vmcnt(0)3482; VI-NEXT: buffer_wbinvl1_vol3483; VI-NEXT: s_endpgm3484;3485; GFX9-LABEL: global_atomic_dec_noret_i64_offset_addr64:3486; GFX9: ; %bb.0:3487; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03488; GFX9-NEXT: v_mov_b32_e32 v1, 423489; GFX9-NEXT: v_mov_b32_e32 v2, 03490; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v03491; GFX9-NEXT: s_waitcnt lgkmcnt(0)3492; GFX9-NEXT: global_atomic_dec_x2 v0, v[1:2], s[0:1] offset:403493; GFX9-NEXT: s_waitcnt vmcnt(0)3494; GFX9-NEXT: buffer_wbinvl1_vol3495; GFX9-NEXT: s_endpgm3496;3497; GFX10-LABEL: global_atomic_dec_noret_i64_offset_addr64:3498; GFX10: ; %bb.0:3499; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03500; GFX10-NEXT: v_mov_b32_e32 v1, 423501; GFX10-NEXT: v_mov_b32_e32 v2, 03502; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v03503; GFX10-NEXT: s_waitcnt lgkmcnt(0)3504; GFX10-NEXT: global_atomic_dec_x2 v0, v[1:2], s[0:1] offset:403505; GFX10-NEXT: s_waitcnt_vscnt null, 0x03506; GFX10-NEXT: buffer_gl1_inv3507; GFX10-NEXT: buffer_gl0_inv3508; GFX10-NEXT: s_endpgm3509;3510; GFX11-LABEL: global_atomic_dec_noret_i64_offset_addr64:3511; GFX11: ; %bb.0:3512; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x03513; GFX11-NEXT: v_and_b32_e32 v2, 0x3ff, v03514; GFX11-NEXT: v_mov_b32_e32 v0, 423515; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)3516; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v2, 3, v23517; GFX11-NEXT: s_waitcnt lgkmcnt(0)3518; GFX11-NEXT: global_atomic_dec_u64 v2, v[0:1], s[0:1] offset:403519; GFX11-NEXT: s_waitcnt_vscnt null, 0x03520; GFX11-NEXT: buffer_gl1_inv3521; GFX11-NEXT: buffer_gl0_inv3522; GFX11-NEXT: s_endpgm3523 %id = call i32 @llvm.amdgcn.workitem.id.x()3524 %gep.tid = getelementptr i64, ptr addrspace(1) %ptr, i32 %id3525 %gep = getelementptr i64, ptr addrspace(1) %gep.tid, i32 53526 %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !13527 ret void3528}3529 3530define amdgpu_kernel void @atomic_dec_shl_base_lds_0_i64(ptr addrspace(1) %out, ptr addrspace(1) %add_use) #1 {3531; CI-LABEL: atomic_dec_shl_base_lds_0_i64:3532; CI: ; %bb.0:3533; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03534; CI-NEXT: v_mov_b32_e32 v1, 93535; CI-NEXT: v_lshlrev_b32_e32 v3, 3, v03536; CI-NEXT: v_mov_b32_e32 v2, 03537; CI-NEXT: s_mov_b32 m0, -13538; CI-NEXT: s_waitcnt lgkmcnt(0)3539; CI-NEXT: ds_dec_rtn_u64 v[1:2], v3, v[1:2] offset:163540; CI-NEXT: s_waitcnt lgkmcnt(0)3541; CI-NEXT: s_add_i32 s12, s12, s173542; CI-NEXT: v_mov_b32_e32 v4, s33543; CI-NEXT: s_mov_b32 flat_scratch_lo, s133544; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83545; CI-NEXT: v_add_i32_e32 v0, vcc, 2, v03546; CI-NEXT: v_mov_b32_e32 v3, s23547; CI-NEXT: flat_store_dword v[3:4], v03548; CI-NEXT: v_mov_b32_e32 v4, s13549; CI-NEXT: v_mov_b32_e32 v3, s03550; CI-NEXT: flat_store_dwordx2 v[3:4], v[1:2]3551; CI-NEXT: s_endpgm3552;3553; VI-LABEL: atomic_dec_shl_base_lds_0_i64:3554; VI: ; %bb.0:3555; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03556; VI-NEXT: v_mov_b32_e32 v1, 93557; VI-NEXT: v_lshlrev_b32_e32 v3, 3, v03558; VI-NEXT: v_mov_b32_e32 v2, 03559; VI-NEXT: s_mov_b32 m0, -13560; VI-NEXT: s_waitcnt lgkmcnt(0)3561; VI-NEXT: ds_dec_rtn_u64 v[1:2], v3, v[1:2] offset:163562; VI-NEXT: s_waitcnt lgkmcnt(0)3563; VI-NEXT: s_add_i32 s12, s12, s173564; VI-NEXT: v_mov_b32_e32 v4, s33565; VI-NEXT: s_mov_b32 flat_scratch_lo, s133566; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83567; VI-NEXT: v_add_u32_e32 v0, vcc, 2, v03568; VI-NEXT: v_mov_b32_e32 v3, s23569; VI-NEXT: flat_store_dword v[3:4], v03570; VI-NEXT: v_mov_b32_e32 v4, s13571; VI-NEXT: v_mov_b32_e32 v3, s03572; VI-NEXT: flat_store_dwordx2 v[3:4], v[1:2]3573; VI-NEXT: s_endpgm3574;3575; GFX9-LABEL: atomic_dec_shl_base_lds_0_i64:3576; GFX9: ; %bb.0:3577; GFX9-NEXT: v_mov_b32_e32 v1, 93578; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03579; GFX9-NEXT: v_mov_b32_e32 v2, 03580; GFX9-NEXT: v_lshlrev_b32_e32 v3, 3, v03581; GFX9-NEXT: s_waitcnt lgkmcnt(0)3582; GFX9-NEXT: ds_dec_rtn_u64 v[1:2], v3, v[1:2] offset:163583; GFX9-NEXT: s_waitcnt lgkmcnt(0)3584; GFX9-NEXT: v_add_u32_e32 v0, 2, v03585; GFX9-NEXT: v_mov_b32_e32 v3, 03586; GFX9-NEXT: global_store_dword v3, v0, s[2:3]3587; GFX9-NEXT: global_store_dwordx2 v3, v[1:2], s[0:1]3588; GFX9-NEXT: s_endpgm3589;3590; GFX10-LABEL: atomic_dec_shl_base_lds_0_i64:3591; GFX10: ; %bb.0:3592; GFX10-NEXT: v_mov_b32_e32 v1, 93593; GFX10-NEXT: v_mov_b32_e32 v2, 03594; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v03595; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03596; GFX10-NEXT: v_add_nc_u32_e32 v0, 2, v03597; GFX10-NEXT: s_waitcnt lgkmcnt(0)3598; GFX10-NEXT: ds_dec_rtn_u64 v[1:2], v3, v[1:2] offset:163599; GFX10-NEXT: s_waitcnt lgkmcnt(0)3600; GFX10-NEXT: buffer_gl0_inv3601; GFX10-NEXT: v_mov_b32_e32 v3, 03602; GFX10-NEXT: global_store_dword v3, v0, s[2:3]3603; GFX10-NEXT: global_store_dwordx2 v3, v[1:2], s[0:1]3604; GFX10-NEXT: s_endpgm3605;3606; GFX11-LABEL: atomic_dec_shl_base_lds_0_i64:3607; GFX11: ; %bb.0:3608; GFX11-NEXT: v_and_b32_e32 v2, 0x3ff, v03609; GFX11-NEXT: v_mov_b32_e32 v0, 93610; GFX11-NEXT: v_mov_b32_e32 v1, 03611; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x03612; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)3613; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v23614; GFX11-NEXT: v_add_nc_u32_e32 v2, 2, v23615; GFX11-NEXT: s_waitcnt lgkmcnt(0)3616; GFX11-NEXT: ds_dec_rtn_u64 v[0:1], v3, v[0:1] offset:163617; GFX11-NEXT: s_waitcnt lgkmcnt(0)3618; GFX11-NEXT: buffer_gl0_inv3619; GFX11-NEXT: v_mov_b32_e32 v3, 03620; GFX11-NEXT: s_clause 0x13621; GFX11-NEXT: global_store_b32 v3, v2, s[2:3]3622; GFX11-NEXT: global_store_b64 v3, v[0:1], s[0:1]3623; GFX11-NEXT: s_endpgm3624 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #23625 %idx.0 = add nsw i32 %tid.x, 23626 %arrayidx0 = getelementptr inbounds [512 x i64], ptr addrspace(3) @lds1, i32 0, i32 %idx.03627 %result = atomicrmw udec_wrap ptr addrspace(3) %arrayidx0, i64 9 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !13628 store i32 %idx.0, ptr addrspace(1) %add_use, align 43629 store i64 %result, ptr addrspace(1) %out, align 43630 ret void3631}3632 3633attributes #0 = { nounwind speculatable willreturn memory(none) }3634attributes #1 = { nounwind }3635attributes #2 = { nounwind memory(none) }3636 3637!0 = !{i32 5, i32 6}3638!1 = !{}3639 3640;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:3641; GCN: {{.*}}3642