1762 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc -mtriple=amdgcn -mcpu=bonaire < %s | FileCheck -enable-var-scope -check-prefix=CI %s3; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=VI %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s5 6declare i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) nocapture, i32, i32, i32, i1) #27declare i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) nocapture, i32, i32, i32, i1) #28declare i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr nocapture, i32, i32, i32, i1) #29 10declare i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) nocapture, i64, i32, i32, i1) #211declare i64 @llvm.amdgcn.atomic.dec.i64.p3(ptr addrspace(3) nocapture, i64, i32, i32, i1) #212declare i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr nocapture, i64, i32, i32, i1) #213 14declare i32 @llvm.amdgcn.workitem.id.x() #115 16define amdgpu_kernel void @lds_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 {17; CI-LABEL: lds_atomic_dec_ret_i32:18; CI: ; %bb.0:19; CI-NEXT: s_load_dword s2, s[4:5], 0xb20; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x921; CI-NEXT: v_mov_b32_e32 v0, 4222; CI-NEXT: s_mov_b32 m0, -123; CI-NEXT: s_mov_b32 s3, 0xf00024; CI-NEXT: s_waitcnt lgkmcnt(0)25; CI-NEXT: v_mov_b32_e32 v1, s226; CI-NEXT: ds_dec_rtn_u32 v0, v1, v027; CI-NEXT: s_waitcnt lgkmcnt(0)28; CI-NEXT: s_mov_b32 s2, -129; CI-NEXT: buffer_store_dword v0, off, s[0:3], 030; CI-NEXT: s_endpgm31;32; VI-LABEL: lds_atomic_dec_ret_i32:33; VI: ; %bb.0:34; VI-NEXT: s_load_dword s2, s[4:5], 0x2c35; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2436; VI-NEXT: v_mov_b32_e32 v0, 4237; VI-NEXT: s_mov_b32 m0, -138; VI-NEXT: s_mov_b32 s3, 0xf00039; VI-NEXT: s_waitcnt lgkmcnt(0)40; VI-NEXT: v_mov_b32_e32 v1, s241; VI-NEXT: ds_dec_rtn_u32 v0, v1, v042; VI-NEXT: s_waitcnt lgkmcnt(0)43; VI-NEXT: s_mov_b32 s2, -144; VI-NEXT: buffer_store_dword v0, off, s[0:3], 045; VI-NEXT: s_endpgm46;47; GFX9-LABEL: lds_atomic_dec_ret_i32:48; GFX9: ; %bb.0:49; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c50; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2451; GFX9-NEXT: v_mov_b32_e32 v0, 4252; GFX9-NEXT: s_waitcnt lgkmcnt(0)53; GFX9-NEXT: v_mov_b32_e32 v1, s254; GFX9-NEXT: ds_dec_rtn_u32 v0, v1, v055; GFX9-NEXT: s_waitcnt lgkmcnt(0)56; GFX9-NEXT: v_mov_b32_e32 v1, 057; GFX9-NEXT: global_store_dword v1, v0, s[0:1]58; GFX9-NEXT: s_endpgm59 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 0, i32 0, i1 false)60 store i32 %result, ptr addrspace(1) %out61 ret void62}63 64define amdgpu_kernel void @lds_atomic_dec_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 {65; CI-LABEL: lds_atomic_dec_ret_i32_offset:66; CI: ; %bb.0:67; CI-NEXT: s_load_dword s2, s[4:5], 0xb68; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x969; CI-NEXT: v_mov_b32_e32 v0, 4270; CI-NEXT: s_mov_b32 m0, -171; CI-NEXT: s_mov_b32 s3, 0xf00072; CI-NEXT: s_waitcnt lgkmcnt(0)73; CI-NEXT: v_mov_b32_e32 v1, s274; CI-NEXT: ds_dec_rtn_u32 v0, v1, v0 offset:1675; CI-NEXT: s_waitcnt lgkmcnt(0)76; CI-NEXT: s_mov_b32 s2, -177; CI-NEXT: buffer_store_dword v0, off, s[0:3], 078; CI-NEXT: s_endpgm79;80; VI-LABEL: lds_atomic_dec_ret_i32_offset:81; VI: ; %bb.0:82; VI-NEXT: s_load_dword s2, s[4:5], 0x2c83; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2484; VI-NEXT: v_mov_b32_e32 v0, 4285; VI-NEXT: s_mov_b32 m0, -186; VI-NEXT: s_mov_b32 s3, 0xf00087; VI-NEXT: s_waitcnt lgkmcnt(0)88; VI-NEXT: v_mov_b32_e32 v1, s289; VI-NEXT: ds_dec_rtn_u32 v0, v1, v0 offset:1690; VI-NEXT: s_waitcnt lgkmcnt(0)91; VI-NEXT: s_mov_b32 s2, -192; VI-NEXT: buffer_store_dword v0, off, s[0:3], 093; VI-NEXT: s_endpgm94;95; GFX9-LABEL: lds_atomic_dec_ret_i32_offset:96; GFX9: ; %bb.0:97; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c98; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2499; GFX9-NEXT: v_mov_b32_e32 v0, 42100; GFX9-NEXT: s_waitcnt lgkmcnt(0)101; GFX9-NEXT: v_mov_b32_e32 v1, s2102; GFX9-NEXT: ds_dec_rtn_u32 v0, v1, v0 offset:16103; GFX9-NEXT: s_waitcnt lgkmcnt(0)104; GFX9-NEXT: v_mov_b32_e32 v1, 0105; GFX9-NEXT: global_store_dword v1, v0, s[0:1]106; GFX9-NEXT: s_endpgm107 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4108 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) %gep, i32 42, i32 0, i32 0, i1 false)109 store i32 %result, ptr addrspace(1) %out110 ret void111}112 113define amdgpu_kernel void @lds_atomic_dec_noret_i32(ptr addrspace(3) %ptr) nounwind {114; CI-LABEL: lds_atomic_dec_noret_i32:115; CI: ; %bb.0:116; CI-NEXT: s_load_dword s0, s[4:5], 0x9117; CI-NEXT: v_mov_b32_e32 v0, 42118; CI-NEXT: s_mov_b32 m0, -1119; CI-NEXT: s_waitcnt lgkmcnt(0)120; CI-NEXT: v_mov_b32_e32 v1, s0121; CI-NEXT: ds_dec_u32 v1, v0122; CI-NEXT: s_waitcnt lgkmcnt(0)123; CI-NEXT: s_endpgm124;125; VI-LABEL: lds_atomic_dec_noret_i32:126; VI: ; %bb.0:127; VI-NEXT: s_load_dword s0, s[4:5], 0x24128; VI-NEXT: v_mov_b32_e32 v0, 42129; VI-NEXT: s_mov_b32 m0, -1130; VI-NEXT: s_waitcnt lgkmcnt(0)131; VI-NEXT: v_mov_b32_e32 v1, s0132; VI-NEXT: ds_dec_u32 v1, v0133; VI-NEXT: s_waitcnt lgkmcnt(0)134; VI-NEXT: s_endpgm135;136; GFX9-LABEL: lds_atomic_dec_noret_i32:137; GFX9: ; %bb.0:138; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24139; GFX9-NEXT: v_mov_b32_e32 v0, 42140; GFX9-NEXT: s_waitcnt lgkmcnt(0)141; GFX9-NEXT: v_mov_b32_e32 v1, s0142; GFX9-NEXT: ds_dec_u32 v1, v0143; GFX9-NEXT: s_waitcnt lgkmcnt(0)144; GFX9-NEXT: s_endpgm145 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 0, i32 0, i1 false)146 ret void147}148 149define amdgpu_kernel void @lds_atomic_dec_noret_i32_offset(ptr addrspace(3) %ptr) nounwind {150; CI-LABEL: lds_atomic_dec_noret_i32_offset:151; CI: ; %bb.0:152; CI-NEXT: s_load_dword s0, s[4:5], 0x9153; CI-NEXT: v_mov_b32_e32 v0, 42154; CI-NEXT: s_mov_b32 m0, -1155; CI-NEXT: s_waitcnt lgkmcnt(0)156; CI-NEXT: v_mov_b32_e32 v1, s0157; CI-NEXT: ds_dec_u32 v1, v0 offset:16158; CI-NEXT: s_waitcnt lgkmcnt(0)159; CI-NEXT: s_endpgm160;161; VI-LABEL: lds_atomic_dec_noret_i32_offset:162; VI: ; %bb.0:163; VI-NEXT: s_load_dword s0, s[4:5], 0x24164; VI-NEXT: v_mov_b32_e32 v0, 42165; VI-NEXT: s_mov_b32 m0, -1166; VI-NEXT: s_waitcnt lgkmcnt(0)167; VI-NEXT: v_mov_b32_e32 v1, s0168; VI-NEXT: ds_dec_u32 v1, v0 offset:16169; VI-NEXT: s_waitcnt lgkmcnt(0)170; VI-NEXT: s_endpgm171;172; GFX9-LABEL: lds_atomic_dec_noret_i32_offset:173; GFX9: ; %bb.0:174; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24175; GFX9-NEXT: v_mov_b32_e32 v0, 42176; GFX9-NEXT: s_waitcnt lgkmcnt(0)177; GFX9-NEXT: v_mov_b32_e32 v1, s0178; GFX9-NEXT: ds_dec_u32 v1, v0 offset:16179; GFX9-NEXT: s_waitcnt lgkmcnt(0)180; GFX9-NEXT: s_endpgm181 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4182 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) %gep, i32 42, i32 0, i32 0, i1 false)183 ret void184}185 186define amdgpu_kernel void @global_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 {187; CI-LABEL: global_atomic_dec_ret_i32:188; CI: ; %bb.0:189; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9190; CI-NEXT: s_mov_b32 s7, 0xf000191; CI-NEXT: s_mov_b32 s6, -1192; CI-NEXT: s_mov_b32 s10, s6193; CI-NEXT: s_mov_b32 s11, s7194; CI-NEXT: s_waitcnt lgkmcnt(0)195; CI-NEXT: s_mov_b32 s8, s2196; CI-NEXT: s_mov_b32 s9, s3197; CI-NEXT: v_mov_b32_e32 v0, 42198; CI-NEXT: buffer_atomic_dec v0, off, s[8:11], 0 glc199; CI-NEXT: s_waitcnt vmcnt(0)200; CI-NEXT: buffer_wbinvl1_vol201; CI-NEXT: s_mov_b32 s4, s0202; CI-NEXT: s_mov_b32 s5, s1203; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0204; CI-NEXT: s_endpgm205;206; VI-LABEL: global_atomic_dec_ret_i32:207; VI: ; %bb.0:208; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24209; VI-NEXT: s_mov_b32 s7, 0xf000210; VI-NEXT: s_mov_b32 s6, -1211; VI-NEXT: s_mov_b32 s10, s6212; VI-NEXT: s_mov_b32 s11, s7213; VI-NEXT: s_waitcnt lgkmcnt(0)214; VI-NEXT: s_mov_b32 s8, s2215; VI-NEXT: s_mov_b32 s9, s3216; VI-NEXT: v_mov_b32_e32 v0, 42217; VI-NEXT: buffer_atomic_dec v0, off, s[8:11], 0 glc218; VI-NEXT: s_waitcnt vmcnt(0)219; VI-NEXT: buffer_wbinvl1_vol220; VI-NEXT: s_mov_b32 s4, s0221; VI-NEXT: s_mov_b32 s5, s1222; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0223; VI-NEXT: s_endpgm224;225; GFX9-LABEL: global_atomic_dec_ret_i32:226; GFX9: ; %bb.0:227; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24228; GFX9-NEXT: v_mov_b32_e32 v0, 0229; GFX9-NEXT: v_mov_b32_e32 v1, 42230; GFX9-NEXT: s_waitcnt lgkmcnt(0)231; GFX9-NEXT: global_atomic_dec v1, v0, v1, s[2:3] glc232; GFX9-NEXT: s_waitcnt vmcnt(0)233; GFX9-NEXT: buffer_wbinvl1_vol234; GFX9-NEXT: global_store_dword v0, v1, s[0:1]235; GFX9-NEXT: s_endpgm236 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %ptr, i32 42, i32 0, i32 0, i1 false)237 store i32 %result, ptr addrspace(1) %out238 ret void239}240 241define amdgpu_kernel void @global_atomic_dec_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 {242; CI-LABEL: global_atomic_dec_ret_i32_offset:243; CI: ; %bb.0:244; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9245; CI-NEXT: s_mov_b32 s7, 0xf000246; CI-NEXT: s_mov_b32 s6, -1247; CI-NEXT: s_mov_b32 s10, s6248; CI-NEXT: s_mov_b32 s11, s7249; CI-NEXT: s_waitcnt lgkmcnt(0)250; CI-NEXT: s_mov_b32 s8, s2251; CI-NEXT: s_mov_b32 s9, s3252; CI-NEXT: v_mov_b32_e32 v0, 42253; CI-NEXT: buffer_atomic_dec v0, off, s[8:11], 0 offset:16 glc254; CI-NEXT: s_waitcnt vmcnt(0)255; CI-NEXT: buffer_wbinvl1_vol256; CI-NEXT: s_mov_b32 s4, s0257; CI-NEXT: s_mov_b32 s5, s1258; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0259; CI-NEXT: s_endpgm260;261; VI-LABEL: global_atomic_dec_ret_i32_offset:262; VI: ; %bb.0:263; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24264; VI-NEXT: s_mov_b32 s7, 0xf000265; VI-NEXT: s_mov_b32 s6, -1266; VI-NEXT: s_mov_b32 s10, s6267; VI-NEXT: s_mov_b32 s11, s7268; VI-NEXT: s_waitcnt lgkmcnt(0)269; VI-NEXT: s_mov_b32 s8, s2270; VI-NEXT: s_mov_b32 s9, s3271; VI-NEXT: v_mov_b32_e32 v0, 42272; VI-NEXT: buffer_atomic_dec v0, off, s[8:11], 0 offset:16 glc273; VI-NEXT: s_waitcnt vmcnt(0)274; VI-NEXT: buffer_wbinvl1_vol275; VI-NEXT: s_mov_b32 s4, s0276; VI-NEXT: s_mov_b32 s5, s1277; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0278; VI-NEXT: s_endpgm279;280; GFX9-LABEL: global_atomic_dec_ret_i32_offset:281; GFX9: ; %bb.0:282; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24283; GFX9-NEXT: v_mov_b32_e32 v0, 0284; GFX9-NEXT: v_mov_b32_e32 v1, 42285; GFX9-NEXT: s_waitcnt lgkmcnt(0)286; GFX9-NEXT: global_atomic_dec v1, v0, v1, s[2:3] offset:16 glc287; GFX9-NEXT: s_waitcnt vmcnt(0)288; GFX9-NEXT: buffer_wbinvl1_vol289; GFX9-NEXT: global_store_dword v0, v1, s[0:1]290; GFX9-NEXT: s_endpgm291 %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4292 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %gep, i32 42, i32 0, i32 0, i1 false)293 store i32 %result, ptr addrspace(1) %out294 ret void295}296 297define amdgpu_kernel void @global_atomic_dec_noret_i32(ptr addrspace(1) %ptr) nounwind {298; CI-LABEL: global_atomic_dec_noret_i32:299; CI: ; %bb.0:300; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9301; CI-NEXT: s_mov_b32 s3, 0xf000302; CI-NEXT: s_mov_b32 s2, -1303; CI-NEXT: v_mov_b32_e32 v0, 42304; CI-NEXT: s_waitcnt lgkmcnt(0)305; CI-NEXT: buffer_atomic_dec v0, off, s[0:3], 0306; CI-NEXT: s_waitcnt vmcnt(0)307; CI-NEXT: buffer_wbinvl1_vol308; CI-NEXT: s_endpgm309;310; VI-LABEL: global_atomic_dec_noret_i32:311; VI: ; %bb.0:312; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24313; VI-NEXT: s_mov_b32 s3, 0xf000314; VI-NEXT: s_mov_b32 s2, -1315; VI-NEXT: v_mov_b32_e32 v0, 42316; VI-NEXT: s_waitcnt lgkmcnt(0)317; VI-NEXT: buffer_atomic_dec v0, off, s[0:3], 0318; VI-NEXT: s_waitcnt vmcnt(0)319; VI-NEXT: buffer_wbinvl1_vol320; VI-NEXT: s_endpgm321;322; GFX9-LABEL: global_atomic_dec_noret_i32:323; GFX9: ; %bb.0:324; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24325; GFX9-NEXT: v_mov_b32_e32 v0, 0326; GFX9-NEXT: v_mov_b32_e32 v1, 42327; GFX9-NEXT: s_waitcnt lgkmcnt(0)328; GFX9-NEXT: global_atomic_dec v0, v1, s[0:1]329; GFX9-NEXT: s_waitcnt vmcnt(0)330; GFX9-NEXT: buffer_wbinvl1_vol331; GFX9-NEXT: s_endpgm332 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %ptr, i32 42, i32 0, i32 0, i1 false)333 ret void334}335 336define amdgpu_kernel void @global_atomic_dec_noret_i32_offset(ptr addrspace(1) %ptr) nounwind {337; CI-LABEL: global_atomic_dec_noret_i32_offset:338; CI: ; %bb.0:339; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9340; CI-NEXT: s_mov_b32 s3, 0xf000341; CI-NEXT: s_mov_b32 s2, -1342; CI-NEXT: v_mov_b32_e32 v0, 42343; CI-NEXT: s_waitcnt lgkmcnt(0)344; CI-NEXT: buffer_atomic_dec v0, off, s[0:3], 0 offset:16345; CI-NEXT: s_waitcnt vmcnt(0)346; CI-NEXT: buffer_wbinvl1_vol347; CI-NEXT: s_endpgm348;349; VI-LABEL: global_atomic_dec_noret_i32_offset:350; VI: ; %bb.0:351; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24352; VI-NEXT: s_mov_b32 s3, 0xf000353; VI-NEXT: s_mov_b32 s2, -1354; VI-NEXT: v_mov_b32_e32 v0, 42355; VI-NEXT: s_waitcnt lgkmcnt(0)356; VI-NEXT: buffer_atomic_dec v0, off, s[0:3], 0 offset:16357; VI-NEXT: s_waitcnt vmcnt(0)358; VI-NEXT: buffer_wbinvl1_vol359; VI-NEXT: s_endpgm360;361; GFX9-LABEL: global_atomic_dec_noret_i32_offset:362; GFX9: ; %bb.0:363; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24364; GFX9-NEXT: v_mov_b32_e32 v0, 0365; GFX9-NEXT: v_mov_b32_e32 v1, 42366; GFX9-NEXT: s_waitcnt lgkmcnt(0)367; GFX9-NEXT: global_atomic_dec v0, v1, s[0:1] offset:16368; GFX9-NEXT: s_waitcnt vmcnt(0)369; GFX9-NEXT: buffer_wbinvl1_vol370; GFX9-NEXT: s_endpgm371 %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4372 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %gep, i32 42, i32 0, i32 0, i1 false)373 ret void374}375 376define amdgpu_kernel void @global_atomic_dec_ret_i32_offset_addr64(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 {377; CI-LABEL: global_atomic_dec_ret_i32_offset_addr64:378; CI: ; %bb.0:379; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9380; CI-NEXT: s_mov_b32 s7, 0xf000381; CI-NEXT: s_mov_b32 s6, 0382; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0383; CI-NEXT: v_mov_b32_e32 v1, 0384; CI-NEXT: s_waitcnt lgkmcnt(0)385; CI-NEXT: s_mov_b64 s[8:9], s[2:3]386; CI-NEXT: s_mov_b64 s[10:11], s[6:7]387; CI-NEXT: v_mov_b32_e32 v2, 42388; CI-NEXT: buffer_atomic_dec v2, v[0:1], s[8:11], 0 addr64 offset:20 glc389; CI-NEXT: s_waitcnt vmcnt(0)390; CI-NEXT: buffer_wbinvl1_vol391; CI-NEXT: s_mov_b64 s[4:5], s[0:1]392; CI-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64393; CI-NEXT: s_endpgm394;395; VI-LABEL: global_atomic_dec_ret_i32_offset_addr64:396; VI: ; %bb.0:397; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24398; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0399; VI-NEXT: s_waitcnt lgkmcnt(0)400; VI-NEXT: v_mov_b32_e32 v0, s3401; VI-NEXT: v_add_u32_e32 v1, vcc, s2, v2402; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v0, vcc403; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v1404; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc405; VI-NEXT: v_mov_b32_e32 v3, 42406; VI-NEXT: flat_atomic_dec v3, v[0:1], v3 glc407; VI-NEXT: s_waitcnt vmcnt(0)408; VI-NEXT: buffer_wbinvl1_vol409; VI-NEXT: v_mov_b32_e32 v1, s1410; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2411; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc412; VI-NEXT: flat_store_dword v[0:1], v3413; VI-NEXT: s_endpgm414;415; GFX9-LABEL: global_atomic_dec_ret_i32_offset_addr64:416; GFX9: ; %bb.0:417; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24418; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0419; GFX9-NEXT: v_mov_b32_e32 v1, 42420; GFX9-NEXT: s_waitcnt lgkmcnt(0)421; GFX9-NEXT: global_atomic_dec v1, v0, v1, s[2:3] offset:20 glc422; GFX9-NEXT: s_waitcnt vmcnt(0)423; GFX9-NEXT: buffer_wbinvl1_vol424; GFX9-NEXT: global_store_dword v0, v1, s[0:1]425; GFX9-NEXT: s_endpgm426 %id = call i32 @llvm.amdgcn.workitem.id.x()427 %gep.tid = getelementptr i32, ptr addrspace(1) %ptr, i32 %id428 %out.gep = getelementptr i32, ptr addrspace(1) %out, i32 %id429 %gep = getelementptr i32, ptr addrspace(1) %gep.tid, i32 5430 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %gep, i32 42, i32 0, i32 0, i1 false)431 store i32 %result, ptr addrspace(1) %out.gep432 ret void433}434 435define amdgpu_kernel void @global_atomic_dec_noret_i32_offset_addr64(ptr addrspace(1) %ptr) #0 {436; CI-LABEL: global_atomic_dec_noret_i32_offset_addr64:437; CI: ; %bb.0:438; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9439; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0440; CI-NEXT: v_mov_b32_e32 v1, 0441; CI-NEXT: s_mov_b32 s3, 0xf000442; CI-NEXT: s_mov_b32 s2, 0443; CI-NEXT: v_mov_b32_e32 v2, 42444; CI-NEXT: s_waitcnt lgkmcnt(0)445; CI-NEXT: buffer_atomic_dec v2, v[0:1], s[0:3], 0 addr64 offset:20446; CI-NEXT: s_waitcnt vmcnt(0)447; CI-NEXT: buffer_wbinvl1_vol448; CI-NEXT: s_endpgm449;450; VI-LABEL: global_atomic_dec_noret_i32_offset_addr64:451; VI: ; %bb.0:452; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24453; VI-NEXT: v_lshlrev_b32_e32 v0, 2, v0454; VI-NEXT: v_mov_b32_e32 v2, 42455; VI-NEXT: s_waitcnt lgkmcnt(0)456; VI-NEXT: v_mov_b32_e32 v1, s1457; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v0458; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc459; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v0460; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc461; VI-NEXT: flat_atomic_dec v[0:1], v2462; VI-NEXT: s_waitcnt vmcnt(0)463; VI-NEXT: buffer_wbinvl1_vol464; VI-NEXT: s_endpgm465;466; GFX9-LABEL: global_atomic_dec_noret_i32_offset_addr64:467; GFX9: ; %bb.0:468; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24469; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0470; GFX9-NEXT: v_mov_b32_e32 v1, 42471; GFX9-NEXT: s_waitcnt lgkmcnt(0)472; GFX9-NEXT: global_atomic_dec v0, v1, s[0:1] offset:20473; GFX9-NEXT: s_waitcnt vmcnt(0)474; GFX9-NEXT: buffer_wbinvl1_vol475; GFX9-NEXT: s_endpgm476 %id = call i32 @llvm.amdgcn.workitem.id.x()477 %gep.tid = getelementptr i32, ptr addrspace(1) %ptr, i32 %id478 %gep = getelementptr i32, ptr addrspace(1) %gep.tid, i32 5479 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %gep, i32 42, i32 0, i32 0, i1 false)480 ret void481}482 483define amdgpu_kernel void @flat_atomic_dec_ret_i32(ptr %out, ptr %ptr) #0 {484; CI-LABEL: flat_atomic_dec_ret_i32:485; CI: ; %bb.0:486; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9487; CI-NEXT: v_mov_b32_e32 v2, 42488; CI-NEXT: s_waitcnt lgkmcnt(0)489; CI-NEXT: v_mov_b32_e32 v0, s2490; CI-NEXT: v_mov_b32_e32 v1, s3491; CI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc492; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)493; CI-NEXT: buffer_wbinvl1_vol494; CI-NEXT: v_mov_b32_e32 v0, s0495; CI-NEXT: v_mov_b32_e32 v1, s1496; CI-NEXT: flat_store_dword v[0:1], v2497; CI-NEXT: s_endpgm498;499; VI-LABEL: flat_atomic_dec_ret_i32:500; VI: ; %bb.0:501; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24502; VI-NEXT: v_mov_b32_e32 v2, 42503; VI-NEXT: s_waitcnt lgkmcnt(0)504; VI-NEXT: v_mov_b32_e32 v0, s2505; VI-NEXT: v_mov_b32_e32 v1, s3506; VI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc507; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)508; VI-NEXT: buffer_wbinvl1_vol509; VI-NEXT: v_mov_b32_e32 v0, s0510; VI-NEXT: v_mov_b32_e32 v1, s1511; VI-NEXT: flat_store_dword v[0:1], v2512; VI-NEXT: s_endpgm513;514; GFX9-LABEL: flat_atomic_dec_ret_i32:515; GFX9: ; %bb.0:516; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24517; GFX9-NEXT: v_mov_b32_e32 v2, 42518; GFX9-NEXT: s_waitcnt lgkmcnt(0)519; GFX9-NEXT: v_mov_b32_e32 v0, s2520; GFX9-NEXT: v_mov_b32_e32 v1, s3521; GFX9-NEXT: flat_atomic_dec v2, v[0:1], v2 glc522; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)523; GFX9-NEXT: buffer_wbinvl1_vol524; GFX9-NEXT: v_mov_b32_e32 v0, s0525; GFX9-NEXT: v_mov_b32_e32 v1, s1526; GFX9-NEXT: flat_store_dword v[0:1], v2527; GFX9-NEXT: s_endpgm528 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %ptr, i32 42, i32 0, i32 0, i1 false)529 store i32 %result, ptr %out530 ret void531}532 533define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset(ptr %out, ptr %ptr) #0 {534; CI-LABEL: flat_atomic_dec_ret_i32_offset:535; CI: ; %bb.0:536; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9537; CI-NEXT: v_mov_b32_e32 v2, 42538; CI-NEXT: s_waitcnt lgkmcnt(0)539; CI-NEXT: s_add_u32 s2, s2, 16540; CI-NEXT: s_addc_u32 s3, s3, 0541; CI-NEXT: v_mov_b32_e32 v0, s2542; CI-NEXT: v_mov_b32_e32 v1, s3543; CI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc544; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)545; CI-NEXT: buffer_wbinvl1_vol546; CI-NEXT: v_mov_b32_e32 v0, s0547; CI-NEXT: v_mov_b32_e32 v1, s1548; CI-NEXT: flat_store_dword v[0:1], v2549; CI-NEXT: s_endpgm550;551; VI-LABEL: flat_atomic_dec_ret_i32_offset:552; VI: ; %bb.0:553; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24554; VI-NEXT: v_mov_b32_e32 v2, 42555; VI-NEXT: s_waitcnt lgkmcnt(0)556; VI-NEXT: s_add_u32 s2, s2, 16557; VI-NEXT: s_addc_u32 s3, s3, 0558; VI-NEXT: v_mov_b32_e32 v0, s2559; VI-NEXT: v_mov_b32_e32 v1, s3560; VI-NEXT: flat_atomic_dec v2, v[0:1], v2 glc561; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)562; VI-NEXT: buffer_wbinvl1_vol563; VI-NEXT: v_mov_b32_e32 v0, s0564; VI-NEXT: v_mov_b32_e32 v1, s1565; VI-NEXT: flat_store_dword v[0:1], v2566; VI-NEXT: s_endpgm567;568; GFX9-LABEL: flat_atomic_dec_ret_i32_offset:569; GFX9: ; %bb.0:570; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24571; GFX9-NEXT: v_mov_b32_e32 v2, 42572; GFX9-NEXT: s_waitcnt lgkmcnt(0)573; GFX9-NEXT: v_mov_b32_e32 v0, s2574; GFX9-NEXT: v_mov_b32_e32 v1, s3575; GFX9-NEXT: flat_atomic_dec v2, v[0:1], v2 offset:16 glc576; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)577; GFX9-NEXT: buffer_wbinvl1_vol578; GFX9-NEXT: v_mov_b32_e32 v0, s0579; GFX9-NEXT: v_mov_b32_e32 v1, s1580; GFX9-NEXT: flat_store_dword v[0:1], v2581; GFX9-NEXT: s_endpgm582 %gep = getelementptr inbounds i32, ptr %ptr, i32 4583 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %gep, i32 42, i32 0, i32 0, i1 false)584 store i32 %result, ptr %out585 ret void586}587 588define amdgpu_kernel void @flat_atomic_dec_noret_i32(ptr %ptr) nounwind {589; CI-LABEL: flat_atomic_dec_noret_i32:590; CI: ; %bb.0:591; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9592; CI-NEXT: v_mov_b32_e32 v2, 42593; CI-NEXT: s_waitcnt lgkmcnt(0)594; CI-NEXT: v_mov_b32_e32 v0, s0595; CI-NEXT: v_mov_b32_e32 v1, s1596; CI-NEXT: flat_atomic_dec v[0:1], v2597; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)598; CI-NEXT: buffer_wbinvl1_vol599; CI-NEXT: s_endpgm600;601; VI-LABEL: flat_atomic_dec_noret_i32:602; VI: ; %bb.0:603; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24604; VI-NEXT: v_mov_b32_e32 v2, 42605; VI-NEXT: s_waitcnt lgkmcnt(0)606; VI-NEXT: v_mov_b32_e32 v0, s0607; VI-NEXT: v_mov_b32_e32 v1, s1608; VI-NEXT: flat_atomic_dec v[0:1], v2609; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)610; VI-NEXT: buffer_wbinvl1_vol611; VI-NEXT: s_endpgm612;613; GFX9-LABEL: flat_atomic_dec_noret_i32:614; GFX9: ; %bb.0:615; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24616; GFX9-NEXT: v_mov_b32_e32 v2, 42617; GFX9-NEXT: s_waitcnt lgkmcnt(0)618; GFX9-NEXT: v_mov_b32_e32 v0, s0619; GFX9-NEXT: v_mov_b32_e32 v1, s1620; GFX9-NEXT: flat_atomic_dec v[0:1], v2621; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)622; GFX9-NEXT: buffer_wbinvl1_vol623; GFX9-NEXT: s_endpgm624 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %ptr, i32 42, i32 0, i32 0, i1 false)625 ret void626}627 628define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset(ptr %ptr) nounwind {629; CI-LABEL: flat_atomic_dec_noret_i32_offset:630; CI: ; %bb.0:631; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9632; CI-NEXT: v_mov_b32_e32 v2, 42633; CI-NEXT: s_waitcnt lgkmcnt(0)634; CI-NEXT: s_add_u32 s0, s0, 16635; CI-NEXT: s_addc_u32 s1, s1, 0636; CI-NEXT: v_mov_b32_e32 v0, s0637; CI-NEXT: v_mov_b32_e32 v1, s1638; CI-NEXT: flat_atomic_dec v[0:1], v2639; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)640; CI-NEXT: buffer_wbinvl1_vol641; CI-NEXT: s_endpgm642;643; VI-LABEL: flat_atomic_dec_noret_i32_offset:644; VI: ; %bb.0:645; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24646; VI-NEXT: v_mov_b32_e32 v2, 42647; VI-NEXT: s_waitcnt lgkmcnt(0)648; VI-NEXT: s_add_u32 s0, s0, 16649; VI-NEXT: s_addc_u32 s1, s1, 0650; VI-NEXT: v_mov_b32_e32 v0, s0651; VI-NEXT: v_mov_b32_e32 v1, s1652; VI-NEXT: flat_atomic_dec v[0:1], v2653; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)654; VI-NEXT: buffer_wbinvl1_vol655; VI-NEXT: s_endpgm656;657; GFX9-LABEL: flat_atomic_dec_noret_i32_offset:658; GFX9: ; %bb.0:659; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24660; GFX9-NEXT: v_mov_b32_e32 v2, 42661; GFX9-NEXT: s_waitcnt lgkmcnt(0)662; GFX9-NEXT: v_mov_b32_e32 v0, s0663; GFX9-NEXT: v_mov_b32_e32 v1, s1664; GFX9-NEXT: flat_atomic_dec v[0:1], v2 offset:16665; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)666; GFX9-NEXT: buffer_wbinvl1_vol667; GFX9-NEXT: s_endpgm668 %gep = getelementptr inbounds i32, ptr %ptr, i32 4669 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %gep, i32 42, i32 0, i32 0, i1 false)670 ret void671}672 673define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_addr64(ptr %out, ptr %ptr) #0 {674; CI-LABEL: flat_atomic_dec_ret_i32_offset_addr64:675; CI: ; %bb.0:676; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9677; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v0678; CI-NEXT: s_waitcnt lgkmcnt(0)679; CI-NEXT: v_mov_b32_e32 v0, s3680; CI-NEXT: v_add_i32_e32 v1, vcc, s2, v2681; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v0, vcc682; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v1683; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc684; CI-NEXT: v_mov_b32_e32 v3, 42685; CI-NEXT: flat_atomic_dec v3, v[0:1], v3 glc686; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)687; CI-NEXT: buffer_wbinvl1_vol688; CI-NEXT: v_mov_b32_e32 v1, s1689; CI-NEXT: v_add_i32_e32 v0, vcc, s0, v2690; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc691; CI-NEXT: flat_store_dword v[0:1], v3692; CI-NEXT: s_endpgm693;694; VI-LABEL: flat_atomic_dec_ret_i32_offset_addr64:695; VI: ; %bb.0:696; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24697; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0698; VI-NEXT: s_waitcnt lgkmcnt(0)699; VI-NEXT: v_mov_b32_e32 v0, s3700; VI-NEXT: v_add_u32_e32 v1, vcc, s2, v2701; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v0, vcc702; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v1703; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc704; VI-NEXT: v_mov_b32_e32 v3, 42705; VI-NEXT: flat_atomic_dec v3, v[0:1], v3 glc706; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)707; VI-NEXT: buffer_wbinvl1_vol708; VI-NEXT: v_mov_b32_e32 v1, s1709; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2710; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc711; VI-NEXT: flat_store_dword v[0:1], v3712; VI-NEXT: s_endpgm713;714; GFX9-LABEL: flat_atomic_dec_ret_i32_offset_addr64:715; GFX9: ; %bb.0:716; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24717; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0718; GFX9-NEXT: v_mov_b32_e32 v3, 42719; GFX9-NEXT: s_waitcnt lgkmcnt(0)720; GFX9-NEXT: v_mov_b32_e32 v1, s3721; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s2, v2722; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc723; GFX9-NEXT: flat_atomic_dec v3, v[0:1], v3 offset:20 glc724; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)725; GFX9-NEXT: buffer_wbinvl1_vol726; GFX9-NEXT: v_mov_b32_e32 v1, s1727; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v2728; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc729; GFX9-NEXT: flat_store_dword v[0:1], v3730; GFX9-NEXT: s_endpgm731 %id = call i32 @llvm.amdgcn.workitem.id.x()732 %gep.tid = getelementptr inbounds i32, ptr %ptr, i32 %id733 %out.gep = getelementptr inbounds i32, ptr %out, i32 %id734 %gep = getelementptr inbounds i32, ptr %gep.tid, i32 5735 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %gep, i32 42, i32 0, i32 0, i1 false)736 store i32 %result, ptr %out.gep737 ret void738}739 740define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_addr64(ptr %ptr) #0 {741; CI-LABEL: flat_atomic_dec_noret_i32_offset_addr64:742; CI: ; %bb.0:743; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9744; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0745; CI-NEXT: v_mov_b32_e32 v2, 42746; CI-NEXT: s_waitcnt lgkmcnt(0)747; CI-NEXT: v_mov_b32_e32 v1, s1748; CI-NEXT: v_add_i32_e32 v0, vcc, s0, v0749; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc750; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v0751; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc752; CI-NEXT: flat_atomic_dec v[0:1], v2753; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)754; CI-NEXT: buffer_wbinvl1_vol755; CI-NEXT: s_endpgm756;757; VI-LABEL: flat_atomic_dec_noret_i32_offset_addr64:758; VI: ; %bb.0:759; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24760; VI-NEXT: v_lshlrev_b32_e32 v0, 2, v0761; VI-NEXT: v_mov_b32_e32 v2, 42762; VI-NEXT: s_waitcnt lgkmcnt(0)763; VI-NEXT: v_mov_b32_e32 v1, s1764; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v0765; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc766; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v0767; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc768; VI-NEXT: flat_atomic_dec v[0:1], v2769; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)770; VI-NEXT: buffer_wbinvl1_vol771; VI-NEXT: s_endpgm772;773; GFX9-LABEL: flat_atomic_dec_noret_i32_offset_addr64:774; GFX9: ; %bb.0:775; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24776; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0777; GFX9-NEXT: v_mov_b32_e32 v2, 42778; GFX9-NEXT: s_waitcnt lgkmcnt(0)779; GFX9-NEXT: v_mov_b32_e32 v1, s1780; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0781; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc782; GFX9-NEXT: flat_atomic_dec v[0:1], v2 offset:20783; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)784; GFX9-NEXT: buffer_wbinvl1_vol785; GFX9-NEXT: s_endpgm786 %id = call i32 @llvm.amdgcn.workitem.id.x()787 %gep.tid = getelementptr inbounds i32, ptr %ptr, i32 %id788 %gep = getelementptr inbounds i32, ptr %gep.tid, i32 5789 %result = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %gep, i32 42, i32 0, i32 0, i1 false)790 ret void791}792 793define amdgpu_kernel void @flat_atomic_dec_ret_i64(ptr %out, ptr %ptr) #0 {794; CI-LABEL: flat_atomic_dec_ret_i64:795; CI: ; %bb.0:796; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9797; CI-NEXT: v_mov_b32_e32 v0, 42798; CI-NEXT: v_mov_b32_e32 v1, 0799; CI-NEXT: s_waitcnt lgkmcnt(0)800; CI-NEXT: v_mov_b32_e32 v2, s2801; CI-NEXT: v_mov_b32_e32 v3, s3802; CI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc803; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)804; CI-NEXT: buffer_wbinvl1_vol805; CI-NEXT: v_mov_b32_e32 v2, s0806; CI-NEXT: v_mov_b32_e32 v3, s1807; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]808; CI-NEXT: s_endpgm809;810; VI-LABEL: flat_atomic_dec_ret_i64:811; VI: ; %bb.0:812; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24813; VI-NEXT: v_mov_b32_e32 v0, 42814; VI-NEXT: v_mov_b32_e32 v1, 0815; VI-NEXT: s_waitcnt lgkmcnt(0)816; VI-NEXT: v_mov_b32_e32 v2, s2817; VI-NEXT: v_mov_b32_e32 v3, s3818; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc819; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)820; VI-NEXT: buffer_wbinvl1_vol821; VI-NEXT: v_mov_b32_e32 v2, s0822; VI-NEXT: v_mov_b32_e32 v3, s1823; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]824; VI-NEXT: s_endpgm825;826; GFX9-LABEL: flat_atomic_dec_ret_i64:827; GFX9: ; %bb.0:828; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24829; GFX9-NEXT: v_mov_b32_e32 v0, 42830; GFX9-NEXT: v_mov_b32_e32 v1, 0831; GFX9-NEXT: s_waitcnt lgkmcnt(0)832; GFX9-NEXT: v_mov_b32_e32 v2, s2833; GFX9-NEXT: v_mov_b32_e32 v3, s3834; GFX9-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc835; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)836; GFX9-NEXT: buffer_wbinvl1_vol837; GFX9-NEXT: v_mov_b32_e32 v2, s0838; GFX9-NEXT: v_mov_b32_e32 v3, s1839; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]840; GFX9-NEXT: s_endpgm841 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr %ptr, i64 42, i32 0, i32 0, i1 false)842 store i64 %result, ptr %out843 ret void844}845 846define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset(ptr %out, ptr %ptr) #0 {847; CI-LABEL: flat_atomic_dec_ret_i64_offset:848; CI: ; %bb.0:849; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9850; CI-NEXT: v_mov_b32_e32 v0, 42851; CI-NEXT: v_mov_b32_e32 v1, 0852; CI-NEXT: s_waitcnt lgkmcnt(0)853; CI-NEXT: s_add_u32 s2, s2, 32854; CI-NEXT: s_addc_u32 s3, s3, 0855; CI-NEXT: v_mov_b32_e32 v2, s2856; CI-NEXT: v_mov_b32_e32 v3, s3857; CI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc858; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)859; CI-NEXT: buffer_wbinvl1_vol860; CI-NEXT: v_mov_b32_e32 v2, s0861; CI-NEXT: v_mov_b32_e32 v3, s1862; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]863; CI-NEXT: s_endpgm864;865; VI-LABEL: flat_atomic_dec_ret_i64_offset:866; VI: ; %bb.0:867; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24868; VI-NEXT: v_mov_b32_e32 v0, 42869; VI-NEXT: v_mov_b32_e32 v1, 0870; VI-NEXT: s_waitcnt lgkmcnt(0)871; VI-NEXT: s_add_u32 s2, s2, 32872; VI-NEXT: s_addc_u32 s3, s3, 0873; VI-NEXT: v_mov_b32_e32 v2, s2874; VI-NEXT: v_mov_b32_e32 v3, s3875; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc876; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)877; VI-NEXT: buffer_wbinvl1_vol878; VI-NEXT: v_mov_b32_e32 v2, s0879; VI-NEXT: v_mov_b32_e32 v3, s1880; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]881; VI-NEXT: s_endpgm882;883; GFX9-LABEL: flat_atomic_dec_ret_i64_offset:884; GFX9: ; %bb.0:885; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24886; GFX9-NEXT: v_mov_b32_e32 v0, 42887; GFX9-NEXT: v_mov_b32_e32 v1, 0888; GFX9-NEXT: s_waitcnt lgkmcnt(0)889; GFX9-NEXT: v_mov_b32_e32 v2, s2890; GFX9-NEXT: v_mov_b32_e32 v3, s3891; GFX9-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] offset:32 glc892; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)893; GFX9-NEXT: buffer_wbinvl1_vol894; GFX9-NEXT: v_mov_b32_e32 v2, s0895; GFX9-NEXT: v_mov_b32_e32 v3, s1896; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]897; GFX9-NEXT: s_endpgm898 %gep = getelementptr inbounds i64, ptr %ptr, i32 4899 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr %gep, i64 42, i32 0, i32 0, i1 false)900 store i64 %result, ptr %out901 ret void902}903 904define amdgpu_kernel void @flat_atomic_dec_noret_i64(ptr %ptr) nounwind {905; CI-LABEL: flat_atomic_dec_noret_i64:906; CI: ; %bb.0:907; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9908; CI-NEXT: v_mov_b32_e32 v0, 42909; CI-NEXT: v_mov_b32_e32 v1, 0910; CI-NEXT: s_waitcnt lgkmcnt(0)911; CI-NEXT: v_mov_b32_e32 v3, s1912; CI-NEXT: v_mov_b32_e32 v2, s0913; CI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]914; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)915; CI-NEXT: buffer_wbinvl1_vol916; CI-NEXT: s_endpgm917;918; VI-LABEL: flat_atomic_dec_noret_i64:919; VI: ; %bb.0:920; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24921; VI-NEXT: v_mov_b32_e32 v0, 42922; VI-NEXT: v_mov_b32_e32 v1, 0923; VI-NEXT: s_waitcnt lgkmcnt(0)924; VI-NEXT: v_mov_b32_e32 v3, s1925; VI-NEXT: v_mov_b32_e32 v2, s0926; VI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]927; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)928; VI-NEXT: buffer_wbinvl1_vol929; VI-NEXT: s_endpgm930;931; GFX9-LABEL: flat_atomic_dec_noret_i64:932; GFX9: ; %bb.0:933; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24934; GFX9-NEXT: v_mov_b32_e32 v0, 42935; GFX9-NEXT: v_mov_b32_e32 v1, 0936; GFX9-NEXT: s_waitcnt lgkmcnt(0)937; GFX9-NEXT: v_mov_b32_e32 v3, s1938; GFX9-NEXT: v_mov_b32_e32 v2, s0939; GFX9-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]940; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)941; GFX9-NEXT: buffer_wbinvl1_vol942; GFX9-NEXT: s_endpgm943 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr %ptr, i64 42, i32 0, i32 0, i1 false)944 ret void945}946 947define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset(ptr %ptr) nounwind {948; CI-LABEL: flat_atomic_dec_noret_i64_offset:949; CI: ; %bb.0:950; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9951; CI-NEXT: v_mov_b32_e32 v0, 42952; CI-NEXT: v_mov_b32_e32 v1, 0953; CI-NEXT: s_waitcnt lgkmcnt(0)954; CI-NEXT: s_add_u32 s0, s0, 32955; CI-NEXT: s_addc_u32 s1, s1, 0956; CI-NEXT: v_mov_b32_e32 v3, s1957; CI-NEXT: v_mov_b32_e32 v2, s0958; CI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]959; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)960; CI-NEXT: buffer_wbinvl1_vol961; CI-NEXT: s_endpgm962;963; VI-LABEL: flat_atomic_dec_noret_i64_offset:964; VI: ; %bb.0:965; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24966; VI-NEXT: v_mov_b32_e32 v0, 42967; VI-NEXT: v_mov_b32_e32 v1, 0968; VI-NEXT: s_waitcnt lgkmcnt(0)969; VI-NEXT: s_add_u32 s0, s0, 32970; VI-NEXT: s_addc_u32 s1, s1, 0971; VI-NEXT: v_mov_b32_e32 v3, s1972; VI-NEXT: v_mov_b32_e32 v2, s0973; VI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]974; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)975; VI-NEXT: buffer_wbinvl1_vol976; VI-NEXT: s_endpgm977;978; GFX9-LABEL: flat_atomic_dec_noret_i64_offset:979; GFX9: ; %bb.0:980; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24981; GFX9-NEXT: v_mov_b32_e32 v0, 42982; GFX9-NEXT: v_mov_b32_e32 v1, 0983; GFX9-NEXT: s_waitcnt lgkmcnt(0)984; GFX9-NEXT: v_mov_b32_e32 v3, s1985; GFX9-NEXT: v_mov_b32_e32 v2, s0986; GFX9-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1] offset:32987; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)988; GFX9-NEXT: buffer_wbinvl1_vol989; GFX9-NEXT: s_endpgm990 %gep = getelementptr inbounds i64, ptr %ptr, i32 4991 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr %gep, i64 42, i32 0, i32 0, i1 false)992 ret void993}994 995define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset_addr64(ptr %out, ptr %ptr) #0 {996; CI-LABEL: flat_atomic_dec_ret_i64_offset_addr64:997; CI: ; %bb.0:998; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9999; CI-NEXT: v_lshlrev_b32_e32 v4, 3, v01000; CI-NEXT: s_waitcnt lgkmcnt(0)1001; CI-NEXT: v_mov_b32_e32 v0, s31002; CI-NEXT: v_add_i32_e32 v2, vcc, s2, v41003; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v0, vcc1004; CI-NEXT: v_mov_b32_e32 v0, 421005; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v21006; CI-NEXT: v_mov_b32_e32 v1, 01007; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc1008; CI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc1009; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1010; CI-NEXT: buffer_wbinvl1_vol1011; CI-NEXT: v_mov_b32_e32 v3, s11012; CI-NEXT: v_add_i32_e32 v2, vcc, s0, v41013; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc1014; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1015; CI-NEXT: s_endpgm1016;1017; VI-LABEL: flat_atomic_dec_ret_i64_offset_addr64:1018; VI: ; %bb.0:1019; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241020; VI-NEXT: v_lshlrev_b32_e32 v4, 3, v01021; VI-NEXT: s_waitcnt lgkmcnt(0)1022; VI-NEXT: v_mov_b32_e32 v0, s31023; VI-NEXT: v_add_u32_e32 v2, vcc, s2, v41024; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v0, vcc1025; VI-NEXT: v_mov_b32_e32 v0, 421026; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v21027; VI-NEXT: v_mov_b32_e32 v1, 01028; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc1029; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc1030; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1031; VI-NEXT: buffer_wbinvl1_vol1032; VI-NEXT: v_mov_b32_e32 v3, s11033; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v41034; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc1035; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1036; VI-NEXT: s_endpgm1037;1038; GFX9-LABEL: flat_atomic_dec_ret_i64_offset_addr64:1039; GFX9: ; %bb.0:1040; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241041; GFX9-NEXT: v_lshlrev_b32_e32 v5, 3, v01042; GFX9-NEXT: v_mov_b32_e32 v1, 421043; GFX9-NEXT: v_mov_b32_e32 v2, 01044; GFX9-NEXT: s_waitcnt lgkmcnt(0)1045; GFX9-NEXT: v_mov_b32_e32 v0, s31046; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, s2, v51047; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v0, vcc1048; GFX9-NEXT: flat_atomic_dec_x2 v[0:1], v[3:4], v[1:2] offset:40 glc1049; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1050; GFX9-NEXT: buffer_wbinvl1_vol1051; GFX9-NEXT: v_mov_b32_e32 v3, s11052; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, s0, v51053; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc1054; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1055; GFX9-NEXT: s_endpgm1056 %id = call i32 @llvm.amdgcn.workitem.id.x()1057 %gep.tid = getelementptr inbounds i64, ptr %ptr, i32 %id1058 %out.gep = getelementptr inbounds i64, ptr %out, i32 %id1059 %gep = getelementptr inbounds i64, ptr %gep.tid, i32 51060 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr %gep, i64 42, i32 0, i32 0, i1 false)1061 store i64 %result, ptr %out.gep1062 ret void1063}1064 1065define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_addr64(ptr %ptr) #0 {1066; CI-LABEL: flat_atomic_dec_noret_i64_offset_addr64:1067; CI: ; %bb.0:1068; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x91069; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v01070; CI-NEXT: s_waitcnt lgkmcnt(0)1071; CI-NEXT: v_mov_b32_e32 v1, s11072; CI-NEXT: v_add_i32_e32 v2, vcc, s0, v01073; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc1074; CI-NEXT: v_mov_b32_e32 v0, 421075; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v21076; CI-NEXT: v_mov_b32_e32 v1, 01077; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc1078; CI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]1079; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1080; CI-NEXT: buffer_wbinvl1_vol1081; CI-NEXT: s_endpgm1082;1083; VI-LABEL: flat_atomic_dec_noret_i64_offset_addr64:1084; VI: ; %bb.0:1085; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241086; VI-NEXT: v_lshlrev_b32_e32 v0, 3, v01087; VI-NEXT: s_waitcnt lgkmcnt(0)1088; VI-NEXT: v_mov_b32_e32 v1, s11089; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v01090; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc1091; VI-NEXT: v_mov_b32_e32 v0, 421092; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v21093; VI-NEXT: v_mov_b32_e32 v1, 01094; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc1095; VI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]1096; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1097; VI-NEXT: buffer_wbinvl1_vol1098; VI-NEXT: s_endpgm1099;1100; GFX9-LABEL: flat_atomic_dec_noret_i64_offset_addr64:1101; GFX9: ; %bb.0:1102; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241103; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v01104; GFX9-NEXT: v_mov_b32_e32 v1, 421105; GFX9-NEXT: v_mov_b32_e32 v2, 01106; GFX9-NEXT: s_waitcnt lgkmcnt(0)1107; GFX9-NEXT: v_mov_b32_e32 v4, s11108; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, s0, v01109; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc1110; GFX9-NEXT: flat_atomic_dec_x2 v[3:4], v[1:2] offset:401111; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1112; GFX9-NEXT: buffer_wbinvl1_vol1113; GFX9-NEXT: s_endpgm1114 %id = call i32 @llvm.amdgcn.workitem.id.x()1115 %gep.tid = getelementptr inbounds i64, ptr %ptr, i32 %id1116 %gep = getelementptr inbounds i64, ptr %gep.tid, i32 51117 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr %gep, i64 42, i32 0, i32 0, i1 false)1118 ret void1119}1120 1121@lds0 = addrspace(3) global [512 x i32] poison1122 1123define amdgpu_kernel void @atomic_dec_shl_base_lds_0(ptr addrspace(1) %out, ptr addrspace(1) %add_use) #0 {1124; CI-LABEL: atomic_dec_shl_base_lds_0:1125; CI: ; %bb.0:1126; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91127; CI-NEXT: v_lshlrev_b32_e32 v1, 2, v01128; CI-NEXT: v_mov_b32_e32 v2, 91129; CI-NEXT: s_mov_b32 m0, -11130; CI-NEXT: s_waitcnt lgkmcnt(0)1131; CI-NEXT: ds_dec_rtn_u32 v1, v1, v2 offset:81132; CI-NEXT: s_waitcnt lgkmcnt(0)1133; CI-NEXT: s_mov_b32 s7, 0xf0001134; CI-NEXT: s_mov_b32 s6, -11135; CI-NEXT: s_mov_b32 s4, s01136; CI-NEXT: s_mov_b32 s5, s11137; CI-NEXT: s_mov_b32 s0, s21138; CI-NEXT: s_mov_b32 s1, s31139; CI-NEXT: s_mov_b32 s2, s61140; CI-NEXT: s_mov_b32 s3, s71141; CI-NEXT: v_add_i32_e32 v0, vcc, 2, v01142; CI-NEXT: buffer_store_dword v0, off, s[0:3], 01143; CI-NEXT: buffer_store_dword v1, off, s[4:7], 01144; CI-NEXT: s_endpgm1145;1146; VI-LABEL: atomic_dec_shl_base_lds_0:1147; VI: ; %bb.0:1148; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241149; VI-NEXT: v_lshlrev_b32_e32 v1, 2, v01150; VI-NEXT: v_mov_b32_e32 v2, 91151; VI-NEXT: s_mov_b32 m0, -11152; VI-NEXT: s_waitcnt lgkmcnt(0)1153; VI-NEXT: ds_dec_rtn_u32 v1, v1, v2 offset:81154; VI-NEXT: s_waitcnt lgkmcnt(0)1155; VI-NEXT: s_mov_b32 s7, 0xf0001156; VI-NEXT: s_mov_b32 s6, -11157; VI-NEXT: s_mov_b32 s4, s01158; VI-NEXT: s_mov_b32 s5, s11159; VI-NEXT: s_mov_b32 s0, s21160; VI-NEXT: s_mov_b32 s1, s31161; VI-NEXT: s_mov_b32 s2, s61162; VI-NEXT: s_mov_b32 s3, s71163; VI-NEXT: v_add_u32_e32 v0, vcc, 2, v01164; VI-NEXT: buffer_store_dword v0, off, s[0:3], 01165; VI-NEXT: buffer_store_dword v1, off, s[4:7], 01166; VI-NEXT: s_endpgm1167;1168; GFX9-LABEL: atomic_dec_shl_base_lds_0:1169; GFX9: ; %bb.0:1170; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241171; GFX9-NEXT: v_lshlrev_b32_e32 v1, 2, v01172; GFX9-NEXT: v_mov_b32_e32 v2, 91173; GFX9-NEXT: s_waitcnt lgkmcnt(0)1174; GFX9-NEXT: ds_dec_rtn_u32 v1, v1, v2 offset:81175; GFX9-NEXT: s_waitcnt lgkmcnt(0)1176; GFX9-NEXT: v_mov_b32_e32 v2, 01177; GFX9-NEXT: v_add_u32_e32 v0, 2, v01178; GFX9-NEXT: global_store_dword v2, v0, s[2:3]1179; GFX9-NEXT: global_store_dword v2, v1, s[0:1]1180; GFX9-NEXT: s_endpgm1181 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #11182 %idx.0 = add nsw i32 %tid.x, 21183 %arrayidx0 = getelementptr inbounds [512 x i32], ptr addrspace(3) @lds0, i32 0, i32 %idx.01184 %val0 = call i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) %arrayidx0, i32 9, i32 0, i32 0, i1 false)1185 store i32 %idx.0, ptr addrspace(1) %add_use1186 store i32 %val0, ptr addrspace(1) %out1187 ret void1188}1189 1190define amdgpu_kernel void @lds_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 {1191; CI-LABEL: lds_atomic_dec_ret_i64:1192; CI: ; %bb.0:1193; CI-NEXT: s_load_dword s2, s[4:5], 0xb1194; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x91195; CI-NEXT: v_mov_b32_e32 v0, 421196; CI-NEXT: v_mov_b32_e32 v1, 01197; CI-NEXT: s_mov_b32 m0, -11198; CI-NEXT: s_waitcnt lgkmcnt(0)1199; CI-NEXT: v_mov_b32_e32 v2, s21200; CI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]1201; CI-NEXT: s_waitcnt lgkmcnt(0)1202; CI-NEXT: s_mov_b32 s3, 0xf0001203; CI-NEXT: s_mov_b32 s2, -11204; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 01205; CI-NEXT: s_endpgm1206;1207; VI-LABEL: lds_atomic_dec_ret_i64:1208; VI: ; %bb.0:1209; VI-NEXT: s_load_dword s2, s[4:5], 0x2c1210; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241211; VI-NEXT: v_mov_b32_e32 v0, 421212; VI-NEXT: v_mov_b32_e32 v1, 01213; VI-NEXT: s_mov_b32 m0, -11214; VI-NEXT: s_waitcnt lgkmcnt(0)1215; VI-NEXT: v_mov_b32_e32 v2, s21216; VI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]1217; VI-NEXT: s_waitcnt lgkmcnt(0)1218; VI-NEXT: s_mov_b32 s3, 0xf0001219; VI-NEXT: s_mov_b32 s2, -11220; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 01221; VI-NEXT: s_endpgm1222;1223; GFX9-LABEL: lds_atomic_dec_ret_i64:1224; GFX9: ; %bb.0:1225; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c1226; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241227; GFX9-NEXT: v_mov_b32_e32 v0, 421228; GFX9-NEXT: v_mov_b32_e32 v1, 01229; GFX9-NEXT: s_waitcnt lgkmcnt(0)1230; GFX9-NEXT: v_mov_b32_e32 v2, s21231; GFX9-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]1232; GFX9-NEXT: s_waitcnt lgkmcnt(0)1233; GFX9-NEXT: v_mov_b32_e32 v2, 01234; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]1235; GFX9-NEXT: s_endpgm1236 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p3(ptr addrspace(3) %ptr, i64 42, i32 0, i32 0, i1 false)1237 store i64 %result, ptr addrspace(1) %out1238 ret void1239}1240 1241define amdgpu_kernel void @lds_atomic_dec_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 {1242; CI-LABEL: lds_atomic_dec_ret_i64_offset:1243; CI: ; %bb.0:1244; CI-NEXT: s_load_dword s2, s[4:5], 0xb1245; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x91246; CI-NEXT: v_mov_b32_e32 v0, 421247; CI-NEXT: v_mov_b32_e32 v1, 01248; CI-NEXT: s_mov_b32 m0, -11249; CI-NEXT: s_waitcnt lgkmcnt(0)1250; CI-NEXT: v_mov_b32_e32 v2, s21251; CI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:321252; CI-NEXT: s_waitcnt lgkmcnt(0)1253; CI-NEXT: s_mov_b32 s3, 0xf0001254; CI-NEXT: s_mov_b32 s2, -11255; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 01256; CI-NEXT: s_endpgm1257;1258; VI-LABEL: lds_atomic_dec_ret_i64_offset:1259; VI: ; %bb.0:1260; VI-NEXT: s_load_dword s2, s[4:5], 0x2c1261; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241262; VI-NEXT: v_mov_b32_e32 v0, 421263; VI-NEXT: v_mov_b32_e32 v1, 01264; VI-NEXT: s_mov_b32 m0, -11265; VI-NEXT: s_waitcnt lgkmcnt(0)1266; VI-NEXT: v_mov_b32_e32 v2, s21267; VI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:321268; VI-NEXT: s_waitcnt lgkmcnt(0)1269; VI-NEXT: s_mov_b32 s3, 0xf0001270; VI-NEXT: s_mov_b32 s2, -11271; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 01272; VI-NEXT: s_endpgm1273;1274; GFX9-LABEL: lds_atomic_dec_ret_i64_offset:1275; GFX9: ; %bb.0:1276; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c1277; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241278; GFX9-NEXT: v_mov_b32_e32 v0, 421279; GFX9-NEXT: v_mov_b32_e32 v1, 01280; GFX9-NEXT: s_waitcnt lgkmcnt(0)1281; GFX9-NEXT: v_mov_b32_e32 v2, s21282; GFX9-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:321283; GFX9-NEXT: s_waitcnt lgkmcnt(0)1284; GFX9-NEXT: v_mov_b32_e32 v2, 01285; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]1286; GFX9-NEXT: s_endpgm1287 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 41288 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p3(ptr addrspace(3) %gep, i64 42, i32 0, i32 0, i1 false)1289 store i64 %result, ptr addrspace(1) %out1290 ret void1291}1292 1293define amdgpu_kernel void @lds_atomic_dec_noret_i64(ptr addrspace(3) %ptr) nounwind {1294; CI-LABEL: lds_atomic_dec_noret_i64:1295; CI: ; %bb.0:1296; CI-NEXT: s_load_dword s0, s[4:5], 0x91297; CI-NEXT: v_mov_b32_e32 v0, 421298; CI-NEXT: v_mov_b32_e32 v1, 01299; CI-NEXT: s_mov_b32 m0, -11300; CI-NEXT: s_waitcnt lgkmcnt(0)1301; CI-NEXT: v_mov_b32_e32 v2, s01302; CI-NEXT: ds_dec_u64 v2, v[0:1]1303; CI-NEXT: s_waitcnt lgkmcnt(0)1304; CI-NEXT: s_endpgm1305;1306; VI-LABEL: lds_atomic_dec_noret_i64:1307; VI: ; %bb.0:1308; VI-NEXT: s_load_dword s0, s[4:5], 0x241309; VI-NEXT: v_mov_b32_e32 v0, 421310; VI-NEXT: v_mov_b32_e32 v1, 01311; VI-NEXT: s_mov_b32 m0, -11312; VI-NEXT: s_waitcnt lgkmcnt(0)1313; VI-NEXT: v_mov_b32_e32 v2, s01314; VI-NEXT: ds_dec_u64 v2, v[0:1]1315; VI-NEXT: s_waitcnt lgkmcnt(0)1316; VI-NEXT: s_endpgm1317;1318; GFX9-LABEL: lds_atomic_dec_noret_i64:1319; GFX9: ; %bb.0:1320; GFX9-NEXT: s_load_dword s0, s[4:5], 0x241321; GFX9-NEXT: v_mov_b32_e32 v0, 421322; GFX9-NEXT: v_mov_b32_e32 v1, 01323; GFX9-NEXT: s_waitcnt lgkmcnt(0)1324; GFX9-NEXT: v_mov_b32_e32 v2, s01325; GFX9-NEXT: ds_dec_u64 v2, v[0:1]1326; GFX9-NEXT: s_waitcnt lgkmcnt(0)1327; GFX9-NEXT: s_endpgm1328 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p3(ptr addrspace(3) %ptr, i64 42, i32 0, i32 0, i1 false)1329 ret void1330}1331 1332define amdgpu_kernel void @lds_atomic_dec_noret_i64_offset(ptr addrspace(3) %ptr) nounwind {1333; CI-LABEL: lds_atomic_dec_noret_i64_offset:1334; CI: ; %bb.0:1335; CI-NEXT: s_load_dword s0, s[4:5], 0x91336; CI-NEXT: v_mov_b32_e32 v0, 421337; CI-NEXT: v_mov_b32_e32 v1, 01338; CI-NEXT: s_mov_b32 m0, -11339; CI-NEXT: s_waitcnt lgkmcnt(0)1340; CI-NEXT: v_mov_b32_e32 v2, s01341; CI-NEXT: ds_dec_u64 v2, v[0:1] offset:321342; CI-NEXT: s_waitcnt lgkmcnt(0)1343; CI-NEXT: s_endpgm1344;1345; VI-LABEL: lds_atomic_dec_noret_i64_offset:1346; VI: ; %bb.0:1347; VI-NEXT: s_load_dword s0, s[4:5], 0x241348; VI-NEXT: v_mov_b32_e32 v0, 421349; VI-NEXT: v_mov_b32_e32 v1, 01350; VI-NEXT: s_mov_b32 m0, -11351; VI-NEXT: s_waitcnt lgkmcnt(0)1352; VI-NEXT: v_mov_b32_e32 v2, s01353; VI-NEXT: ds_dec_u64 v2, v[0:1] offset:321354; VI-NEXT: s_waitcnt lgkmcnt(0)1355; VI-NEXT: s_endpgm1356;1357; GFX9-LABEL: lds_atomic_dec_noret_i64_offset:1358; GFX9: ; %bb.0:1359; GFX9-NEXT: s_load_dword s0, s[4:5], 0x241360; GFX9-NEXT: v_mov_b32_e32 v0, 421361; GFX9-NEXT: v_mov_b32_e32 v1, 01362; GFX9-NEXT: s_waitcnt lgkmcnt(0)1363; GFX9-NEXT: v_mov_b32_e32 v2, s01364; GFX9-NEXT: ds_dec_u64 v2, v[0:1] offset:321365; GFX9-NEXT: s_waitcnt lgkmcnt(0)1366; GFX9-NEXT: s_endpgm1367 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 41368 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p3(ptr addrspace(3) %gep, i64 42, i32 0, i32 0, i1 false)1369 ret void1370}1371 1372define amdgpu_kernel void @global_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 {1373; CI-LABEL: global_atomic_dec_ret_i64:1374; CI: ; %bb.0:1375; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91376; CI-NEXT: s_mov_b32 s7, 0xf0001377; CI-NEXT: s_mov_b32 s6, -11378; CI-NEXT: v_mov_b32_e32 v0, 421379; CI-NEXT: s_mov_b32 s10, s61380; CI-NEXT: s_waitcnt lgkmcnt(0)1381; CI-NEXT: s_mov_b32 s8, s21382; CI-NEXT: s_mov_b32 s9, s31383; CI-NEXT: v_mov_b32_e32 v1, 01384; CI-NEXT: s_mov_b32 s11, s71385; CI-NEXT: buffer_atomic_dec_x2 v[0:1], off, s[8:11], 0 glc1386; CI-NEXT: s_waitcnt vmcnt(0)1387; CI-NEXT: buffer_wbinvl1_vol1388; CI-NEXT: s_mov_b32 s4, s01389; CI-NEXT: s_mov_b32 s5, s11390; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 01391; CI-NEXT: s_endpgm1392;1393; VI-LABEL: global_atomic_dec_ret_i64:1394; VI: ; %bb.0:1395; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241396; VI-NEXT: s_mov_b32 s7, 0xf0001397; VI-NEXT: s_mov_b32 s6, -11398; VI-NEXT: v_mov_b32_e32 v0, 421399; VI-NEXT: s_mov_b32 s10, s61400; VI-NEXT: s_waitcnt lgkmcnt(0)1401; VI-NEXT: s_mov_b32 s8, s21402; VI-NEXT: s_mov_b32 s9, s31403; VI-NEXT: v_mov_b32_e32 v1, 01404; VI-NEXT: s_mov_b32 s11, s71405; VI-NEXT: buffer_atomic_dec_x2 v[0:1], off, s[8:11], 0 glc1406; VI-NEXT: s_waitcnt vmcnt(0)1407; VI-NEXT: buffer_wbinvl1_vol1408; VI-NEXT: s_mov_b32 s4, s01409; VI-NEXT: s_mov_b32 s5, s11410; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 01411; VI-NEXT: s_endpgm1412;1413; GFX9-LABEL: global_atomic_dec_ret_i64:1414; GFX9: ; %bb.0:1415; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241416; GFX9-NEXT: v_mov_b32_e32 v0, 421417; GFX9-NEXT: v_mov_b32_e32 v1, 01418; GFX9-NEXT: v_mov_b32_e32 v2, 01419; GFX9-NEXT: s_waitcnt lgkmcnt(0)1420; GFX9-NEXT: global_atomic_dec_x2 v[0:1], v2, v[0:1], s[2:3] glc1421; GFX9-NEXT: s_waitcnt vmcnt(0)1422; GFX9-NEXT: buffer_wbinvl1_vol1423; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]1424; GFX9-NEXT: s_endpgm1425 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %ptr, i64 42, i32 0, i32 0, i1 false)1426 store i64 %result, ptr addrspace(1) %out1427 ret void1428}1429 1430define amdgpu_kernel void @global_atomic_dec_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 {1431; CI-LABEL: global_atomic_dec_ret_i64_offset:1432; CI: ; %bb.0:1433; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91434; CI-NEXT: s_mov_b32 s7, 0xf0001435; CI-NEXT: s_mov_b32 s6, -11436; CI-NEXT: v_mov_b32_e32 v0, 421437; CI-NEXT: s_mov_b32 s10, s61438; CI-NEXT: s_waitcnt lgkmcnt(0)1439; CI-NEXT: s_mov_b32 s8, s21440; CI-NEXT: s_mov_b32 s9, s31441; CI-NEXT: v_mov_b32_e32 v1, 01442; CI-NEXT: s_mov_b32 s11, s71443; CI-NEXT: buffer_atomic_dec_x2 v[0:1], off, s[8:11], 0 offset:32 glc1444; CI-NEXT: s_waitcnt vmcnt(0)1445; CI-NEXT: buffer_wbinvl1_vol1446; CI-NEXT: s_mov_b32 s4, s01447; CI-NEXT: s_mov_b32 s5, s11448; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 01449; CI-NEXT: s_endpgm1450;1451; VI-LABEL: global_atomic_dec_ret_i64_offset:1452; VI: ; %bb.0:1453; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241454; VI-NEXT: s_mov_b32 s7, 0xf0001455; VI-NEXT: s_mov_b32 s6, -11456; VI-NEXT: v_mov_b32_e32 v0, 421457; VI-NEXT: s_mov_b32 s10, s61458; VI-NEXT: s_waitcnt lgkmcnt(0)1459; VI-NEXT: s_mov_b32 s8, s21460; VI-NEXT: s_mov_b32 s9, s31461; VI-NEXT: v_mov_b32_e32 v1, 01462; VI-NEXT: s_mov_b32 s11, s71463; VI-NEXT: buffer_atomic_dec_x2 v[0:1], off, s[8:11], 0 offset:32 glc1464; VI-NEXT: s_waitcnt vmcnt(0)1465; VI-NEXT: buffer_wbinvl1_vol1466; VI-NEXT: s_mov_b32 s4, s01467; VI-NEXT: s_mov_b32 s5, s11468; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 01469; VI-NEXT: s_endpgm1470;1471; GFX9-LABEL: global_atomic_dec_ret_i64_offset:1472; GFX9: ; %bb.0:1473; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241474; GFX9-NEXT: v_mov_b32_e32 v0, 421475; GFX9-NEXT: v_mov_b32_e32 v1, 01476; GFX9-NEXT: v_mov_b32_e32 v2, 01477; GFX9-NEXT: s_waitcnt lgkmcnt(0)1478; GFX9-NEXT: global_atomic_dec_x2 v[0:1], v2, v[0:1], s[2:3] offset:32 glc1479; GFX9-NEXT: s_waitcnt vmcnt(0)1480; GFX9-NEXT: buffer_wbinvl1_vol1481; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]1482; GFX9-NEXT: s_endpgm1483 %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 41484 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %gep, i64 42, i32 0, i32 0, i1 false)1485 store i64 %result, ptr addrspace(1) %out1486 ret void1487}1488 1489define amdgpu_kernel void @global_atomic_dec_noret_i64(ptr addrspace(1) %ptr) nounwind {1490; CI-LABEL: global_atomic_dec_noret_i64:1491; CI: ; %bb.0:1492; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x91493; CI-NEXT: v_mov_b32_e32 v0, 421494; CI-NEXT: s_mov_b32 s3, 0xf0001495; CI-NEXT: v_mov_b32_e32 v1, 01496; CI-NEXT: s_mov_b32 s2, -11497; CI-NEXT: s_waitcnt lgkmcnt(0)1498; CI-NEXT: buffer_atomic_dec_x2 v[0:1], off, s[0:3], 01499; CI-NEXT: s_waitcnt vmcnt(0)1500; CI-NEXT: buffer_wbinvl1_vol1501; CI-NEXT: s_endpgm1502;1503; VI-LABEL: global_atomic_dec_noret_i64:1504; VI: ; %bb.0:1505; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241506; VI-NEXT: v_mov_b32_e32 v0, 421507; VI-NEXT: s_mov_b32 s3, 0xf0001508; VI-NEXT: v_mov_b32_e32 v1, 01509; VI-NEXT: s_mov_b32 s2, -11510; VI-NEXT: s_waitcnt lgkmcnt(0)1511; VI-NEXT: buffer_atomic_dec_x2 v[0:1], off, s[0:3], 01512; VI-NEXT: s_waitcnt vmcnt(0)1513; VI-NEXT: buffer_wbinvl1_vol1514; VI-NEXT: s_endpgm1515;1516; GFX9-LABEL: global_atomic_dec_noret_i64:1517; GFX9: ; %bb.0:1518; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241519; GFX9-NEXT: v_mov_b32_e32 v0, 421520; GFX9-NEXT: v_mov_b32_e32 v1, 01521; GFX9-NEXT: v_mov_b32_e32 v2, 01522; GFX9-NEXT: s_waitcnt lgkmcnt(0)1523; GFX9-NEXT: global_atomic_dec_x2 v2, v[0:1], s[0:1]1524; GFX9-NEXT: s_waitcnt vmcnt(0)1525; GFX9-NEXT: buffer_wbinvl1_vol1526; GFX9-NEXT: s_endpgm1527 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %ptr, i64 42, i32 0, i32 0, i1 false)1528 ret void1529}1530 1531define amdgpu_kernel void @global_atomic_dec_noret_i64_offset(ptr addrspace(1) %ptr) nounwind {1532; CI-LABEL: global_atomic_dec_noret_i64_offset:1533; CI: ; %bb.0:1534; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x91535; CI-NEXT: v_mov_b32_e32 v0, 421536; CI-NEXT: s_mov_b32 s3, 0xf0001537; CI-NEXT: v_mov_b32_e32 v1, 01538; CI-NEXT: s_mov_b32 s2, -11539; CI-NEXT: s_waitcnt lgkmcnt(0)1540; CI-NEXT: buffer_atomic_dec_x2 v[0:1], off, s[0:3], 0 offset:321541; CI-NEXT: s_waitcnt vmcnt(0)1542; CI-NEXT: buffer_wbinvl1_vol1543; CI-NEXT: s_endpgm1544;1545; VI-LABEL: global_atomic_dec_noret_i64_offset:1546; VI: ; %bb.0:1547; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241548; VI-NEXT: v_mov_b32_e32 v0, 421549; VI-NEXT: s_mov_b32 s3, 0xf0001550; VI-NEXT: v_mov_b32_e32 v1, 01551; VI-NEXT: s_mov_b32 s2, -11552; VI-NEXT: s_waitcnt lgkmcnt(0)1553; VI-NEXT: buffer_atomic_dec_x2 v[0:1], off, s[0:3], 0 offset:321554; VI-NEXT: s_waitcnt vmcnt(0)1555; VI-NEXT: buffer_wbinvl1_vol1556; VI-NEXT: s_endpgm1557;1558; GFX9-LABEL: global_atomic_dec_noret_i64_offset:1559; GFX9: ; %bb.0:1560; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241561; GFX9-NEXT: v_mov_b32_e32 v0, 421562; GFX9-NEXT: v_mov_b32_e32 v1, 01563; GFX9-NEXT: v_mov_b32_e32 v2, 01564; GFX9-NEXT: s_waitcnt lgkmcnt(0)1565; GFX9-NEXT: global_atomic_dec_x2 v2, v[0:1], s[0:1] offset:321566; GFX9-NEXT: s_waitcnt vmcnt(0)1567; GFX9-NEXT: buffer_wbinvl1_vol1568; GFX9-NEXT: s_endpgm1569 %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 41570 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %gep, i64 42, i32 0, i32 0, i1 false)1571 ret void1572}1573 1574define amdgpu_kernel void @global_atomic_dec_ret_i64_offset_addr64(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 {1575; CI-LABEL: global_atomic_dec_ret_i64_offset_addr64:1576; CI: ; %bb.0:1577; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91578; CI-NEXT: s_mov_b32 s7, 0xf0001579; CI-NEXT: s_mov_b32 s6, 01580; CI-NEXT: v_mov_b32_e32 v2, 421581; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v01582; CI-NEXT: v_mov_b32_e32 v1, 01583; CI-NEXT: s_waitcnt lgkmcnt(0)1584; CI-NEXT: s_mov_b64 s[8:9], s[2:3]1585; CI-NEXT: v_mov_b32_e32 v3, 01586; CI-NEXT: s_mov_b64 s[10:11], s[6:7]1587; CI-NEXT: buffer_atomic_dec_x2 v[2:3], v[0:1], s[8:11], 0 addr64 offset:40 glc1588; CI-NEXT: s_waitcnt vmcnt(0)1589; CI-NEXT: buffer_wbinvl1_vol1590; CI-NEXT: s_mov_b64 s[4:5], s[0:1]1591; CI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr641592; CI-NEXT: s_endpgm1593;1594; VI-LABEL: global_atomic_dec_ret_i64_offset_addr64:1595; VI: ; %bb.0:1596; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241597; VI-NEXT: v_lshlrev_b32_e32 v4, 3, v01598; VI-NEXT: s_waitcnt lgkmcnt(0)1599; VI-NEXT: v_mov_b32_e32 v0, s31600; VI-NEXT: v_add_u32_e32 v2, vcc, s2, v41601; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v0, vcc1602; VI-NEXT: v_mov_b32_e32 v0, 421603; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v21604; VI-NEXT: v_mov_b32_e32 v1, 01605; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc1606; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc1607; VI-NEXT: s_waitcnt vmcnt(0)1608; VI-NEXT: buffer_wbinvl1_vol1609; VI-NEXT: v_mov_b32_e32 v3, s11610; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v41611; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc1612; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1613; VI-NEXT: s_endpgm1614;1615; GFX9-LABEL: global_atomic_dec_ret_i64_offset_addr64:1616; GFX9: ; %bb.0:1617; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241618; GFX9-NEXT: v_mov_b32_e32 v1, 421619; GFX9-NEXT: v_mov_b32_e32 v2, 01620; GFX9-NEXT: v_lshlrev_b32_e32 v3, 3, v01621; GFX9-NEXT: s_waitcnt lgkmcnt(0)1622; GFX9-NEXT: global_atomic_dec_x2 v[0:1], v3, v[1:2], s[2:3] offset:40 glc1623; GFX9-NEXT: s_waitcnt vmcnt(0)1624; GFX9-NEXT: buffer_wbinvl1_vol1625; GFX9-NEXT: global_store_dwordx2 v3, v[0:1], s[0:1]1626; GFX9-NEXT: s_endpgm1627 %id = call i32 @llvm.amdgcn.workitem.id.x()1628 %gep.tid = getelementptr i64, ptr addrspace(1) %ptr, i32 %id1629 %out.gep = getelementptr i64, ptr addrspace(1) %out, i32 %id1630 %gep = getelementptr i64, ptr addrspace(1) %gep.tid, i32 51631 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %gep, i64 42, i32 0, i32 0, i1 false)1632 store i64 %result, ptr addrspace(1) %out.gep1633 ret void1634}1635 1636define amdgpu_kernel void @global_atomic_dec_noret_i64_offset_addr64(ptr addrspace(1) %ptr) #0 {1637; CI-LABEL: global_atomic_dec_noret_i64_offset_addr64:1638; CI: ; %bb.0:1639; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x91640; CI-NEXT: v_mov_b32_e32 v2, 421641; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v01642; CI-NEXT: v_mov_b32_e32 v1, 01643; CI-NEXT: s_mov_b32 s3, 0xf0001644; CI-NEXT: v_mov_b32_e32 v3, 01645; CI-NEXT: s_mov_b32 s2, 01646; CI-NEXT: s_waitcnt lgkmcnt(0)1647; CI-NEXT: buffer_atomic_dec_x2 v[2:3], v[0:1], s[0:3], 0 addr64 offset:401648; CI-NEXT: s_waitcnt vmcnt(0)1649; CI-NEXT: buffer_wbinvl1_vol1650; CI-NEXT: s_endpgm1651;1652; VI-LABEL: global_atomic_dec_noret_i64_offset_addr64:1653; VI: ; %bb.0:1654; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241655; VI-NEXT: v_lshlrev_b32_e32 v0, 3, v01656; VI-NEXT: s_waitcnt lgkmcnt(0)1657; VI-NEXT: v_mov_b32_e32 v1, s11658; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v01659; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc1660; VI-NEXT: v_mov_b32_e32 v0, 421661; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v21662; VI-NEXT: v_mov_b32_e32 v1, 01663; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc1664; VI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]1665; VI-NEXT: s_waitcnt vmcnt(0)1666; VI-NEXT: buffer_wbinvl1_vol1667; VI-NEXT: s_endpgm1668;1669; GFX9-LABEL: global_atomic_dec_noret_i64_offset_addr64:1670; GFX9: ; %bb.0:1671; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241672; GFX9-NEXT: v_mov_b32_e32 v1, 421673; GFX9-NEXT: v_mov_b32_e32 v2, 01674; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v01675; GFX9-NEXT: s_waitcnt lgkmcnt(0)1676; GFX9-NEXT: global_atomic_dec_x2 v0, v[1:2], s[0:1] offset:401677; GFX9-NEXT: s_waitcnt vmcnt(0)1678; GFX9-NEXT: buffer_wbinvl1_vol1679; GFX9-NEXT: s_endpgm1680 %id = call i32 @llvm.amdgcn.workitem.id.x()1681 %gep.tid = getelementptr i64, ptr addrspace(1) %ptr, i32 %id1682 %gep = getelementptr i64, ptr addrspace(1) %gep.tid, i32 51683 %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %gep, i64 42, i32 0, i32 0, i1 false)1684 ret void1685}1686 1687@lds1 = addrspace(3) global [512 x i64] poison, align 81688 1689define amdgpu_kernel void @atomic_dec_shl_base_lds_0_i64(ptr addrspace(1) %out, ptr addrspace(1) %add_use) #0 {1690; CI-LABEL: atomic_dec_shl_base_lds_0_i64:1691; CI: ; %bb.0:1692; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91693; CI-NEXT: v_mov_b32_e32 v1, 91694; CI-NEXT: v_lshlrev_b32_e32 v3, 3, v01695; CI-NEXT: v_mov_b32_e32 v2, 01696; CI-NEXT: s_mov_b32 m0, -11697; CI-NEXT: s_waitcnt lgkmcnt(0)1698; CI-NEXT: ds_dec_rtn_u64 v[1:2], v3, v[1:2] offset:161699; CI-NEXT: s_waitcnt lgkmcnt(0)1700; CI-NEXT: s_mov_b32 s7, 0xf0001701; CI-NEXT: s_mov_b32 s6, -11702; CI-NEXT: s_mov_b32 s4, s01703; CI-NEXT: s_mov_b32 s5, s11704; CI-NEXT: s_mov_b32 s0, s21705; CI-NEXT: s_mov_b32 s1, s31706; CI-NEXT: s_mov_b32 s2, s61707; CI-NEXT: s_mov_b32 s3, s71708; CI-NEXT: v_add_i32_e32 v0, vcc, 2, v01709; CI-NEXT: buffer_store_dword v0, off, s[0:3], 01710; CI-NEXT: buffer_store_dwordx2 v[1:2], off, s[4:7], 01711; CI-NEXT: s_endpgm1712;1713; VI-LABEL: atomic_dec_shl_base_lds_0_i64:1714; VI: ; %bb.0:1715; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241716; VI-NEXT: v_mov_b32_e32 v1, 91717; VI-NEXT: v_lshlrev_b32_e32 v3, 3, v01718; VI-NEXT: v_mov_b32_e32 v2, 01719; VI-NEXT: s_mov_b32 m0, -11720; VI-NEXT: s_waitcnt lgkmcnt(0)1721; VI-NEXT: ds_dec_rtn_u64 v[1:2], v3, v[1:2] offset:161722; VI-NEXT: s_waitcnt lgkmcnt(0)1723; VI-NEXT: s_mov_b32 s7, 0xf0001724; VI-NEXT: s_mov_b32 s6, -11725; VI-NEXT: s_mov_b32 s4, s01726; VI-NEXT: s_mov_b32 s5, s11727; VI-NEXT: s_mov_b32 s0, s21728; VI-NEXT: s_mov_b32 s1, s31729; VI-NEXT: s_mov_b32 s2, s61730; VI-NEXT: s_mov_b32 s3, s71731; VI-NEXT: v_add_u32_e32 v0, vcc, 2, v01732; VI-NEXT: buffer_store_dword v0, off, s[0:3], 01733; VI-NEXT: buffer_store_dwordx2 v[1:2], off, s[4:7], 01734; VI-NEXT: s_endpgm1735;1736; GFX9-LABEL: atomic_dec_shl_base_lds_0_i64:1737; GFX9: ; %bb.0:1738; GFX9-NEXT: v_mov_b32_e32 v1, 91739; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241740; GFX9-NEXT: v_mov_b32_e32 v2, 01741; GFX9-NEXT: v_lshlrev_b32_e32 v3, 3, v01742; GFX9-NEXT: s_waitcnt lgkmcnt(0)1743; GFX9-NEXT: ds_dec_rtn_u64 v[1:2], v3, v[1:2] offset:161744; GFX9-NEXT: s_waitcnt lgkmcnt(0)1745; GFX9-NEXT: v_mov_b32_e32 v3, 01746; GFX9-NEXT: v_add_u32_e32 v0, 2, v01747; GFX9-NEXT: global_store_dword v3, v0, s[2:3]1748; GFX9-NEXT: global_store_dwordx2 v3, v[1:2], s[0:1]1749; GFX9-NEXT: s_endpgm1750 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #11751 %idx.0 = add nsw i32 %tid.x, 21752 %arrayidx0 = getelementptr inbounds [512 x i64], ptr addrspace(3) @lds1, i32 0, i32 %idx.01753 %val0 = call i64 @llvm.amdgcn.atomic.dec.i64.p3(ptr addrspace(3) %arrayidx0, i64 9, i32 0, i32 0, i1 false)1754 store i32 %idx.0, ptr addrspace(1) %add_use1755 store i64 %val0, ptr addrspace(1) %out1756 ret void1757}1758 1759attributes #0 = { nounwind }1760attributes #1 = { nounwind readnone }1761attributes #2 = { nounwind argmemonly }1762