1834 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc -mtriple=amdgcn -mcpu=bonaire < %s | FileCheck -enable-var-scope -check-prefix=CI %s3; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefix=VI %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -enable-var-scope -check-prefix=GFX9 %s5 6declare i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) nocapture, i32, i32, i32, i1) #27declare i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) nocapture, i32, i32, i32, i1) #28declare i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr nocapture, i32, i32, i32, i1) #29 10declare i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) nocapture, i64, i32, i32, i1) #211declare i64 @llvm.amdgcn.atomic.inc.i64.p3(ptr addrspace(3) nocapture, i64, i32, i32, i1) #212declare i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr nocapture, i64, i32, i32, i1) #213 14declare i32 @llvm.amdgcn.workitem.id.x() #115 16define amdgpu_kernel void @lds_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 {17; CI-LABEL: lds_atomic_inc_ret_i32:18; CI: ; %bb.0:19; CI-NEXT: s_load_dword s2, s[4:5], 0xb20; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x921; CI-NEXT: v_mov_b32_e32 v0, 4222; CI-NEXT: s_mov_b32 m0, -123; CI-NEXT: s_mov_b32 s3, 0xf00024; CI-NEXT: s_waitcnt lgkmcnt(0)25; CI-NEXT: v_mov_b32_e32 v1, s226; CI-NEXT: ds_inc_rtn_u32 v0, v1, v027; CI-NEXT: s_waitcnt lgkmcnt(0)28; CI-NEXT: s_mov_b32 s2, -129; CI-NEXT: buffer_store_dword v0, off, s[0:3], 030; CI-NEXT: s_endpgm31;32; VI-LABEL: lds_atomic_inc_ret_i32:33; VI: ; %bb.0:34; VI-NEXT: s_load_dword s2, s[4:5], 0x2c35; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2436; VI-NEXT: v_mov_b32_e32 v0, 4237; VI-NEXT: s_mov_b32 m0, -138; VI-NEXT: s_mov_b32 s3, 0xf00039; VI-NEXT: s_waitcnt lgkmcnt(0)40; VI-NEXT: v_mov_b32_e32 v1, s241; VI-NEXT: ds_inc_rtn_u32 v0, v1, v042; VI-NEXT: s_waitcnt lgkmcnt(0)43; VI-NEXT: s_mov_b32 s2, -144; VI-NEXT: buffer_store_dword v0, off, s[0:3], 045; VI-NEXT: s_endpgm46;47; GFX9-LABEL: lds_atomic_inc_ret_i32:48; GFX9: ; %bb.0:49; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c50; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2451; GFX9-NEXT: v_mov_b32_e32 v0, 4252; GFX9-NEXT: s_waitcnt lgkmcnt(0)53; GFX9-NEXT: v_mov_b32_e32 v1, s254; GFX9-NEXT: ds_inc_rtn_u32 v0, v1, v055; GFX9-NEXT: s_waitcnt lgkmcnt(0)56; GFX9-NEXT: v_mov_b32_e32 v1, 057; GFX9-NEXT: global_store_dword v1, v0, s[0:1]58; GFX9-NEXT: s_endpgm59 %result = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 0, i32 0, i1 false)60 store i32 %result, ptr addrspace(1) %out61 ret void62}63 64define amdgpu_kernel void @lds_atomic_inc_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 {65; CI-LABEL: lds_atomic_inc_ret_i32_offset:66; CI: ; %bb.0:67; CI-NEXT: s_load_dword s2, s[4:5], 0xb68; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x969; CI-NEXT: v_mov_b32_e32 v0, 4270; CI-NEXT: s_mov_b32 m0, -171; CI-NEXT: s_mov_b32 s3, 0xf00072; CI-NEXT: s_waitcnt lgkmcnt(0)73; CI-NEXT: v_mov_b32_e32 v1, s274; CI-NEXT: ds_inc_rtn_u32 v0, v1, v0 offset:1675; CI-NEXT: s_waitcnt lgkmcnt(0)76; CI-NEXT: s_mov_b32 s2, -177; CI-NEXT: buffer_store_dword v0, off, s[0:3], 078; CI-NEXT: s_endpgm79;80; VI-LABEL: lds_atomic_inc_ret_i32_offset:81; VI: ; %bb.0:82; VI-NEXT: s_load_dword s2, s[4:5], 0x2c83; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2484; VI-NEXT: v_mov_b32_e32 v0, 4285; VI-NEXT: s_mov_b32 m0, -186; VI-NEXT: s_mov_b32 s3, 0xf00087; VI-NEXT: s_waitcnt lgkmcnt(0)88; VI-NEXT: v_mov_b32_e32 v1, s289; VI-NEXT: ds_inc_rtn_u32 v0, v1, v0 offset:1690; VI-NEXT: s_waitcnt lgkmcnt(0)91; VI-NEXT: s_mov_b32 s2, -192; VI-NEXT: buffer_store_dword v0, off, s[0:3], 093; VI-NEXT: s_endpgm94;95; GFX9-LABEL: lds_atomic_inc_ret_i32_offset:96; GFX9: ; %bb.0:97; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c98; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2499; GFX9-NEXT: v_mov_b32_e32 v0, 42100; GFX9-NEXT: s_waitcnt lgkmcnt(0)101; GFX9-NEXT: v_mov_b32_e32 v1, s2102; GFX9-NEXT: ds_inc_rtn_u32 v0, v1, v0 offset:16103; GFX9-NEXT: s_waitcnt lgkmcnt(0)104; GFX9-NEXT: v_mov_b32_e32 v1, 0105; GFX9-NEXT: global_store_dword v1, v0, s[0:1]106; GFX9-NEXT: s_endpgm107 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4108 %result = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %gep, i32 42, i32 0, i32 0, i1 false)109 store i32 %result, ptr addrspace(1) %out110 ret void111}112 113define amdgpu_kernel void @lds_atomic_inc_noret_i32(ptr addrspace(3) %ptr) nounwind {114; CI-LABEL: lds_atomic_inc_noret_i32:115; CI: ; %bb.0:116; CI-NEXT: s_load_dword s0, s[4:5], 0x9117; CI-NEXT: v_mov_b32_e32 v0, 42118; CI-NEXT: s_mov_b32 m0, -1119; CI-NEXT: s_waitcnt lgkmcnt(0)120; CI-NEXT: v_mov_b32_e32 v1, s0121; CI-NEXT: ds_inc_u32 v1, v0122; CI-NEXT: s_waitcnt lgkmcnt(0)123; CI-NEXT: s_endpgm124;125; VI-LABEL: lds_atomic_inc_noret_i32:126; VI: ; %bb.0:127; VI-NEXT: s_load_dword s0, s[4:5], 0x24128; VI-NEXT: v_mov_b32_e32 v0, 42129; VI-NEXT: s_mov_b32 m0, -1130; VI-NEXT: s_waitcnt lgkmcnt(0)131; VI-NEXT: v_mov_b32_e32 v1, s0132; VI-NEXT: ds_inc_u32 v1, v0133; VI-NEXT: s_waitcnt lgkmcnt(0)134; VI-NEXT: s_endpgm135;136; GFX9-LABEL: lds_atomic_inc_noret_i32:137; GFX9: ; %bb.0:138; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24139; GFX9-NEXT: v_mov_b32_e32 v0, 42140; GFX9-NEXT: s_waitcnt lgkmcnt(0)141; GFX9-NEXT: v_mov_b32_e32 v1, s0142; GFX9-NEXT: ds_inc_u32 v1, v0143; GFX9-NEXT: s_waitcnt lgkmcnt(0)144; GFX9-NEXT: s_endpgm145 %result = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 0, i32 0, i1 false)146 ret void147}148 149define amdgpu_kernel void @lds_atomic_inc_noret_i32_offset(ptr addrspace(3) %ptr) nounwind {150; CI-LABEL: lds_atomic_inc_noret_i32_offset:151; CI: ; %bb.0:152; CI-NEXT: s_load_dword s0, s[4:5], 0x9153; CI-NEXT: v_mov_b32_e32 v0, 42154; CI-NEXT: s_mov_b32 m0, -1155; CI-NEXT: s_waitcnt lgkmcnt(0)156; CI-NEXT: v_mov_b32_e32 v1, s0157; CI-NEXT: ds_inc_u32 v1, v0 offset:16158; CI-NEXT: s_waitcnt lgkmcnt(0)159; CI-NEXT: s_endpgm160;161; VI-LABEL: lds_atomic_inc_noret_i32_offset:162; VI: ; %bb.0:163; VI-NEXT: s_load_dword s0, s[4:5], 0x24164; VI-NEXT: v_mov_b32_e32 v0, 42165; VI-NEXT: s_mov_b32 m0, -1166; VI-NEXT: s_waitcnt lgkmcnt(0)167; VI-NEXT: v_mov_b32_e32 v1, s0168; VI-NEXT: ds_inc_u32 v1, v0 offset:16169; VI-NEXT: s_waitcnt lgkmcnt(0)170; VI-NEXT: s_endpgm171;172; GFX9-LABEL: lds_atomic_inc_noret_i32_offset:173; GFX9: ; %bb.0:174; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24175; GFX9-NEXT: v_mov_b32_e32 v0, 42176; GFX9-NEXT: s_waitcnt lgkmcnt(0)177; GFX9-NEXT: v_mov_b32_e32 v1, s0178; GFX9-NEXT: ds_inc_u32 v1, v0 offset:16179; GFX9-NEXT: s_waitcnt lgkmcnt(0)180; GFX9-NEXT: s_endpgm181 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4182 %result = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %gep, i32 42, i32 0, i32 0, i1 false)183 ret void184}185 186define amdgpu_kernel void @global_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 {187; CI-LABEL: global_atomic_inc_ret_i32:188; CI: ; %bb.0:189; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9190; CI-NEXT: s_mov_b32 s7, 0xf000191; CI-NEXT: s_mov_b32 s6, -1192; CI-NEXT: s_mov_b32 s10, s6193; CI-NEXT: s_mov_b32 s11, s7194; CI-NEXT: s_waitcnt lgkmcnt(0)195; CI-NEXT: s_mov_b32 s8, s2196; CI-NEXT: s_mov_b32 s9, s3197; CI-NEXT: v_mov_b32_e32 v0, 42198; CI-NEXT: buffer_atomic_inc v0, off, s[8:11], 0 glc199; CI-NEXT: s_waitcnt vmcnt(0)200; CI-NEXT: buffer_wbinvl1_vol201; CI-NEXT: s_mov_b32 s4, s0202; CI-NEXT: s_mov_b32 s5, s1203; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0204; CI-NEXT: s_endpgm205;206; VI-LABEL: global_atomic_inc_ret_i32:207; VI: ; %bb.0:208; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24209; VI-NEXT: s_mov_b32 s7, 0xf000210; VI-NEXT: s_mov_b32 s6, -1211; VI-NEXT: s_mov_b32 s10, s6212; VI-NEXT: s_mov_b32 s11, s7213; VI-NEXT: s_waitcnt lgkmcnt(0)214; VI-NEXT: s_mov_b32 s8, s2215; VI-NEXT: s_mov_b32 s9, s3216; VI-NEXT: v_mov_b32_e32 v0, 42217; VI-NEXT: buffer_atomic_inc v0, off, s[8:11], 0 glc218; VI-NEXT: s_waitcnt vmcnt(0)219; VI-NEXT: buffer_wbinvl1_vol220; VI-NEXT: s_mov_b32 s4, s0221; VI-NEXT: s_mov_b32 s5, s1222; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0223; VI-NEXT: s_endpgm224;225; GFX9-LABEL: global_atomic_inc_ret_i32:226; GFX9: ; %bb.0:227; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24228; GFX9-NEXT: v_mov_b32_e32 v0, 0229; GFX9-NEXT: v_mov_b32_e32 v1, 42230; GFX9-NEXT: s_waitcnt lgkmcnt(0)231; GFX9-NEXT: global_atomic_inc v1, v0, v1, s[2:3] glc232; GFX9-NEXT: s_waitcnt vmcnt(0)233; GFX9-NEXT: buffer_wbinvl1_vol234; GFX9-NEXT: global_store_dword v0, v1, s[0:1]235; GFX9-NEXT: s_endpgm236 %result = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %ptr, i32 42, i32 0, i32 0, i1 false)237 store i32 %result, ptr addrspace(1) %out238 ret void239}240 241define amdgpu_kernel void @global_atomic_inc_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 {242; CI-LABEL: global_atomic_inc_ret_i32_offset:243; CI: ; %bb.0:244; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9245; CI-NEXT: s_mov_b32 s7, 0xf000246; CI-NEXT: s_mov_b32 s6, -1247; CI-NEXT: s_mov_b32 s10, s6248; CI-NEXT: s_mov_b32 s11, s7249; CI-NEXT: s_waitcnt lgkmcnt(0)250; CI-NEXT: s_mov_b32 s8, s2251; CI-NEXT: s_mov_b32 s9, s3252; CI-NEXT: v_mov_b32_e32 v0, 42253; CI-NEXT: buffer_atomic_inc v0, off, s[8:11], 0 offset:16 glc254; CI-NEXT: s_waitcnt vmcnt(0)255; CI-NEXT: buffer_wbinvl1_vol256; CI-NEXT: s_mov_b32 s4, s0257; CI-NEXT: s_mov_b32 s5, s1258; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0259; CI-NEXT: s_endpgm260;261; VI-LABEL: global_atomic_inc_ret_i32_offset:262; VI: ; %bb.0:263; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24264; VI-NEXT: s_mov_b32 s7, 0xf000265; VI-NEXT: s_mov_b32 s6, -1266; VI-NEXT: s_mov_b32 s10, s6267; VI-NEXT: s_mov_b32 s11, s7268; VI-NEXT: s_waitcnt lgkmcnt(0)269; VI-NEXT: s_mov_b32 s8, s2270; VI-NEXT: s_mov_b32 s9, s3271; VI-NEXT: v_mov_b32_e32 v0, 42272; VI-NEXT: buffer_atomic_inc v0, off, s[8:11], 0 offset:16 glc273; VI-NEXT: s_waitcnt vmcnt(0)274; VI-NEXT: buffer_wbinvl1_vol275; VI-NEXT: s_mov_b32 s4, s0276; VI-NEXT: s_mov_b32 s5, s1277; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0278; VI-NEXT: s_endpgm279;280; GFX9-LABEL: global_atomic_inc_ret_i32_offset:281; GFX9: ; %bb.0:282; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24283; GFX9-NEXT: v_mov_b32_e32 v0, 0284; GFX9-NEXT: v_mov_b32_e32 v1, 42285; GFX9-NEXT: s_waitcnt lgkmcnt(0)286; GFX9-NEXT: global_atomic_inc v1, v0, v1, s[2:3] offset:16 glc287; GFX9-NEXT: s_waitcnt vmcnt(0)288; GFX9-NEXT: buffer_wbinvl1_vol289; GFX9-NEXT: global_store_dword v0, v1, s[0:1]290; GFX9-NEXT: s_endpgm291 %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4292 %result = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %gep, i32 42, i32 0, i32 0, i1 false)293 store i32 %result, ptr addrspace(1) %out294 ret void295}296 297define amdgpu_kernel void @global_atomic_inc_noret_i32(ptr addrspace(1) %ptr) nounwind {298; CI-LABEL: global_atomic_inc_noret_i32:299; CI: ; %bb.0:300; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9301; CI-NEXT: s_mov_b32 s3, 0xf000302; CI-NEXT: s_mov_b32 s2, -1303; CI-NEXT: v_mov_b32_e32 v0, 42304; CI-NEXT: s_waitcnt lgkmcnt(0)305; CI-NEXT: buffer_atomic_inc v0, off, s[0:3], 0306; CI-NEXT: s_waitcnt vmcnt(0)307; CI-NEXT: buffer_wbinvl1_vol308; CI-NEXT: s_endpgm309;310; VI-LABEL: global_atomic_inc_noret_i32:311; VI: ; %bb.0:312; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24313; VI-NEXT: s_mov_b32 s3, 0xf000314; VI-NEXT: s_mov_b32 s2, -1315; VI-NEXT: v_mov_b32_e32 v0, 42316; VI-NEXT: s_waitcnt lgkmcnt(0)317; VI-NEXT: buffer_atomic_inc v0, off, s[0:3], 0318; VI-NEXT: s_waitcnt vmcnt(0)319; VI-NEXT: buffer_wbinvl1_vol320; VI-NEXT: s_endpgm321;322; GFX9-LABEL: global_atomic_inc_noret_i32:323; GFX9: ; %bb.0:324; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24325; GFX9-NEXT: v_mov_b32_e32 v0, 0326; GFX9-NEXT: v_mov_b32_e32 v1, 42327; GFX9-NEXT: s_waitcnt lgkmcnt(0)328; GFX9-NEXT: global_atomic_inc v0, v1, s[0:1]329; GFX9-NEXT: s_waitcnt vmcnt(0)330; GFX9-NEXT: buffer_wbinvl1_vol331; GFX9-NEXT: s_endpgm332 %result = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %ptr, i32 42, i32 0, i32 0, i1 false)333 ret void334}335 336define amdgpu_kernel void @global_atomic_inc_noret_i32_offset(ptr addrspace(1) %ptr) nounwind {337; CI-LABEL: global_atomic_inc_noret_i32_offset:338; CI: ; %bb.0:339; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9340; CI-NEXT: s_mov_b32 s3, 0xf000341; CI-NEXT: s_mov_b32 s2, -1342; CI-NEXT: v_mov_b32_e32 v0, 42343; CI-NEXT: s_waitcnt lgkmcnt(0)344; CI-NEXT: buffer_atomic_inc v0, off, s[0:3], 0 offset:16345; CI-NEXT: s_waitcnt vmcnt(0)346; CI-NEXT: buffer_wbinvl1_vol347; CI-NEXT: s_endpgm348;349; VI-LABEL: global_atomic_inc_noret_i32_offset:350; VI: ; %bb.0:351; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24352; VI-NEXT: s_mov_b32 s3, 0xf000353; VI-NEXT: s_mov_b32 s2, -1354; VI-NEXT: v_mov_b32_e32 v0, 42355; VI-NEXT: s_waitcnt lgkmcnt(0)356; VI-NEXT: buffer_atomic_inc v0, off, s[0:3], 0 offset:16357; VI-NEXT: s_waitcnt vmcnt(0)358; VI-NEXT: buffer_wbinvl1_vol359; VI-NEXT: s_endpgm360;361; GFX9-LABEL: global_atomic_inc_noret_i32_offset:362; GFX9: ; %bb.0:363; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24364; GFX9-NEXT: v_mov_b32_e32 v0, 0365; GFX9-NEXT: v_mov_b32_e32 v1, 42366; GFX9-NEXT: s_waitcnt lgkmcnt(0)367; GFX9-NEXT: global_atomic_inc v0, v1, s[0:1] offset:16368; GFX9-NEXT: s_waitcnt vmcnt(0)369; GFX9-NEXT: buffer_wbinvl1_vol370; GFX9-NEXT: s_endpgm371 %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4372 %result = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %gep, i32 42, i32 0, i32 0, i1 false)373 ret void374}375 376define amdgpu_kernel void @global_atomic_inc_ret_i32_offset_addr64(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 {377; CI-LABEL: global_atomic_inc_ret_i32_offset_addr64:378; CI: ; %bb.0:379; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9380; CI-NEXT: s_mov_b32 s7, 0xf000381; CI-NEXT: s_mov_b32 s6, 0382; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0383; CI-NEXT: v_mov_b32_e32 v1, 0384; CI-NEXT: s_waitcnt lgkmcnt(0)385; CI-NEXT: s_mov_b64 s[8:9], s[2:3]386; CI-NEXT: s_mov_b64 s[10:11], s[6:7]387; CI-NEXT: v_mov_b32_e32 v2, 42388; CI-NEXT: buffer_atomic_inc v2, v[0:1], s[8:11], 0 addr64 offset:20 glc389; CI-NEXT: s_waitcnt vmcnt(0)390; CI-NEXT: buffer_wbinvl1_vol391; CI-NEXT: s_mov_b64 s[4:5], s[0:1]392; CI-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64393; CI-NEXT: s_endpgm394;395; VI-LABEL: global_atomic_inc_ret_i32_offset_addr64:396; VI: ; %bb.0:397; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24398; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0399; VI-NEXT: s_waitcnt lgkmcnt(0)400; VI-NEXT: v_mov_b32_e32 v0, s3401; VI-NEXT: v_add_u32_e32 v1, vcc, s2, v2402; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v0, vcc403; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v1404; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc405; VI-NEXT: v_mov_b32_e32 v3, 42406; VI-NEXT: flat_atomic_inc v3, v[0:1], v3 glc407; VI-NEXT: s_waitcnt vmcnt(0)408; VI-NEXT: buffer_wbinvl1_vol409; VI-NEXT: v_mov_b32_e32 v1, s1410; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2411; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc412; VI-NEXT: flat_store_dword v[0:1], v3413; VI-NEXT: s_endpgm414;415; GFX9-LABEL: global_atomic_inc_ret_i32_offset_addr64:416; GFX9: ; %bb.0:417; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24418; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0419; GFX9-NEXT: v_mov_b32_e32 v1, 42420; GFX9-NEXT: s_waitcnt lgkmcnt(0)421; GFX9-NEXT: global_atomic_inc v1, v0, v1, s[2:3] offset:20 glc422; GFX9-NEXT: s_waitcnt vmcnt(0)423; GFX9-NEXT: buffer_wbinvl1_vol424; GFX9-NEXT: global_store_dword v0, v1, s[0:1]425; GFX9-NEXT: s_endpgm426 %id = call i32 @llvm.amdgcn.workitem.id.x()427 %gep.tid = getelementptr i32, ptr addrspace(1) %ptr, i32 %id428 %out.gep = getelementptr i32, ptr addrspace(1) %out, i32 %id429 %gep = getelementptr i32, ptr addrspace(1) %gep.tid, i32 5430 %result = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %gep, i32 42, i32 0, i32 0, i1 false)431 store i32 %result, ptr addrspace(1) %out.gep432 ret void433}434 435define amdgpu_kernel void @global_atomic_inc_noret_i32_offset_addr64(ptr addrspace(1) %ptr) #0 {436; CI-LABEL: global_atomic_inc_noret_i32_offset_addr64:437; CI: ; %bb.0:438; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9439; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0440; CI-NEXT: v_mov_b32_e32 v1, 0441; CI-NEXT: s_mov_b32 s3, 0xf000442; CI-NEXT: s_mov_b32 s2, 0443; CI-NEXT: v_mov_b32_e32 v2, 42444; CI-NEXT: s_waitcnt lgkmcnt(0)445; CI-NEXT: buffer_atomic_inc v2, v[0:1], s[0:3], 0 addr64 offset:20446; CI-NEXT: s_waitcnt vmcnt(0)447; CI-NEXT: buffer_wbinvl1_vol448; CI-NEXT: s_endpgm449;450; VI-LABEL: global_atomic_inc_noret_i32_offset_addr64:451; VI: ; %bb.0:452; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24453; VI-NEXT: v_lshlrev_b32_e32 v0, 2, v0454; VI-NEXT: v_mov_b32_e32 v2, 42455; VI-NEXT: s_waitcnt lgkmcnt(0)456; VI-NEXT: v_mov_b32_e32 v1, s1457; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v0458; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc459; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v0460; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc461; VI-NEXT: flat_atomic_inc v[0:1], v2462; VI-NEXT: s_waitcnt vmcnt(0)463; VI-NEXT: buffer_wbinvl1_vol464; VI-NEXT: s_endpgm465;466; GFX9-LABEL: global_atomic_inc_noret_i32_offset_addr64:467; GFX9: ; %bb.0:468; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24469; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0470; GFX9-NEXT: v_mov_b32_e32 v1, 42471; GFX9-NEXT: s_waitcnt lgkmcnt(0)472; GFX9-NEXT: global_atomic_inc v0, v1, s[0:1] offset:20473; GFX9-NEXT: s_waitcnt vmcnt(0)474; GFX9-NEXT: buffer_wbinvl1_vol475; GFX9-NEXT: s_endpgm476 %id = call i32 @llvm.amdgcn.workitem.id.x()477 %gep.tid = getelementptr i32, ptr addrspace(1) %ptr, i32 %id478 %gep = getelementptr i32, ptr addrspace(1) %gep.tid, i32 5479 %result = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %gep, i32 42, i32 0, i32 0, i1 false)480 ret void481}482 483@lds0 = addrspace(3) global [512 x i32] poison, align 4484 485define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i32(ptr addrspace(1) %out, ptr addrspace(1) %add_use) #0 {486; CI-LABEL: atomic_inc_shl_base_lds_0_i32:487; CI: ; %bb.0:488; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9489; CI-NEXT: v_lshlrev_b32_e32 v1, 2, v0490; CI-NEXT: v_mov_b32_e32 v2, 9491; CI-NEXT: s_mov_b32 m0, -1492; CI-NEXT: s_waitcnt lgkmcnt(0)493; CI-NEXT: ds_inc_rtn_u32 v1, v1, v2 offset:8494; CI-NEXT: s_waitcnt lgkmcnt(0)495; CI-NEXT: s_mov_b32 s7, 0xf000496; CI-NEXT: s_mov_b32 s6, -1497; CI-NEXT: s_mov_b32 s4, s0498; CI-NEXT: s_mov_b32 s5, s1499; CI-NEXT: s_mov_b32 s0, s2500; CI-NEXT: s_mov_b32 s1, s3501; CI-NEXT: s_mov_b32 s2, s6502; CI-NEXT: s_mov_b32 s3, s7503; CI-NEXT: v_add_i32_e32 v0, vcc, 2, v0504; CI-NEXT: buffer_store_dword v0, off, s[0:3], 0505; CI-NEXT: buffer_store_dword v1, off, s[4:7], 0506; CI-NEXT: s_endpgm507;508; VI-LABEL: atomic_inc_shl_base_lds_0_i32:509; VI: ; %bb.0:510; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24511; VI-NEXT: v_lshlrev_b32_e32 v1, 2, v0512; VI-NEXT: v_mov_b32_e32 v2, 9513; VI-NEXT: s_mov_b32 m0, -1514; VI-NEXT: s_waitcnt lgkmcnt(0)515; VI-NEXT: ds_inc_rtn_u32 v1, v1, v2 offset:8516; VI-NEXT: s_waitcnt lgkmcnt(0)517; VI-NEXT: s_mov_b32 s7, 0xf000518; VI-NEXT: s_mov_b32 s6, -1519; VI-NEXT: s_mov_b32 s4, s0520; VI-NEXT: s_mov_b32 s5, s1521; VI-NEXT: s_mov_b32 s0, s2522; VI-NEXT: s_mov_b32 s1, s3523; VI-NEXT: s_mov_b32 s2, s6524; VI-NEXT: s_mov_b32 s3, s7525; VI-NEXT: v_add_u32_e32 v0, vcc, 2, v0526; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0527; VI-NEXT: buffer_store_dword v1, off, s[4:7], 0528; VI-NEXT: s_endpgm529;530; GFX9-LABEL: atomic_inc_shl_base_lds_0_i32:531; GFX9: ; %bb.0:532; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24533; GFX9-NEXT: v_lshlrev_b32_e32 v1, 2, v0534; GFX9-NEXT: v_mov_b32_e32 v2, 9535; GFX9-NEXT: s_waitcnt lgkmcnt(0)536; GFX9-NEXT: ds_inc_rtn_u32 v1, v1, v2 offset:8537; GFX9-NEXT: s_waitcnt lgkmcnt(0)538; GFX9-NEXT: v_mov_b32_e32 v2, 0539; GFX9-NEXT: v_add_u32_e32 v0, 2, v0540; GFX9-NEXT: global_store_dword v2, v0, s[2:3]541; GFX9-NEXT: global_store_dword v2, v1, s[0:1]542; GFX9-NEXT: s_endpgm543 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1544 %idx.0 = add nsw i32 %tid.x, 2545 %arrayidx0 = getelementptr inbounds [512 x i32], ptr addrspace(3) @lds0, i32 0, i32 %idx.0546 %val0 = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %arrayidx0, i32 9, i32 0, i32 0, i1 false)547 store i32 %idx.0, ptr addrspace(1) %add_use548 store i32 %val0, ptr addrspace(1) %out549 ret void550}551 552define amdgpu_kernel void @lds_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 {553; CI-LABEL: lds_atomic_inc_ret_i64:554; CI: ; %bb.0:555; CI-NEXT: s_load_dword s2, s[4:5], 0xb556; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9557; CI-NEXT: v_mov_b32_e32 v0, 42558; CI-NEXT: v_mov_b32_e32 v1, 0559; CI-NEXT: s_mov_b32 m0, -1560; CI-NEXT: s_waitcnt lgkmcnt(0)561; CI-NEXT: v_mov_b32_e32 v2, s2562; CI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1]563; CI-NEXT: s_waitcnt lgkmcnt(0)564; CI-NEXT: s_mov_b32 s3, 0xf000565; CI-NEXT: s_mov_b32 s2, -1566; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0567; CI-NEXT: s_endpgm568;569; VI-LABEL: lds_atomic_inc_ret_i64:570; VI: ; %bb.0:571; VI-NEXT: s_load_dword s2, s[4:5], 0x2c572; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24573; VI-NEXT: v_mov_b32_e32 v0, 42574; VI-NEXT: v_mov_b32_e32 v1, 0575; VI-NEXT: s_mov_b32 m0, -1576; VI-NEXT: s_waitcnt lgkmcnt(0)577; VI-NEXT: v_mov_b32_e32 v2, s2578; VI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1]579; VI-NEXT: s_waitcnt lgkmcnt(0)580; VI-NEXT: s_mov_b32 s3, 0xf000581; VI-NEXT: s_mov_b32 s2, -1582; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0583; VI-NEXT: s_endpgm584;585; GFX9-LABEL: lds_atomic_inc_ret_i64:586; GFX9: ; %bb.0:587; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c588; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24589; GFX9-NEXT: v_mov_b32_e32 v0, 42590; GFX9-NEXT: v_mov_b32_e32 v1, 0591; GFX9-NEXT: s_waitcnt lgkmcnt(0)592; GFX9-NEXT: v_mov_b32_e32 v2, s2593; GFX9-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1]594; GFX9-NEXT: s_waitcnt lgkmcnt(0)595; GFX9-NEXT: v_mov_b32_e32 v2, 0596; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]597; GFX9-NEXT: s_endpgm598 %result = call i64 @llvm.amdgcn.atomic.inc.i64.p3(ptr addrspace(3) %ptr, i64 42, i32 0, i32 0, i1 false)599 store i64 %result, ptr addrspace(1) %out600 ret void601}602 603define amdgpu_kernel void @lds_atomic_inc_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 {604; CI-LABEL: lds_atomic_inc_ret_i64_offset:605; CI: ; %bb.0:606; CI-NEXT: s_load_dword s2, s[4:5], 0xb607; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9608; CI-NEXT: v_mov_b32_e32 v0, 42609; CI-NEXT: v_mov_b32_e32 v1, 0610; CI-NEXT: s_mov_b32 m0, -1611; CI-NEXT: s_waitcnt lgkmcnt(0)612; CI-NEXT: v_mov_b32_e32 v2, s2613; CI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32614; CI-NEXT: s_waitcnt lgkmcnt(0)615; CI-NEXT: s_mov_b32 s3, 0xf000616; CI-NEXT: s_mov_b32 s2, -1617; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0618; CI-NEXT: s_endpgm619;620; VI-LABEL: lds_atomic_inc_ret_i64_offset:621; VI: ; %bb.0:622; VI-NEXT: s_load_dword s2, s[4:5], 0x2c623; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24624; VI-NEXT: v_mov_b32_e32 v0, 42625; VI-NEXT: v_mov_b32_e32 v1, 0626; VI-NEXT: s_mov_b32 m0, -1627; VI-NEXT: s_waitcnt lgkmcnt(0)628; VI-NEXT: v_mov_b32_e32 v2, s2629; VI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32630; VI-NEXT: s_waitcnt lgkmcnt(0)631; VI-NEXT: s_mov_b32 s3, 0xf000632; VI-NEXT: s_mov_b32 s2, -1633; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0634; VI-NEXT: s_endpgm635;636; GFX9-LABEL: lds_atomic_inc_ret_i64_offset:637; GFX9: ; %bb.0:638; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c639; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24640; GFX9-NEXT: v_mov_b32_e32 v0, 42641; GFX9-NEXT: v_mov_b32_e32 v1, 0642; GFX9-NEXT: s_waitcnt lgkmcnt(0)643; GFX9-NEXT: v_mov_b32_e32 v2, s2644; GFX9-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32645; GFX9-NEXT: s_waitcnt lgkmcnt(0)646; GFX9-NEXT: v_mov_b32_e32 v2, 0647; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]648; GFX9-NEXT: s_endpgm649 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4650 %result = call i64 @llvm.amdgcn.atomic.inc.i64.p3(ptr addrspace(3) %gep, i64 42, i32 0, i32 0, i1 false)651 store i64 %result, ptr addrspace(1) %out652 ret void653}654 655define amdgpu_kernel void @lds_atomic_inc_noret_i64(ptr addrspace(3) %ptr) nounwind {656; CI-LABEL: lds_atomic_inc_noret_i64:657; CI: ; %bb.0:658; CI-NEXT: s_load_dword s0, s[4:5], 0x9659; CI-NEXT: v_mov_b32_e32 v0, 42660; CI-NEXT: v_mov_b32_e32 v1, 0661; CI-NEXT: s_mov_b32 m0, -1662; CI-NEXT: s_waitcnt lgkmcnt(0)663; CI-NEXT: v_mov_b32_e32 v2, s0664; CI-NEXT: ds_inc_u64 v2, v[0:1]665; CI-NEXT: s_waitcnt lgkmcnt(0)666; CI-NEXT: s_endpgm667;668; VI-LABEL: lds_atomic_inc_noret_i64:669; VI: ; %bb.0:670; VI-NEXT: s_load_dword s0, s[4:5], 0x24671; VI-NEXT: v_mov_b32_e32 v0, 42672; VI-NEXT: v_mov_b32_e32 v1, 0673; VI-NEXT: s_mov_b32 m0, -1674; VI-NEXT: s_waitcnt lgkmcnt(0)675; VI-NEXT: v_mov_b32_e32 v2, s0676; VI-NEXT: ds_inc_u64 v2, v[0:1]677; VI-NEXT: s_waitcnt lgkmcnt(0)678; VI-NEXT: s_endpgm679;680; GFX9-LABEL: lds_atomic_inc_noret_i64:681; GFX9: ; %bb.0:682; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24683; GFX9-NEXT: v_mov_b32_e32 v0, 42684; GFX9-NEXT: v_mov_b32_e32 v1, 0685; GFX9-NEXT: s_waitcnt lgkmcnt(0)686; GFX9-NEXT: v_mov_b32_e32 v2, s0687; GFX9-NEXT: ds_inc_u64 v2, v[0:1]688; GFX9-NEXT: s_waitcnt lgkmcnt(0)689; GFX9-NEXT: s_endpgm690 %result = call i64 @llvm.amdgcn.atomic.inc.i64.p3(ptr addrspace(3) %ptr, i64 42, i32 0, i32 0, i1 false)691 ret void692}693 694define amdgpu_kernel void @lds_atomic_inc_noret_i64_offset(ptr addrspace(3) %ptr) nounwind {695; CI-LABEL: lds_atomic_inc_noret_i64_offset:696; CI: ; %bb.0:697; CI-NEXT: s_load_dword s0, s[4:5], 0x9698; CI-NEXT: v_mov_b32_e32 v0, 42699; CI-NEXT: v_mov_b32_e32 v1, 0700; CI-NEXT: s_mov_b32 m0, -1701; CI-NEXT: s_waitcnt lgkmcnt(0)702; CI-NEXT: v_mov_b32_e32 v2, s0703; CI-NEXT: ds_inc_u64 v2, v[0:1] offset:32704; CI-NEXT: s_waitcnt lgkmcnt(0)705; CI-NEXT: s_endpgm706;707; VI-LABEL: lds_atomic_inc_noret_i64_offset:708; VI: ; %bb.0:709; VI-NEXT: s_load_dword s0, s[4:5], 0x24710; VI-NEXT: v_mov_b32_e32 v0, 42711; VI-NEXT: v_mov_b32_e32 v1, 0712; VI-NEXT: s_mov_b32 m0, -1713; VI-NEXT: s_waitcnt lgkmcnt(0)714; VI-NEXT: v_mov_b32_e32 v2, s0715; VI-NEXT: ds_inc_u64 v2, v[0:1] offset:32716; VI-NEXT: s_waitcnt lgkmcnt(0)717; VI-NEXT: s_endpgm718;719; GFX9-LABEL: lds_atomic_inc_noret_i64_offset:720; GFX9: ; %bb.0:721; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24722; GFX9-NEXT: v_mov_b32_e32 v0, 42723; GFX9-NEXT: v_mov_b32_e32 v1, 0724; GFX9-NEXT: s_waitcnt lgkmcnt(0)725; GFX9-NEXT: v_mov_b32_e32 v2, s0726; GFX9-NEXT: ds_inc_u64 v2, v[0:1] offset:32727; GFX9-NEXT: s_waitcnt lgkmcnt(0)728; GFX9-NEXT: s_endpgm729 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4730 %result = call i64 @llvm.amdgcn.atomic.inc.i64.p3(ptr addrspace(3) %gep, i64 42, i32 0, i32 0, i1 false)731 ret void732}733 734define amdgpu_kernel void @global_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 {735; CI-LABEL: global_atomic_inc_ret_i64:736; CI: ; %bb.0:737; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9738; CI-NEXT: s_mov_b32 s7, 0xf000739; CI-NEXT: s_mov_b32 s6, -1740; CI-NEXT: v_mov_b32_e32 v0, 42741; CI-NEXT: s_mov_b32 s10, s6742; CI-NEXT: s_waitcnt lgkmcnt(0)743; CI-NEXT: s_mov_b32 s8, s2744; CI-NEXT: s_mov_b32 s9, s3745; CI-NEXT: v_mov_b32_e32 v1, 0746; CI-NEXT: s_mov_b32 s11, s7747; CI-NEXT: buffer_atomic_inc_x2 v[0:1], off, s[8:11], 0 glc748; CI-NEXT: s_waitcnt vmcnt(0)749; CI-NEXT: buffer_wbinvl1_vol750; CI-NEXT: s_mov_b32 s4, s0751; CI-NEXT: s_mov_b32 s5, s1752; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0753; CI-NEXT: s_endpgm754;755; VI-LABEL: global_atomic_inc_ret_i64:756; VI: ; %bb.0:757; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24758; VI-NEXT: s_mov_b32 s7, 0xf000759; VI-NEXT: s_mov_b32 s6, -1760; VI-NEXT: v_mov_b32_e32 v0, 42761; VI-NEXT: s_mov_b32 s10, s6762; VI-NEXT: s_waitcnt lgkmcnt(0)763; VI-NEXT: s_mov_b32 s8, s2764; VI-NEXT: s_mov_b32 s9, s3765; VI-NEXT: v_mov_b32_e32 v1, 0766; VI-NEXT: s_mov_b32 s11, s7767; VI-NEXT: buffer_atomic_inc_x2 v[0:1], off, s[8:11], 0 glc768; VI-NEXT: s_waitcnt vmcnt(0)769; VI-NEXT: buffer_wbinvl1_vol770; VI-NEXT: s_mov_b32 s4, s0771; VI-NEXT: s_mov_b32 s5, s1772; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0773; VI-NEXT: s_endpgm774;775; GFX9-LABEL: global_atomic_inc_ret_i64:776; GFX9: ; %bb.0:777; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24778; GFX9-NEXT: v_mov_b32_e32 v0, 42779; GFX9-NEXT: v_mov_b32_e32 v1, 0780; GFX9-NEXT: v_mov_b32_e32 v2, 0781; GFX9-NEXT: s_waitcnt lgkmcnt(0)782; GFX9-NEXT: global_atomic_inc_x2 v[0:1], v2, v[0:1], s[2:3] glc783; GFX9-NEXT: s_waitcnt vmcnt(0)784; GFX9-NEXT: buffer_wbinvl1_vol785; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]786; GFX9-NEXT: s_endpgm787 %result = call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %ptr, i64 42, i32 0, i32 0, i1 false)788 store i64 %result, ptr addrspace(1) %out789 ret void790}791 792define amdgpu_kernel void @global_atomic_inc_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 {793; CI-LABEL: global_atomic_inc_ret_i64_offset:794; CI: ; %bb.0:795; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9796; CI-NEXT: s_mov_b32 s7, 0xf000797; CI-NEXT: s_mov_b32 s6, -1798; CI-NEXT: v_mov_b32_e32 v0, 42799; CI-NEXT: s_mov_b32 s10, s6800; CI-NEXT: s_waitcnt lgkmcnt(0)801; CI-NEXT: s_mov_b32 s8, s2802; CI-NEXT: s_mov_b32 s9, s3803; CI-NEXT: v_mov_b32_e32 v1, 0804; CI-NEXT: s_mov_b32 s11, s7805; CI-NEXT: buffer_atomic_inc_x2 v[0:1], off, s[8:11], 0 offset:32 glc806; CI-NEXT: s_waitcnt vmcnt(0)807; CI-NEXT: buffer_wbinvl1_vol808; CI-NEXT: s_mov_b32 s4, s0809; CI-NEXT: s_mov_b32 s5, s1810; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0811; CI-NEXT: s_endpgm812;813; VI-LABEL: global_atomic_inc_ret_i64_offset:814; VI: ; %bb.0:815; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24816; VI-NEXT: s_mov_b32 s7, 0xf000817; VI-NEXT: s_mov_b32 s6, -1818; VI-NEXT: v_mov_b32_e32 v0, 42819; VI-NEXT: s_mov_b32 s10, s6820; VI-NEXT: s_waitcnt lgkmcnt(0)821; VI-NEXT: s_mov_b32 s8, s2822; VI-NEXT: s_mov_b32 s9, s3823; VI-NEXT: v_mov_b32_e32 v1, 0824; VI-NEXT: s_mov_b32 s11, s7825; VI-NEXT: buffer_atomic_inc_x2 v[0:1], off, s[8:11], 0 offset:32 glc826; VI-NEXT: s_waitcnt vmcnt(0)827; VI-NEXT: buffer_wbinvl1_vol828; VI-NEXT: s_mov_b32 s4, s0829; VI-NEXT: s_mov_b32 s5, s1830; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0831; VI-NEXT: s_endpgm832;833; GFX9-LABEL: global_atomic_inc_ret_i64_offset:834; GFX9: ; %bb.0:835; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24836; GFX9-NEXT: v_mov_b32_e32 v0, 42837; GFX9-NEXT: v_mov_b32_e32 v1, 0838; GFX9-NEXT: v_mov_b32_e32 v2, 0839; GFX9-NEXT: s_waitcnt lgkmcnt(0)840; GFX9-NEXT: global_atomic_inc_x2 v[0:1], v2, v[0:1], s[2:3] offset:32 glc841; GFX9-NEXT: s_waitcnt vmcnt(0)842; GFX9-NEXT: buffer_wbinvl1_vol843; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]844; GFX9-NEXT: s_endpgm845 %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 4846 %result = call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %gep, i64 42, i32 0, i32 0, i1 false)847 store i64 %result, ptr addrspace(1) %out848 ret void849}850 851define amdgpu_kernel void @global_atomic_inc_noret_i64(ptr addrspace(1) %ptr) nounwind {852; CI-LABEL: global_atomic_inc_noret_i64:853; CI: ; %bb.0:854; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9855; CI-NEXT: v_mov_b32_e32 v0, 42856; CI-NEXT: s_mov_b32 s3, 0xf000857; CI-NEXT: v_mov_b32_e32 v1, 0858; CI-NEXT: s_mov_b32 s2, -1859; CI-NEXT: s_waitcnt lgkmcnt(0)860; CI-NEXT: buffer_atomic_inc_x2 v[0:1], off, s[0:3], 0861; CI-NEXT: s_waitcnt vmcnt(0)862; CI-NEXT: buffer_wbinvl1_vol863; CI-NEXT: s_endpgm864;865; VI-LABEL: global_atomic_inc_noret_i64:866; VI: ; %bb.0:867; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24868; VI-NEXT: v_mov_b32_e32 v0, 42869; VI-NEXT: s_mov_b32 s3, 0xf000870; VI-NEXT: v_mov_b32_e32 v1, 0871; VI-NEXT: s_mov_b32 s2, -1872; VI-NEXT: s_waitcnt lgkmcnt(0)873; VI-NEXT: buffer_atomic_inc_x2 v[0:1], off, s[0:3], 0874; VI-NEXT: s_waitcnt vmcnt(0)875; VI-NEXT: buffer_wbinvl1_vol876; VI-NEXT: s_endpgm877;878; GFX9-LABEL: global_atomic_inc_noret_i64:879; GFX9: ; %bb.0:880; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24881; GFX9-NEXT: v_mov_b32_e32 v0, 42882; GFX9-NEXT: v_mov_b32_e32 v1, 0883; GFX9-NEXT: v_mov_b32_e32 v2, 0884; GFX9-NEXT: s_waitcnt lgkmcnt(0)885; GFX9-NEXT: global_atomic_inc_x2 v2, v[0:1], s[0:1]886; GFX9-NEXT: s_waitcnt vmcnt(0)887; GFX9-NEXT: buffer_wbinvl1_vol888; GFX9-NEXT: s_endpgm889 %result = call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %ptr, i64 42, i32 0, i32 0, i1 false)890 ret void891}892 893define amdgpu_kernel void @global_atomic_inc_noret_i64_offset(ptr addrspace(1) %ptr) nounwind {894; CI-LABEL: global_atomic_inc_noret_i64_offset:895; CI: ; %bb.0:896; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9897; CI-NEXT: v_mov_b32_e32 v0, 42898; CI-NEXT: s_mov_b32 s3, 0xf000899; CI-NEXT: v_mov_b32_e32 v1, 0900; CI-NEXT: s_mov_b32 s2, -1901; CI-NEXT: s_waitcnt lgkmcnt(0)902; CI-NEXT: buffer_atomic_inc_x2 v[0:1], off, s[0:3], 0 offset:32903; CI-NEXT: s_waitcnt vmcnt(0)904; CI-NEXT: buffer_wbinvl1_vol905; CI-NEXT: s_endpgm906;907; VI-LABEL: global_atomic_inc_noret_i64_offset:908; VI: ; %bb.0:909; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24910; VI-NEXT: v_mov_b32_e32 v0, 42911; VI-NEXT: s_mov_b32 s3, 0xf000912; VI-NEXT: v_mov_b32_e32 v1, 0913; VI-NEXT: s_mov_b32 s2, -1914; VI-NEXT: s_waitcnt lgkmcnt(0)915; VI-NEXT: buffer_atomic_inc_x2 v[0:1], off, s[0:3], 0 offset:32916; VI-NEXT: s_waitcnt vmcnt(0)917; VI-NEXT: buffer_wbinvl1_vol918; VI-NEXT: s_endpgm919;920; GFX9-LABEL: global_atomic_inc_noret_i64_offset:921; GFX9: ; %bb.0:922; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24923; GFX9-NEXT: v_mov_b32_e32 v0, 42924; GFX9-NEXT: v_mov_b32_e32 v1, 0925; GFX9-NEXT: v_mov_b32_e32 v2, 0926; GFX9-NEXT: s_waitcnt lgkmcnt(0)927; GFX9-NEXT: global_atomic_inc_x2 v2, v[0:1], s[0:1] offset:32928; GFX9-NEXT: s_waitcnt vmcnt(0)929; GFX9-NEXT: buffer_wbinvl1_vol930; GFX9-NEXT: s_endpgm931 %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 4932 %result = call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %gep, i64 42, i32 0, i32 0, i1 false)933 ret void934}935 936define amdgpu_kernel void @global_atomic_inc_ret_i64_offset_addr64(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 {937; CI-LABEL: global_atomic_inc_ret_i64_offset_addr64:938; CI: ; %bb.0:939; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9940; CI-NEXT: s_mov_b32 s7, 0xf000941; CI-NEXT: s_mov_b32 s6, 0942; CI-NEXT: v_mov_b32_e32 v2, 42943; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0944; CI-NEXT: v_mov_b32_e32 v1, 0945; CI-NEXT: s_waitcnt lgkmcnt(0)946; CI-NEXT: s_mov_b64 s[8:9], s[2:3]947; CI-NEXT: v_mov_b32_e32 v3, 0948; CI-NEXT: s_mov_b64 s[10:11], s[6:7]949; CI-NEXT: buffer_atomic_inc_x2 v[2:3], v[0:1], s[8:11], 0 addr64 offset:40 glc950; CI-NEXT: s_waitcnt vmcnt(0)951; CI-NEXT: buffer_wbinvl1_vol952; CI-NEXT: s_mov_b64 s[4:5], s[0:1]953; CI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64954; CI-NEXT: s_endpgm955;956; VI-LABEL: global_atomic_inc_ret_i64_offset_addr64:957; VI: ; %bb.0:958; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24959; VI-NEXT: v_lshlrev_b32_e32 v4, 3, v0960; VI-NEXT: s_waitcnt lgkmcnt(0)961; VI-NEXT: v_mov_b32_e32 v0, s3962; VI-NEXT: v_add_u32_e32 v2, vcc, s2, v4963; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v0, vcc964; VI-NEXT: v_mov_b32_e32 v0, 42965; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v2966; VI-NEXT: v_mov_b32_e32 v1, 0967; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc968; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc969; VI-NEXT: s_waitcnt vmcnt(0)970; VI-NEXT: buffer_wbinvl1_vol971; VI-NEXT: v_mov_b32_e32 v3, s1972; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v4973; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc974; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]975; VI-NEXT: s_endpgm976;977; GFX9-LABEL: global_atomic_inc_ret_i64_offset_addr64:978; GFX9: ; %bb.0:979; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24980; GFX9-NEXT: v_mov_b32_e32 v1, 42981; GFX9-NEXT: v_mov_b32_e32 v2, 0982; GFX9-NEXT: v_lshlrev_b32_e32 v3, 3, v0983; GFX9-NEXT: s_waitcnt lgkmcnt(0)984; GFX9-NEXT: global_atomic_inc_x2 v[0:1], v3, v[1:2], s[2:3] offset:40 glc985; GFX9-NEXT: s_waitcnt vmcnt(0)986; GFX9-NEXT: buffer_wbinvl1_vol987; GFX9-NEXT: global_store_dwordx2 v3, v[0:1], s[0:1]988; GFX9-NEXT: s_endpgm989 %id = call i32 @llvm.amdgcn.workitem.id.x()990 %gep.tid = getelementptr i64, ptr addrspace(1) %ptr, i32 %id991 %out.gep = getelementptr i64, ptr addrspace(1) %out, i32 %id992 %gep = getelementptr i64, ptr addrspace(1) %gep.tid, i32 5993 %result = call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %gep, i64 42, i32 0, i32 0, i1 false)994 store i64 %result, ptr addrspace(1) %out.gep995 ret void996}997 998define amdgpu_kernel void @global_atomic_inc_noret_i64_offset_addr64(ptr addrspace(1) %ptr) #0 {999; CI-LABEL: global_atomic_inc_noret_i64_offset_addr64:1000; CI: ; %bb.0:1001; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x91002; CI-NEXT: v_mov_b32_e32 v2, 421003; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v01004; CI-NEXT: v_mov_b32_e32 v1, 01005; CI-NEXT: s_mov_b32 s3, 0xf0001006; CI-NEXT: v_mov_b32_e32 v3, 01007; CI-NEXT: s_mov_b32 s2, 01008; CI-NEXT: s_waitcnt lgkmcnt(0)1009; CI-NEXT: buffer_atomic_inc_x2 v[2:3], v[0:1], s[0:3], 0 addr64 offset:401010; CI-NEXT: s_waitcnt vmcnt(0)1011; CI-NEXT: buffer_wbinvl1_vol1012; CI-NEXT: s_endpgm1013;1014; VI-LABEL: global_atomic_inc_noret_i64_offset_addr64:1015; VI: ; %bb.0:1016; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241017; VI-NEXT: v_lshlrev_b32_e32 v0, 3, v01018; VI-NEXT: s_waitcnt lgkmcnt(0)1019; VI-NEXT: v_mov_b32_e32 v1, s11020; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v01021; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc1022; VI-NEXT: v_mov_b32_e32 v0, 421023; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v21024; VI-NEXT: v_mov_b32_e32 v1, 01025; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc1026; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]1027; VI-NEXT: s_waitcnt vmcnt(0)1028; VI-NEXT: buffer_wbinvl1_vol1029; VI-NEXT: s_endpgm1030;1031; GFX9-LABEL: global_atomic_inc_noret_i64_offset_addr64:1032; GFX9: ; %bb.0:1033; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241034; GFX9-NEXT: v_mov_b32_e32 v1, 421035; GFX9-NEXT: v_mov_b32_e32 v2, 01036; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v01037; GFX9-NEXT: s_waitcnt lgkmcnt(0)1038; GFX9-NEXT: global_atomic_inc_x2 v0, v[1:2], s[0:1] offset:401039; GFX9-NEXT: s_waitcnt vmcnt(0)1040; GFX9-NEXT: buffer_wbinvl1_vol1041; GFX9-NEXT: s_endpgm1042 %id = call i32 @llvm.amdgcn.workitem.id.x()1043 %gep.tid = getelementptr i64, ptr addrspace(1) %ptr, i32 %id1044 %gep = getelementptr i64, ptr addrspace(1) %gep.tid, i32 51045 %result = call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %gep, i64 42, i32 0, i32 0, i1 false)1046 ret void1047}1048 1049define amdgpu_kernel void @flat_atomic_inc_ret_i32(ptr %out, ptr %ptr) #0 {1050; CI-LABEL: flat_atomic_inc_ret_i32:1051; CI: ; %bb.0:1052; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91053; CI-NEXT: v_mov_b32_e32 v2, 421054; CI-NEXT: s_waitcnt lgkmcnt(0)1055; CI-NEXT: v_mov_b32_e32 v0, s21056; CI-NEXT: v_mov_b32_e32 v1, s31057; CI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc1058; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1059; CI-NEXT: buffer_wbinvl1_vol1060; CI-NEXT: v_mov_b32_e32 v0, s01061; CI-NEXT: v_mov_b32_e32 v1, s11062; CI-NEXT: flat_store_dword v[0:1], v21063; CI-NEXT: s_endpgm1064;1065; VI-LABEL: flat_atomic_inc_ret_i32:1066; VI: ; %bb.0:1067; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241068; VI-NEXT: v_mov_b32_e32 v2, 421069; VI-NEXT: s_waitcnt lgkmcnt(0)1070; VI-NEXT: v_mov_b32_e32 v0, s21071; VI-NEXT: v_mov_b32_e32 v1, s31072; VI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc1073; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1074; VI-NEXT: buffer_wbinvl1_vol1075; VI-NEXT: v_mov_b32_e32 v0, s01076; VI-NEXT: v_mov_b32_e32 v1, s11077; VI-NEXT: flat_store_dword v[0:1], v21078; VI-NEXT: s_endpgm1079;1080; GFX9-LABEL: flat_atomic_inc_ret_i32:1081; GFX9: ; %bb.0:1082; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241083; GFX9-NEXT: v_mov_b32_e32 v2, 421084; GFX9-NEXT: s_waitcnt lgkmcnt(0)1085; GFX9-NEXT: v_mov_b32_e32 v0, s21086; GFX9-NEXT: v_mov_b32_e32 v1, s31087; GFX9-NEXT: flat_atomic_inc v2, v[0:1], v2 glc1088; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1089; GFX9-NEXT: buffer_wbinvl1_vol1090; GFX9-NEXT: v_mov_b32_e32 v0, s01091; GFX9-NEXT: v_mov_b32_e32 v1, s11092; GFX9-NEXT: flat_store_dword v[0:1], v21093; GFX9-NEXT: s_endpgm1094 %result = call i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr %ptr, i32 42, i32 0, i32 0, i1 false)1095 store i32 %result, ptr %out1096 ret void1097}1098 1099define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset(ptr %out, ptr %ptr) #0 {1100; CI-LABEL: flat_atomic_inc_ret_i32_offset:1101; CI: ; %bb.0:1102; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91103; CI-NEXT: v_mov_b32_e32 v2, 421104; CI-NEXT: s_waitcnt lgkmcnt(0)1105; CI-NEXT: s_add_u32 s2, s2, 161106; CI-NEXT: s_addc_u32 s3, s3, 01107; CI-NEXT: v_mov_b32_e32 v0, s21108; CI-NEXT: v_mov_b32_e32 v1, s31109; CI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc1110; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1111; CI-NEXT: buffer_wbinvl1_vol1112; CI-NEXT: v_mov_b32_e32 v0, s01113; CI-NEXT: v_mov_b32_e32 v1, s11114; CI-NEXT: flat_store_dword v[0:1], v21115; CI-NEXT: s_endpgm1116;1117; VI-LABEL: flat_atomic_inc_ret_i32_offset:1118; VI: ; %bb.0:1119; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241120; VI-NEXT: v_mov_b32_e32 v2, 421121; VI-NEXT: s_waitcnt lgkmcnt(0)1122; VI-NEXT: s_add_u32 s2, s2, 161123; VI-NEXT: s_addc_u32 s3, s3, 01124; VI-NEXT: v_mov_b32_e32 v0, s21125; VI-NEXT: v_mov_b32_e32 v1, s31126; VI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc1127; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1128; VI-NEXT: buffer_wbinvl1_vol1129; VI-NEXT: v_mov_b32_e32 v0, s01130; VI-NEXT: v_mov_b32_e32 v1, s11131; VI-NEXT: flat_store_dword v[0:1], v21132; VI-NEXT: s_endpgm1133;1134; GFX9-LABEL: flat_atomic_inc_ret_i32_offset:1135; GFX9: ; %bb.0:1136; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241137; GFX9-NEXT: v_mov_b32_e32 v2, 421138; GFX9-NEXT: s_waitcnt lgkmcnt(0)1139; GFX9-NEXT: v_mov_b32_e32 v0, s21140; GFX9-NEXT: v_mov_b32_e32 v1, s31141; GFX9-NEXT: flat_atomic_inc v2, v[0:1], v2 offset:16 glc1142; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1143; GFX9-NEXT: buffer_wbinvl1_vol1144; GFX9-NEXT: v_mov_b32_e32 v0, s01145; GFX9-NEXT: v_mov_b32_e32 v1, s11146; GFX9-NEXT: flat_store_dword v[0:1], v21147; GFX9-NEXT: s_endpgm1148 %gep = getelementptr inbounds i32, ptr %ptr, i32 41149 %result = call i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr %gep, i32 42, i32 0, i32 0, i1 false)1150 store i32 %result, ptr %out1151 ret void1152}1153 1154define amdgpu_kernel void @flat_atomic_inc_noret_i32(ptr %ptr) nounwind {1155; CI-LABEL: flat_atomic_inc_noret_i32:1156; CI: ; %bb.0:1157; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x91158; CI-NEXT: v_mov_b32_e32 v2, 421159; CI-NEXT: s_waitcnt lgkmcnt(0)1160; CI-NEXT: v_mov_b32_e32 v0, s01161; CI-NEXT: v_mov_b32_e32 v1, s11162; CI-NEXT: flat_atomic_inc v[0:1], v21163; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1164; CI-NEXT: buffer_wbinvl1_vol1165; CI-NEXT: s_endpgm1166;1167; VI-LABEL: flat_atomic_inc_noret_i32:1168; VI: ; %bb.0:1169; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241170; VI-NEXT: v_mov_b32_e32 v2, 421171; VI-NEXT: s_waitcnt lgkmcnt(0)1172; VI-NEXT: v_mov_b32_e32 v0, s01173; VI-NEXT: v_mov_b32_e32 v1, s11174; VI-NEXT: flat_atomic_inc v[0:1], v21175; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1176; VI-NEXT: buffer_wbinvl1_vol1177; VI-NEXT: s_endpgm1178;1179; GFX9-LABEL: flat_atomic_inc_noret_i32:1180; GFX9: ; %bb.0:1181; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241182; GFX9-NEXT: v_mov_b32_e32 v2, 421183; GFX9-NEXT: s_waitcnt lgkmcnt(0)1184; GFX9-NEXT: v_mov_b32_e32 v0, s01185; GFX9-NEXT: v_mov_b32_e32 v1, s11186; GFX9-NEXT: flat_atomic_inc v[0:1], v21187; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1188; GFX9-NEXT: buffer_wbinvl1_vol1189; GFX9-NEXT: s_endpgm1190 %result = call i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr %ptr, i32 42, i32 0, i32 0, i1 false)1191 ret void1192}1193 1194define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset(ptr %ptr) nounwind {1195; CI-LABEL: flat_atomic_inc_noret_i32_offset:1196; CI: ; %bb.0:1197; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x91198; CI-NEXT: v_mov_b32_e32 v2, 421199; CI-NEXT: s_waitcnt lgkmcnt(0)1200; CI-NEXT: s_add_u32 s0, s0, 161201; CI-NEXT: s_addc_u32 s1, s1, 01202; CI-NEXT: v_mov_b32_e32 v0, s01203; CI-NEXT: v_mov_b32_e32 v1, s11204; CI-NEXT: flat_atomic_inc v[0:1], v21205; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1206; CI-NEXT: buffer_wbinvl1_vol1207; CI-NEXT: s_endpgm1208;1209; VI-LABEL: flat_atomic_inc_noret_i32_offset:1210; VI: ; %bb.0:1211; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241212; VI-NEXT: v_mov_b32_e32 v2, 421213; VI-NEXT: s_waitcnt lgkmcnt(0)1214; VI-NEXT: s_add_u32 s0, s0, 161215; VI-NEXT: s_addc_u32 s1, s1, 01216; VI-NEXT: v_mov_b32_e32 v0, s01217; VI-NEXT: v_mov_b32_e32 v1, s11218; VI-NEXT: flat_atomic_inc v[0:1], v21219; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1220; VI-NEXT: buffer_wbinvl1_vol1221; VI-NEXT: s_endpgm1222;1223; GFX9-LABEL: flat_atomic_inc_noret_i32_offset:1224; GFX9: ; %bb.0:1225; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241226; GFX9-NEXT: v_mov_b32_e32 v2, 421227; GFX9-NEXT: s_waitcnt lgkmcnt(0)1228; GFX9-NEXT: v_mov_b32_e32 v0, s01229; GFX9-NEXT: v_mov_b32_e32 v1, s11230; GFX9-NEXT: flat_atomic_inc v[0:1], v2 offset:161231; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1232; GFX9-NEXT: buffer_wbinvl1_vol1233; GFX9-NEXT: s_endpgm1234 %gep = getelementptr inbounds i32, ptr %ptr, i32 41235 %result = call i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr %gep, i32 42, i32 0, i32 0, i1 false)1236 ret void1237}1238 1239define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_addr64(ptr %out, ptr %ptr) #0 {1240; CI-LABEL: flat_atomic_inc_ret_i32_offset_addr64:1241; CI: ; %bb.0:1242; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91243; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v01244; CI-NEXT: s_waitcnt lgkmcnt(0)1245; CI-NEXT: v_mov_b32_e32 v0, s31246; CI-NEXT: v_add_i32_e32 v1, vcc, s2, v21247; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v0, vcc1248; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v11249; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc1250; CI-NEXT: v_mov_b32_e32 v3, 421251; CI-NEXT: flat_atomic_inc v3, v[0:1], v3 glc1252; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1253; CI-NEXT: buffer_wbinvl1_vol1254; CI-NEXT: v_mov_b32_e32 v1, s11255; CI-NEXT: v_add_i32_e32 v0, vcc, s0, v21256; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1257; CI-NEXT: flat_store_dword v[0:1], v31258; CI-NEXT: s_endpgm1259;1260; VI-LABEL: flat_atomic_inc_ret_i32_offset_addr64:1261; VI: ; %bb.0:1262; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241263; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v01264; VI-NEXT: s_waitcnt lgkmcnt(0)1265; VI-NEXT: v_mov_b32_e32 v0, s31266; VI-NEXT: v_add_u32_e32 v1, vcc, s2, v21267; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v0, vcc1268; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v11269; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc1270; VI-NEXT: v_mov_b32_e32 v3, 421271; VI-NEXT: flat_atomic_inc v3, v[0:1], v3 glc1272; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1273; VI-NEXT: buffer_wbinvl1_vol1274; VI-NEXT: v_mov_b32_e32 v1, s11275; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v21276; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1277; VI-NEXT: flat_store_dword v[0:1], v31278; VI-NEXT: s_endpgm1279;1280; GFX9-LABEL: flat_atomic_inc_ret_i32_offset_addr64:1281; GFX9: ; %bb.0:1282; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241283; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v01284; GFX9-NEXT: v_mov_b32_e32 v3, 421285; GFX9-NEXT: s_waitcnt lgkmcnt(0)1286; GFX9-NEXT: v_mov_b32_e32 v1, s31287; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s2, v21288; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc1289; GFX9-NEXT: flat_atomic_inc v3, v[0:1], v3 offset:20 glc1290; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1291; GFX9-NEXT: buffer_wbinvl1_vol1292; GFX9-NEXT: v_mov_b32_e32 v1, s11293; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v21294; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc1295; GFX9-NEXT: flat_store_dword v[0:1], v31296; GFX9-NEXT: s_endpgm1297 %id = call i32 @llvm.amdgcn.workitem.id.x()1298 %gep.tid = getelementptr inbounds i32, ptr %ptr, i32 %id1299 %out.gep = getelementptr inbounds i32, ptr %out, i32 %id1300 %gep = getelementptr inbounds i32, ptr %gep.tid, i32 51301 %result = call i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr %gep, i32 42, i32 0, i32 0, i1 false)1302 store i32 %result, ptr %out.gep1303 ret void1304}1305 1306define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_addr64(ptr %ptr) #0 {1307; CI-LABEL: flat_atomic_inc_noret_i32_offset_addr64:1308; CI: ; %bb.0:1309; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x91310; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v01311; CI-NEXT: v_mov_b32_e32 v2, 421312; CI-NEXT: s_waitcnt lgkmcnt(0)1313; CI-NEXT: v_mov_b32_e32 v1, s11314; CI-NEXT: v_add_i32_e32 v0, vcc, s0, v01315; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1316; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v01317; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1318; CI-NEXT: flat_atomic_inc v[0:1], v21319; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1320; CI-NEXT: buffer_wbinvl1_vol1321; CI-NEXT: s_endpgm1322;1323; VI-LABEL: flat_atomic_inc_noret_i32_offset_addr64:1324; VI: ; %bb.0:1325; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241326; VI-NEXT: v_lshlrev_b32_e32 v0, 2, v01327; VI-NEXT: v_mov_b32_e32 v2, 421328; VI-NEXT: s_waitcnt lgkmcnt(0)1329; VI-NEXT: v_mov_b32_e32 v1, s11330; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v01331; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1332; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v01333; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1334; VI-NEXT: flat_atomic_inc v[0:1], v21335; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1336; VI-NEXT: buffer_wbinvl1_vol1337; VI-NEXT: s_endpgm1338;1339; GFX9-LABEL: flat_atomic_inc_noret_i32_offset_addr64:1340; GFX9: ; %bb.0:1341; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241342; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v01343; GFX9-NEXT: v_mov_b32_e32 v2, 421344; GFX9-NEXT: s_waitcnt lgkmcnt(0)1345; GFX9-NEXT: v_mov_b32_e32 v1, s11346; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v01347; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc1348; GFX9-NEXT: flat_atomic_inc v[0:1], v2 offset:201349; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1350; GFX9-NEXT: buffer_wbinvl1_vol1351; GFX9-NEXT: s_endpgm1352 %id = call i32 @llvm.amdgcn.workitem.id.x()1353 %gep.tid = getelementptr inbounds i32, ptr %ptr, i32 %id1354 %gep = getelementptr inbounds i32, ptr %gep.tid, i32 51355 %result = call i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr %gep, i32 42, i32 0, i32 0, i1 false)1356 ret void1357}1358 1359@lds1 = addrspace(3) global [512 x i64] poison, align 81360 1361define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i64(ptr addrspace(1) %out, ptr addrspace(1) %add_use) #0 {1362; CI-LABEL: atomic_inc_shl_base_lds_0_i64:1363; CI: ; %bb.0:1364; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91365; CI-NEXT: v_mov_b32_e32 v1, 91366; CI-NEXT: v_lshlrev_b32_e32 v3, 3, v01367; CI-NEXT: v_mov_b32_e32 v2, 01368; CI-NEXT: s_mov_b32 m0, -11369; CI-NEXT: s_waitcnt lgkmcnt(0)1370; CI-NEXT: ds_inc_rtn_u64 v[1:2], v3, v[1:2] offset:161371; CI-NEXT: s_waitcnt lgkmcnt(0)1372; CI-NEXT: s_mov_b32 s7, 0xf0001373; CI-NEXT: s_mov_b32 s6, -11374; CI-NEXT: s_mov_b32 s4, s01375; CI-NEXT: s_mov_b32 s5, s11376; CI-NEXT: s_mov_b32 s0, s21377; CI-NEXT: s_mov_b32 s1, s31378; CI-NEXT: s_mov_b32 s2, s61379; CI-NEXT: s_mov_b32 s3, s71380; CI-NEXT: v_add_i32_e32 v0, vcc, 2, v01381; CI-NEXT: buffer_store_dword v0, off, s[0:3], 01382; CI-NEXT: buffer_store_dwordx2 v[1:2], off, s[4:7], 01383; CI-NEXT: s_endpgm1384;1385; VI-LABEL: atomic_inc_shl_base_lds_0_i64:1386; VI: ; %bb.0:1387; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241388; VI-NEXT: v_mov_b32_e32 v1, 91389; VI-NEXT: v_lshlrev_b32_e32 v3, 3, v01390; VI-NEXT: v_mov_b32_e32 v2, 01391; VI-NEXT: s_mov_b32 m0, -11392; VI-NEXT: s_waitcnt lgkmcnt(0)1393; VI-NEXT: ds_inc_rtn_u64 v[1:2], v3, v[1:2] offset:161394; VI-NEXT: s_waitcnt lgkmcnt(0)1395; VI-NEXT: s_mov_b32 s7, 0xf0001396; VI-NEXT: s_mov_b32 s6, -11397; VI-NEXT: s_mov_b32 s4, s01398; VI-NEXT: s_mov_b32 s5, s11399; VI-NEXT: s_mov_b32 s0, s21400; VI-NEXT: s_mov_b32 s1, s31401; VI-NEXT: s_mov_b32 s2, s61402; VI-NEXT: s_mov_b32 s3, s71403; VI-NEXT: v_add_u32_e32 v0, vcc, 2, v01404; VI-NEXT: buffer_store_dword v0, off, s[0:3], 01405; VI-NEXT: buffer_store_dwordx2 v[1:2], off, s[4:7], 01406; VI-NEXT: s_endpgm1407;1408; GFX9-LABEL: atomic_inc_shl_base_lds_0_i64:1409; GFX9: ; %bb.0:1410; GFX9-NEXT: v_mov_b32_e32 v1, 91411; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241412; GFX9-NEXT: v_mov_b32_e32 v2, 01413; GFX9-NEXT: v_lshlrev_b32_e32 v3, 3, v01414; GFX9-NEXT: s_waitcnt lgkmcnt(0)1415; GFX9-NEXT: ds_inc_rtn_u64 v[1:2], v3, v[1:2] offset:161416; GFX9-NEXT: s_waitcnt lgkmcnt(0)1417; GFX9-NEXT: v_mov_b32_e32 v3, 01418; GFX9-NEXT: v_add_u32_e32 v0, 2, v01419; GFX9-NEXT: global_store_dword v3, v0, s[2:3]1420; GFX9-NEXT: global_store_dwordx2 v3, v[1:2], s[0:1]1421; GFX9-NEXT: s_endpgm1422 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #11423 %idx.0 = add nsw i32 %tid.x, 21424 %arrayidx0 = getelementptr inbounds [512 x i64], ptr addrspace(3) @lds1, i32 0, i32 %idx.01425 %val0 = call i64 @llvm.amdgcn.atomic.inc.i64.p3(ptr addrspace(3) %arrayidx0, i64 9, i32 0, i32 0, i1 false)1426 store i32 %idx.0, ptr addrspace(1) %add_use1427 store i64 %val0, ptr addrspace(1) %out1428 ret void1429}1430 1431define amdgpu_kernel void @flat_atomic_inc_ret_i64(ptr %out, ptr %ptr) #0 {1432; CI-LABEL: flat_atomic_inc_ret_i64:1433; CI: ; %bb.0:1434; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91435; CI-NEXT: v_mov_b32_e32 v0, 421436; CI-NEXT: v_mov_b32_e32 v1, 01437; CI-NEXT: s_waitcnt lgkmcnt(0)1438; CI-NEXT: v_mov_b32_e32 v2, s21439; CI-NEXT: v_mov_b32_e32 v3, s31440; CI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc1441; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1442; CI-NEXT: buffer_wbinvl1_vol1443; CI-NEXT: v_mov_b32_e32 v2, s01444; CI-NEXT: v_mov_b32_e32 v3, s11445; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1446; CI-NEXT: s_endpgm1447;1448; VI-LABEL: flat_atomic_inc_ret_i64:1449; VI: ; %bb.0:1450; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241451; VI-NEXT: v_mov_b32_e32 v0, 421452; VI-NEXT: v_mov_b32_e32 v1, 01453; VI-NEXT: s_waitcnt lgkmcnt(0)1454; VI-NEXT: v_mov_b32_e32 v2, s21455; VI-NEXT: v_mov_b32_e32 v3, s31456; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc1457; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1458; VI-NEXT: buffer_wbinvl1_vol1459; VI-NEXT: v_mov_b32_e32 v2, s01460; VI-NEXT: v_mov_b32_e32 v3, s11461; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1462; VI-NEXT: s_endpgm1463;1464; GFX9-LABEL: flat_atomic_inc_ret_i64:1465; GFX9: ; %bb.0:1466; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241467; GFX9-NEXT: v_mov_b32_e32 v0, 421468; GFX9-NEXT: v_mov_b32_e32 v1, 01469; GFX9-NEXT: s_waitcnt lgkmcnt(0)1470; GFX9-NEXT: v_mov_b32_e32 v2, s21471; GFX9-NEXT: v_mov_b32_e32 v3, s31472; GFX9-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc1473; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1474; GFX9-NEXT: buffer_wbinvl1_vol1475; GFX9-NEXT: v_mov_b32_e32 v2, s01476; GFX9-NEXT: v_mov_b32_e32 v3, s11477; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1478; GFX9-NEXT: s_endpgm1479 %result = call i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr %ptr, i64 42, i32 0, i32 0, i1 false)1480 store i64 %result, ptr %out1481 ret void1482}1483 1484define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset(ptr %out, ptr %ptr) #0 {1485; CI-LABEL: flat_atomic_inc_ret_i64_offset:1486; CI: ; %bb.0:1487; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91488; CI-NEXT: v_mov_b32_e32 v0, 421489; CI-NEXT: v_mov_b32_e32 v1, 01490; CI-NEXT: s_waitcnt lgkmcnt(0)1491; CI-NEXT: s_add_u32 s2, s2, 321492; CI-NEXT: s_addc_u32 s3, s3, 01493; CI-NEXT: v_mov_b32_e32 v2, s21494; CI-NEXT: v_mov_b32_e32 v3, s31495; CI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc1496; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1497; CI-NEXT: buffer_wbinvl1_vol1498; CI-NEXT: v_mov_b32_e32 v2, s01499; CI-NEXT: v_mov_b32_e32 v3, s11500; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1501; CI-NEXT: s_endpgm1502;1503; VI-LABEL: flat_atomic_inc_ret_i64_offset:1504; VI: ; %bb.0:1505; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241506; VI-NEXT: v_mov_b32_e32 v0, 421507; VI-NEXT: v_mov_b32_e32 v1, 01508; VI-NEXT: s_waitcnt lgkmcnt(0)1509; VI-NEXT: s_add_u32 s2, s2, 321510; VI-NEXT: s_addc_u32 s3, s3, 01511; VI-NEXT: v_mov_b32_e32 v2, s21512; VI-NEXT: v_mov_b32_e32 v3, s31513; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc1514; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1515; VI-NEXT: buffer_wbinvl1_vol1516; VI-NEXT: v_mov_b32_e32 v2, s01517; VI-NEXT: v_mov_b32_e32 v3, s11518; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1519; VI-NEXT: s_endpgm1520;1521; GFX9-LABEL: flat_atomic_inc_ret_i64_offset:1522; GFX9: ; %bb.0:1523; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241524; GFX9-NEXT: v_mov_b32_e32 v0, 421525; GFX9-NEXT: v_mov_b32_e32 v1, 01526; GFX9-NEXT: s_waitcnt lgkmcnt(0)1527; GFX9-NEXT: v_mov_b32_e32 v2, s21528; GFX9-NEXT: v_mov_b32_e32 v3, s31529; GFX9-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] offset:32 glc1530; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1531; GFX9-NEXT: buffer_wbinvl1_vol1532; GFX9-NEXT: v_mov_b32_e32 v2, s01533; GFX9-NEXT: v_mov_b32_e32 v3, s11534; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1535; GFX9-NEXT: s_endpgm1536 %gep = getelementptr inbounds i64, ptr %ptr, i32 41537 %result = call i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr %gep, i64 42, i32 0, i32 0, i1 false)1538 store i64 %result, ptr %out1539 ret void1540}1541 1542define amdgpu_kernel void @flat_atomic_inc_noret_i64(ptr %ptr) nounwind {1543; CI-LABEL: flat_atomic_inc_noret_i64:1544; CI: ; %bb.0:1545; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x91546; CI-NEXT: v_mov_b32_e32 v0, 421547; CI-NEXT: v_mov_b32_e32 v1, 01548; CI-NEXT: s_waitcnt lgkmcnt(0)1549; CI-NEXT: v_mov_b32_e32 v3, s11550; CI-NEXT: v_mov_b32_e32 v2, s01551; CI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]1552; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1553; CI-NEXT: buffer_wbinvl1_vol1554; CI-NEXT: s_endpgm1555;1556; VI-LABEL: flat_atomic_inc_noret_i64:1557; VI: ; %bb.0:1558; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241559; VI-NEXT: v_mov_b32_e32 v0, 421560; VI-NEXT: v_mov_b32_e32 v1, 01561; VI-NEXT: s_waitcnt lgkmcnt(0)1562; VI-NEXT: v_mov_b32_e32 v3, s11563; VI-NEXT: v_mov_b32_e32 v2, s01564; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]1565; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1566; VI-NEXT: buffer_wbinvl1_vol1567; VI-NEXT: s_endpgm1568;1569; GFX9-LABEL: flat_atomic_inc_noret_i64:1570; GFX9: ; %bb.0:1571; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241572; GFX9-NEXT: v_mov_b32_e32 v0, 421573; GFX9-NEXT: v_mov_b32_e32 v1, 01574; GFX9-NEXT: s_waitcnt lgkmcnt(0)1575; GFX9-NEXT: v_mov_b32_e32 v3, s11576; GFX9-NEXT: v_mov_b32_e32 v2, s01577; GFX9-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]1578; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1579; GFX9-NEXT: buffer_wbinvl1_vol1580; GFX9-NEXT: s_endpgm1581 %result = call i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr %ptr, i64 42, i32 0, i32 0, i1 false)1582 ret void1583}1584 1585define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset(ptr %ptr) nounwind {1586; CI-LABEL: flat_atomic_inc_noret_i64_offset:1587; CI: ; %bb.0:1588; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x91589; CI-NEXT: v_mov_b32_e32 v0, 421590; CI-NEXT: v_mov_b32_e32 v1, 01591; CI-NEXT: s_waitcnt lgkmcnt(0)1592; CI-NEXT: s_add_u32 s0, s0, 321593; CI-NEXT: s_addc_u32 s1, s1, 01594; CI-NEXT: v_mov_b32_e32 v3, s11595; CI-NEXT: v_mov_b32_e32 v2, s01596; CI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]1597; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1598; CI-NEXT: buffer_wbinvl1_vol1599; CI-NEXT: s_endpgm1600;1601; VI-LABEL: flat_atomic_inc_noret_i64_offset:1602; VI: ; %bb.0:1603; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241604; VI-NEXT: v_mov_b32_e32 v0, 421605; VI-NEXT: v_mov_b32_e32 v1, 01606; VI-NEXT: s_waitcnt lgkmcnt(0)1607; VI-NEXT: s_add_u32 s0, s0, 321608; VI-NEXT: s_addc_u32 s1, s1, 01609; VI-NEXT: v_mov_b32_e32 v3, s11610; VI-NEXT: v_mov_b32_e32 v2, s01611; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]1612; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1613; VI-NEXT: buffer_wbinvl1_vol1614; VI-NEXT: s_endpgm1615;1616; GFX9-LABEL: flat_atomic_inc_noret_i64_offset:1617; GFX9: ; %bb.0:1618; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241619; GFX9-NEXT: v_mov_b32_e32 v0, 421620; GFX9-NEXT: v_mov_b32_e32 v1, 01621; GFX9-NEXT: s_waitcnt lgkmcnt(0)1622; GFX9-NEXT: v_mov_b32_e32 v3, s11623; GFX9-NEXT: v_mov_b32_e32 v2, s01624; GFX9-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] offset:321625; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1626; GFX9-NEXT: buffer_wbinvl1_vol1627; GFX9-NEXT: s_endpgm1628 %gep = getelementptr inbounds i64, ptr %ptr, i32 41629 %result = call i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr %gep, i64 42, i32 0, i32 0, i1 false)1630 ret void1631}1632 1633define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_addr64(ptr %out, ptr %ptr) #0 {1634; CI-LABEL: flat_atomic_inc_ret_i64_offset_addr64:1635; CI: ; %bb.0:1636; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91637; CI-NEXT: v_lshlrev_b32_e32 v4, 3, v01638; CI-NEXT: s_waitcnt lgkmcnt(0)1639; CI-NEXT: v_mov_b32_e32 v0, s31640; CI-NEXT: v_add_i32_e32 v2, vcc, s2, v41641; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v0, vcc1642; CI-NEXT: v_mov_b32_e32 v0, 421643; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v21644; CI-NEXT: v_mov_b32_e32 v1, 01645; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc1646; CI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc1647; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1648; CI-NEXT: buffer_wbinvl1_vol1649; CI-NEXT: v_mov_b32_e32 v3, s11650; CI-NEXT: v_add_i32_e32 v2, vcc, s0, v41651; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc1652; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1653; CI-NEXT: s_endpgm1654;1655; VI-LABEL: flat_atomic_inc_ret_i64_offset_addr64:1656; VI: ; %bb.0:1657; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241658; VI-NEXT: v_lshlrev_b32_e32 v4, 3, v01659; VI-NEXT: s_waitcnt lgkmcnt(0)1660; VI-NEXT: v_mov_b32_e32 v0, s31661; VI-NEXT: v_add_u32_e32 v2, vcc, s2, v41662; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v0, vcc1663; VI-NEXT: v_mov_b32_e32 v0, 421664; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v21665; VI-NEXT: v_mov_b32_e32 v1, 01666; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc1667; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc1668; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1669; VI-NEXT: buffer_wbinvl1_vol1670; VI-NEXT: v_mov_b32_e32 v3, s11671; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v41672; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc1673; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1674; VI-NEXT: s_endpgm1675;1676; GFX9-LABEL: flat_atomic_inc_ret_i64_offset_addr64:1677; GFX9: ; %bb.0:1678; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241679; GFX9-NEXT: v_lshlrev_b32_e32 v5, 3, v01680; GFX9-NEXT: v_mov_b32_e32 v1, 421681; GFX9-NEXT: v_mov_b32_e32 v2, 01682; GFX9-NEXT: s_waitcnt lgkmcnt(0)1683; GFX9-NEXT: v_mov_b32_e32 v0, s31684; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, s2, v51685; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v0, vcc1686; GFX9-NEXT: flat_atomic_inc_x2 v[0:1], v[3:4], v[1:2] offset:40 glc1687; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1688; GFX9-NEXT: buffer_wbinvl1_vol1689; GFX9-NEXT: v_mov_b32_e32 v3, s11690; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, s0, v51691; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc1692; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1693; GFX9-NEXT: s_endpgm1694 %id = call i32 @llvm.amdgcn.workitem.id.x()1695 %gep.tid = getelementptr inbounds i64, ptr %ptr, i32 %id1696 %out.gep = getelementptr inbounds i64, ptr %out, i32 %id1697 %gep = getelementptr inbounds i64, ptr %gep.tid, i32 51698 %result = call i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr %gep, i64 42, i32 0, i32 0, i1 false)1699 store i64 %result, ptr %out.gep1700 ret void1701}1702 1703define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_addr64(ptr %ptr) #0 {1704; CI-LABEL: flat_atomic_inc_noret_i64_offset_addr64:1705; CI: ; %bb.0:1706; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x91707; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v01708; CI-NEXT: s_waitcnt lgkmcnt(0)1709; CI-NEXT: v_mov_b32_e32 v1, s11710; CI-NEXT: v_add_i32_e32 v2, vcc, s0, v01711; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc1712; CI-NEXT: v_mov_b32_e32 v0, 421713; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v21714; CI-NEXT: v_mov_b32_e32 v1, 01715; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc1716; CI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]1717; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1718; CI-NEXT: buffer_wbinvl1_vol1719; CI-NEXT: s_endpgm1720;1721; VI-LABEL: flat_atomic_inc_noret_i64_offset_addr64:1722; VI: ; %bb.0:1723; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241724; VI-NEXT: v_lshlrev_b32_e32 v0, 3, v01725; VI-NEXT: s_waitcnt lgkmcnt(0)1726; VI-NEXT: v_mov_b32_e32 v1, s11727; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v01728; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc1729; VI-NEXT: v_mov_b32_e32 v0, 421730; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v21731; VI-NEXT: v_mov_b32_e32 v1, 01732; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc1733; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]1734; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1735; VI-NEXT: buffer_wbinvl1_vol1736; VI-NEXT: s_endpgm1737;1738; GFX9-LABEL: flat_atomic_inc_noret_i64_offset_addr64:1739; GFX9: ; %bb.0:1740; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241741; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v01742; GFX9-NEXT: v_mov_b32_e32 v1, 421743; GFX9-NEXT: v_mov_b32_e32 v2, 01744; GFX9-NEXT: s_waitcnt lgkmcnt(0)1745; GFX9-NEXT: v_mov_b32_e32 v4, s11746; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, s0, v01747; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc1748; GFX9-NEXT: flat_atomic_inc_x2 v[3:4], v[1:2] offset:401749; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1750; GFX9-NEXT: buffer_wbinvl1_vol1751; GFX9-NEXT: s_endpgm1752 %id = call i32 @llvm.amdgcn.workitem.id.x()1753 %gep.tid = getelementptr inbounds i64, ptr %ptr, i32 %id1754 %gep = getelementptr inbounds i64, ptr %gep.tid, i32 51755 %result = call i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr %gep, i64 42, i32 0, i32 0, i1 false)1756 ret void1757}1758 1759define amdgpu_kernel void @nocse_lds_atomic_inc_ret_i32(ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(3) %ptr) #0 {1760; CI-LABEL: nocse_lds_atomic_inc_ret_i32:1761; CI: ; %bb.0:1762; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91763; CI-NEXT: s_load_dword s8, s[4:5], 0xd1764; CI-NEXT: v_mov_b32_e32 v0, 421765; CI-NEXT: s_mov_b32 m0, -11766; CI-NEXT: s_mov_b32 s7, 0xf0001767; CI-NEXT: s_mov_b32 s6, -11768; CI-NEXT: s_waitcnt lgkmcnt(0)1769; CI-NEXT: v_mov_b32_e32 v1, s81770; CI-NEXT: ds_inc_rtn_u32 v2, v1, v01771; CI-NEXT: s_waitcnt lgkmcnt(0)1772; CI-NEXT: ds_inc_rtn_u32 v0, v1, v01773; CI-NEXT: s_waitcnt lgkmcnt(0)1774; CI-NEXT: s_mov_b32 s4, s01775; CI-NEXT: s_mov_b32 s5, s11776; CI-NEXT: s_mov_b32 s0, s21777; CI-NEXT: s_mov_b32 s1, s31778; CI-NEXT: s_mov_b32 s2, s61779; CI-NEXT: s_mov_b32 s3, s71780; CI-NEXT: buffer_store_dword v2, off, s[4:7], 01781; CI-NEXT: buffer_store_dword v0, off, s[0:3], 01782; CI-NEXT: s_endpgm1783;1784; VI-LABEL: nocse_lds_atomic_inc_ret_i32:1785; VI: ; %bb.0:1786; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241787; VI-NEXT: s_load_dword s8, s[4:5], 0x341788; VI-NEXT: v_mov_b32_e32 v0, 421789; VI-NEXT: s_mov_b32 m0, -11790; VI-NEXT: s_mov_b32 s7, 0xf0001791; VI-NEXT: s_mov_b32 s6, -11792; VI-NEXT: s_waitcnt lgkmcnt(0)1793; VI-NEXT: v_mov_b32_e32 v1, s81794; VI-NEXT: ds_inc_rtn_u32 v2, v1, v01795; VI-NEXT: s_waitcnt lgkmcnt(0)1796; VI-NEXT: ds_inc_rtn_u32 v0, v1, v01797; VI-NEXT: s_waitcnt lgkmcnt(0)1798; VI-NEXT: s_mov_b32 s4, s01799; VI-NEXT: s_mov_b32 s5, s11800; VI-NEXT: s_mov_b32 s0, s21801; VI-NEXT: s_mov_b32 s1, s31802; VI-NEXT: s_mov_b32 s2, s61803; VI-NEXT: s_mov_b32 s3, s71804; VI-NEXT: buffer_store_dword v2, off, s[4:7], 01805; VI-NEXT: buffer_store_dword v0, off, s[0:3], 01806; VI-NEXT: s_endpgm1807;1808; GFX9-LABEL: nocse_lds_atomic_inc_ret_i32:1809; GFX9: ; %bb.0:1810; GFX9-NEXT: s_load_dword s6, s[4:5], 0x341811; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241812; GFX9-NEXT: v_mov_b32_e32 v0, 421813; GFX9-NEXT: s_waitcnt lgkmcnt(0)1814; GFX9-NEXT: v_mov_b32_e32 v1, s61815; GFX9-NEXT: ds_inc_rtn_u32 v2, v1, v01816; GFX9-NEXT: s_waitcnt lgkmcnt(0)1817; GFX9-NEXT: ds_inc_rtn_u32 v0, v1, v01818; GFX9-NEXT: s_waitcnt lgkmcnt(0)1819; GFX9-NEXT: v_mov_b32_e32 v1, 01820; GFX9-NEXT: global_store_dword v1, v2, s[0:1]1821; GFX9-NEXT: global_store_dword v1, v0, s[2:3]1822; GFX9-NEXT: s_endpgm1823 %result0 = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 0, i32 0, i1 false)1824 %result1 = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 0, i32 0, i1 false)1825 1826 store i32 %result0, ptr addrspace(1) %out01827 store i32 %result1, ptr addrspace(1) %out11828 ret void1829}1830 1831attributes #0 = { nounwind }1832attributes #1 = { nounwind readnone }1833attributes #2 = { nounwind argmemonly }1834