4441 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=bonaire -mattr=+flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s3; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=tonga < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI %s4; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s5; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX10 %s6; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11 %s7; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck -enable-var-scope -check-prefixes=GFX12 %s8 9; FIXME: Merge with other test. DS offset folding doesn't work due to10; register bank copies, and no return optimization is missing.11 12@lds0 = internal addrspace(3) global [512 x i32] poison, align 413@lds1 = internal addrspace(3) global [512 x i64] poison, align 814 15declare i32 @llvm.amdgcn.workitem.id.x() #016 17define amdgpu_kernel void @lds_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {18; CI-LABEL: lds_atomic_inc_ret_i32:19; CI: ; %bb.0:20; CI-NEXT: s_load_dword s2, s[8:9], 0x221; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x022; CI-NEXT: v_mov_b32_e32 v0, 4223; CI-NEXT: s_mov_b32 m0, -124; CI-NEXT: s_add_i32 s12, s12, s1725; CI-NEXT: s_waitcnt lgkmcnt(0)26; CI-NEXT: v_mov_b32_e32 v1, s227; CI-NEXT: ds_inc_rtn_u32 v2, v1, v028; CI-NEXT: s_waitcnt lgkmcnt(0)29; CI-NEXT: v_mov_b32_e32 v0, s030; CI-NEXT: s_mov_b32 flat_scratch_lo, s1331; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 832; CI-NEXT: v_mov_b32_e32 v1, s133; CI-NEXT: flat_store_dword v[0:1], v234; CI-NEXT: s_endpgm35;36; VI-LABEL: lds_atomic_inc_ret_i32:37; VI: ; %bb.0:38; VI-NEXT: s_load_dword s2, s[8:9], 0x839; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x040; VI-NEXT: v_mov_b32_e32 v0, 4241; VI-NEXT: s_mov_b32 m0, -142; VI-NEXT: s_add_i32 s12, s12, s1743; VI-NEXT: s_waitcnt lgkmcnt(0)44; VI-NEXT: v_mov_b32_e32 v1, s245; VI-NEXT: ds_inc_rtn_u32 v2, v1, v046; VI-NEXT: s_waitcnt lgkmcnt(0)47; VI-NEXT: v_mov_b32_e32 v0, s048; VI-NEXT: s_mov_b32 flat_scratch_lo, s1349; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 850; VI-NEXT: v_mov_b32_e32 v1, s151; VI-NEXT: flat_store_dword v[0:1], v252; VI-NEXT: s_endpgm53;54; GFX9-LABEL: lds_atomic_inc_ret_i32:55; GFX9: ; %bb.0:56; GFX9-NEXT: s_load_dword s2, s[8:9], 0x857; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x058; GFX9-NEXT: v_mov_b32_e32 v1, 4259; GFX9-NEXT: s_waitcnt lgkmcnt(0)60; GFX9-NEXT: v_mov_b32_e32 v0, s261; GFX9-NEXT: ds_inc_rtn_u32 v0, v0, v162; GFX9-NEXT: s_waitcnt lgkmcnt(0)63; GFX9-NEXT: v_mov_b32_e32 v1, 064; GFX9-NEXT: global_store_dword v1, v0, s[0:1]65; GFX9-NEXT: s_endpgm66;67; GFX10-LABEL: lds_atomic_inc_ret_i32:68; GFX10: ; %bb.0:69; GFX10-NEXT: s_load_dword s0, s[8:9], 0x870; GFX10-NEXT: v_mov_b32_e32 v1, 4271; GFX10-NEXT: s_waitcnt lgkmcnt(0)72; GFX10-NEXT: v_mov_b32_e32 v0, s073; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x074; GFX10-NEXT: s_waitcnt lgkmcnt(0)75; GFX10-NEXT: ds_inc_rtn_u32 v0, v0, v176; GFX10-NEXT: s_waitcnt lgkmcnt(0)77; GFX10-NEXT: buffer_gl0_inv78; GFX10-NEXT: v_mov_b32_e32 v1, 079; GFX10-NEXT: global_store_dword v1, v0, s[0:1]80; GFX10-NEXT: s_endpgm81;82; GFX11-LABEL: lds_atomic_inc_ret_i32:83; GFX11: ; %bb.0:84; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x885; GFX11-NEXT: s_waitcnt lgkmcnt(0)86; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s087; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x088; GFX11-NEXT: s_waitcnt lgkmcnt(0)89; GFX11-NEXT: ds_inc_rtn_u32 v0, v0, v190; GFX11-NEXT: s_waitcnt lgkmcnt(0)91; GFX11-NEXT: buffer_gl0_inv92; GFX11-NEXT: v_mov_b32_e32 v1, 093; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]94; GFX11-NEXT: s_endpgm95;96; GFX12-LABEL: lds_atomic_inc_ret_i32:97; GFX12: ; %bb.0:98; GFX12-NEXT: s_load_b96 s[0:2], s[4:5], 0x099; GFX12-NEXT: s_wait_kmcnt 0x0100; GFX12-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s2101; GFX12-NEXT: ds_inc_rtn_u32 v0, v0, v1102; GFX12-NEXT: s_wait_dscnt 0x0103; GFX12-NEXT: global_inv scope:SCOPE_SE104; GFX12-NEXT: v_mov_b32_e32 v1, 0105; GFX12-NEXT: global_store_b32 v1, v0, s[0:1]106; GFX12-NEXT: s_endpgm107 %result = atomicrmw uinc_wrap ptr addrspace(3) %ptr, i32 42 syncscope("agent") seq_cst, align 4108 store i32 %result, ptr addrspace(1) %out, align 4109 ret void110}111 112define amdgpu_kernel void @lds_atomic_inc_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {113; CI-LABEL: lds_atomic_inc_ret_i32_offset:114; CI: ; %bb.0:115; CI-NEXT: s_load_dword s2, s[8:9], 0x2116; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0117; CI-NEXT: v_mov_b32_e32 v0, 42118; CI-NEXT: s_mov_b32 m0, -1119; CI-NEXT: s_add_i32 s12, s12, s17120; CI-NEXT: s_waitcnt lgkmcnt(0)121; CI-NEXT: v_mov_b32_e32 v1, s2122; CI-NEXT: ds_inc_rtn_u32 v2, v1, v0 offset:16123; CI-NEXT: s_waitcnt lgkmcnt(0)124; CI-NEXT: v_mov_b32_e32 v0, s0125; CI-NEXT: s_mov_b32 flat_scratch_lo, s13126; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8127; CI-NEXT: v_mov_b32_e32 v1, s1128; CI-NEXT: flat_store_dword v[0:1], v2129; CI-NEXT: s_endpgm130;131; VI-LABEL: lds_atomic_inc_ret_i32_offset:132; VI: ; %bb.0:133; VI-NEXT: s_load_dword s2, s[8:9], 0x8134; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0135; VI-NEXT: v_mov_b32_e32 v0, 42136; VI-NEXT: s_mov_b32 m0, -1137; VI-NEXT: s_add_i32 s12, s12, s17138; VI-NEXT: s_waitcnt lgkmcnt(0)139; VI-NEXT: v_mov_b32_e32 v1, s2140; VI-NEXT: ds_inc_rtn_u32 v2, v1, v0 offset:16141; VI-NEXT: s_waitcnt lgkmcnt(0)142; VI-NEXT: v_mov_b32_e32 v0, s0143; VI-NEXT: s_mov_b32 flat_scratch_lo, s13144; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8145; VI-NEXT: v_mov_b32_e32 v1, s1146; VI-NEXT: flat_store_dword v[0:1], v2147; VI-NEXT: s_endpgm148;149; GFX9-LABEL: lds_atomic_inc_ret_i32_offset:150; GFX9: ; %bb.0:151; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8152; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0153; GFX9-NEXT: v_mov_b32_e32 v0, 42154; GFX9-NEXT: s_waitcnt lgkmcnt(0)155; GFX9-NEXT: v_mov_b32_e32 v1, s2156; GFX9-NEXT: ds_inc_rtn_u32 v0, v1, v0 offset:16157; GFX9-NEXT: s_waitcnt lgkmcnt(0)158; GFX9-NEXT: v_mov_b32_e32 v1, 0159; GFX9-NEXT: global_store_dword v1, v0, s[0:1]160; GFX9-NEXT: s_endpgm161;162; GFX10-LABEL: lds_atomic_inc_ret_i32_offset:163; GFX10: ; %bb.0:164; GFX10-NEXT: s_load_dword s0, s[8:9], 0x8165; GFX10-NEXT: v_mov_b32_e32 v0, 42166; GFX10-NEXT: s_waitcnt lgkmcnt(0)167; GFX10-NEXT: v_mov_b32_e32 v1, s0168; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0169; GFX10-NEXT: s_waitcnt lgkmcnt(0)170; GFX10-NEXT: ds_inc_rtn_u32 v0, v1, v0 offset:16171; GFX10-NEXT: s_waitcnt lgkmcnt(0)172; GFX10-NEXT: buffer_gl0_inv173; GFX10-NEXT: v_mov_b32_e32 v1, 0174; GFX10-NEXT: global_store_dword v1, v0, s[0:1]175; GFX10-NEXT: s_endpgm176;177; GFX11-LABEL: lds_atomic_inc_ret_i32_offset:178; GFX11: ; %bb.0:179; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8180; GFX11-NEXT: s_waitcnt lgkmcnt(0)181; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, s0182; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0183; GFX11-NEXT: s_waitcnt lgkmcnt(0)184; GFX11-NEXT: ds_inc_rtn_u32 v0, v1, v0 offset:16185; GFX11-NEXT: s_waitcnt lgkmcnt(0)186; GFX11-NEXT: buffer_gl0_inv187; GFX11-NEXT: v_mov_b32_e32 v1, 0188; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]189; GFX11-NEXT: s_endpgm190;191; GFX12-LABEL: lds_atomic_inc_ret_i32_offset:192; GFX12: ; %bb.0:193; GFX12-NEXT: s_load_b96 s[0:2], s[4:5], 0x0194; GFX12-NEXT: s_wait_kmcnt 0x0195; GFX12-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, s2196; GFX12-NEXT: ds_inc_rtn_u32 v0, v1, v0 offset:16197; GFX12-NEXT: s_wait_dscnt 0x0198; GFX12-NEXT: global_inv scope:SCOPE_SE199; GFX12-NEXT: v_mov_b32_e32 v1, 0200; GFX12-NEXT: global_store_b32 v1, v0, s[0:1]201; GFX12-NEXT: s_endpgm202 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4203 %result = atomicrmw uinc_wrap ptr addrspace(3) %gep, i32 42 syncscope("agent") seq_cst, align 4204 store i32 %result, ptr addrspace(1) %out, align 4205 ret void206}207 208define amdgpu_kernel void @lds_atomic_inc_noret_i32(ptr addrspace(3) %ptr) #1 {209; CI-LABEL: lds_atomic_inc_noret_i32:210; CI: ; %bb.0:211; CI-NEXT: s_load_dword s0, s[8:9], 0x0212; CI-NEXT: v_mov_b32_e32 v0, 42213; CI-NEXT: s_mov_b32 m0, -1214; CI-NEXT: s_waitcnt lgkmcnt(0)215; CI-NEXT: v_mov_b32_e32 v1, s0216; CI-NEXT: ds_inc_u32 v1, v0217; CI-NEXT: s_waitcnt lgkmcnt(0)218; CI-NEXT: s_endpgm219;220; VI-LABEL: lds_atomic_inc_noret_i32:221; VI: ; %bb.0:222; VI-NEXT: s_load_dword s0, s[8:9], 0x0223; VI-NEXT: v_mov_b32_e32 v0, 42224; VI-NEXT: s_mov_b32 m0, -1225; VI-NEXT: s_waitcnt lgkmcnt(0)226; VI-NEXT: v_mov_b32_e32 v1, s0227; VI-NEXT: ds_inc_u32 v1, v0228; VI-NEXT: s_waitcnt lgkmcnt(0)229; VI-NEXT: s_endpgm230;231; GFX9-LABEL: lds_atomic_inc_noret_i32:232; GFX9: ; %bb.0:233; GFX9-NEXT: s_load_dword s0, s[8:9], 0x0234; GFX9-NEXT: v_mov_b32_e32 v1, 42235; GFX9-NEXT: s_waitcnt lgkmcnt(0)236; GFX9-NEXT: v_mov_b32_e32 v0, s0237; GFX9-NEXT: ds_inc_u32 v0, v1238; GFX9-NEXT: s_waitcnt lgkmcnt(0)239; GFX9-NEXT: s_endpgm240;241; GFX10-LABEL: lds_atomic_inc_noret_i32:242; GFX10: ; %bb.0:243; GFX10-NEXT: s_load_dword s0, s[8:9], 0x0244; GFX10-NEXT: v_mov_b32_e32 v1, 42245; GFX10-NEXT: s_waitcnt lgkmcnt(0)246; GFX10-NEXT: v_mov_b32_e32 v0, s0247; GFX10-NEXT: ds_inc_u32 v0, v1248; GFX10-NEXT: s_waitcnt lgkmcnt(0)249; GFX10-NEXT: buffer_gl0_inv250; GFX10-NEXT: s_endpgm251;252; GFX11-LABEL: lds_atomic_inc_noret_i32:253; GFX11: ; %bb.0:254; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x0255; GFX11-NEXT: s_waitcnt lgkmcnt(0)256; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s0257; GFX11-NEXT: ds_inc_u32 v0, v1258; GFX11-NEXT: s_waitcnt lgkmcnt(0)259; GFX11-NEXT: buffer_gl0_inv260; GFX11-NEXT: s_endpgm261;262; GFX12-LABEL: lds_atomic_inc_noret_i32:263; GFX12: ; %bb.0:264; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x0265; GFX12-NEXT: s_wait_kmcnt 0x0266; GFX12-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s0267; GFX12-NEXT: ds_inc_u32 v0, v1268; GFX12-NEXT: s_wait_dscnt 0x0269; GFX12-NEXT: global_inv scope:SCOPE_SE270; GFX12-NEXT: s_endpgm271 %result = atomicrmw uinc_wrap ptr addrspace(3) %ptr, i32 42 syncscope("agent") seq_cst, align 4272 ret void273}274 275define amdgpu_kernel void @lds_atomic_inc_noret_i32_offset(ptr addrspace(3) %ptr) #1 {276; CI-LABEL: lds_atomic_inc_noret_i32_offset:277; CI: ; %bb.0:278; CI-NEXT: s_load_dword s0, s[8:9], 0x0279; CI-NEXT: v_mov_b32_e32 v0, 42280; CI-NEXT: s_mov_b32 m0, -1281; CI-NEXT: s_waitcnt lgkmcnt(0)282; CI-NEXT: v_mov_b32_e32 v1, s0283; CI-NEXT: ds_inc_u32 v1, v0 offset:16284; CI-NEXT: s_waitcnt lgkmcnt(0)285; CI-NEXT: s_endpgm286;287; VI-LABEL: lds_atomic_inc_noret_i32_offset:288; VI: ; %bb.0:289; VI-NEXT: s_load_dword s0, s[8:9], 0x0290; VI-NEXT: v_mov_b32_e32 v0, 42291; VI-NEXT: s_mov_b32 m0, -1292; VI-NEXT: s_waitcnt lgkmcnt(0)293; VI-NEXT: v_mov_b32_e32 v1, s0294; VI-NEXT: ds_inc_u32 v1, v0 offset:16295; VI-NEXT: s_waitcnt lgkmcnt(0)296; VI-NEXT: s_endpgm297;298; GFX9-LABEL: lds_atomic_inc_noret_i32_offset:299; GFX9: ; %bb.0:300; GFX9-NEXT: s_load_dword s0, s[8:9], 0x0301; GFX9-NEXT: v_mov_b32_e32 v0, 42302; GFX9-NEXT: s_waitcnt lgkmcnt(0)303; GFX9-NEXT: v_mov_b32_e32 v1, s0304; GFX9-NEXT: ds_inc_u32 v1, v0 offset:16305; GFX9-NEXT: s_waitcnt lgkmcnt(0)306; GFX9-NEXT: s_endpgm307;308; GFX10-LABEL: lds_atomic_inc_noret_i32_offset:309; GFX10: ; %bb.0:310; GFX10-NEXT: s_load_dword s0, s[8:9], 0x0311; GFX10-NEXT: v_mov_b32_e32 v0, 42312; GFX10-NEXT: s_waitcnt lgkmcnt(0)313; GFX10-NEXT: v_mov_b32_e32 v1, s0314; GFX10-NEXT: ds_inc_u32 v1, v0 offset:16315; GFX10-NEXT: s_waitcnt lgkmcnt(0)316; GFX10-NEXT: buffer_gl0_inv317; GFX10-NEXT: s_endpgm318;319; GFX11-LABEL: lds_atomic_inc_noret_i32_offset:320; GFX11: ; %bb.0:321; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x0322; GFX11-NEXT: s_waitcnt lgkmcnt(0)323; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, s0324; GFX11-NEXT: ds_inc_u32 v1, v0 offset:16325; GFX11-NEXT: s_waitcnt lgkmcnt(0)326; GFX11-NEXT: buffer_gl0_inv327; GFX11-NEXT: s_endpgm328;329; GFX12-LABEL: lds_atomic_inc_noret_i32_offset:330; GFX12: ; %bb.0:331; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x0332; GFX12-NEXT: s_wait_kmcnt 0x0333; GFX12-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, s0334; GFX12-NEXT: ds_inc_u32 v1, v0 offset:16335; GFX12-NEXT: s_wait_dscnt 0x0336; GFX12-NEXT: global_inv scope:SCOPE_SE337; GFX12-NEXT: s_endpgm338 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4339 %result = atomicrmw uinc_wrap ptr addrspace(3) %gep, i32 42 syncscope("agent") seq_cst, align 4340 ret void341}342 343define amdgpu_kernel void @global_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #1 {344; CI-LABEL: global_atomic_inc_ret_i32:345; CI: ; %bb.0:346; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0347; CI-NEXT: s_add_i32 s12, s12, s17348; CI-NEXT: s_mov_b32 flat_scratch_lo, s13349; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8350; CI-NEXT: v_mov_b32_e32 v2, 42351; CI-NEXT: s_waitcnt lgkmcnt(0)352; CI-NEXT: v_mov_b32_e32 v0, s2353; CI-NEXT: v_mov_b32_e32 v1, s3354; CI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc355; CI-NEXT: s_waitcnt vmcnt(0)356; CI-NEXT: buffer_wbinvl1_vol357; CI-NEXT: v_mov_b32_e32 v0, s0358; CI-NEXT: v_mov_b32_e32 v1, s1359; CI-NEXT: flat_store_dword v[0:1], v2360; CI-NEXT: s_endpgm361;362; VI-LABEL: global_atomic_inc_ret_i32:363; VI: ; %bb.0:364; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0365; VI-NEXT: s_add_i32 s12, s12, s17366; VI-NEXT: s_mov_b32 flat_scratch_lo, s13367; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8368; VI-NEXT: v_mov_b32_e32 v2, 42369; VI-NEXT: s_waitcnt lgkmcnt(0)370; VI-NEXT: v_mov_b32_e32 v0, s2371; VI-NEXT: v_mov_b32_e32 v1, s3372; VI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc373; VI-NEXT: s_waitcnt vmcnt(0)374; VI-NEXT: buffer_wbinvl1_vol375; VI-NEXT: v_mov_b32_e32 v0, s0376; VI-NEXT: v_mov_b32_e32 v1, s1377; VI-NEXT: flat_store_dword v[0:1], v2378; VI-NEXT: s_endpgm379;380; GFX9-LABEL: global_atomic_inc_ret_i32:381; GFX9: ; %bb.0:382; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0383; GFX9-NEXT: v_mov_b32_e32 v0, 42384; GFX9-NEXT: v_mov_b32_e32 v1, 0385; GFX9-NEXT: s_waitcnt lgkmcnt(0)386; GFX9-NEXT: global_atomic_inc v0, v1, v0, s[2:3] glc387; GFX9-NEXT: s_waitcnt vmcnt(0)388; GFX9-NEXT: buffer_wbinvl1_vol389; GFX9-NEXT: global_store_dword v1, v0, s[0:1]390; GFX9-NEXT: s_endpgm391;392; GFX10-LABEL: global_atomic_inc_ret_i32:393; GFX10: ; %bb.0:394; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0395; GFX10-NEXT: v_mov_b32_e32 v0, 42396; GFX10-NEXT: v_mov_b32_e32 v1, 0397; GFX10-NEXT: s_waitcnt lgkmcnt(0)398; GFX10-NEXT: global_atomic_inc v0, v1, v0, s[2:3] glc399; GFX10-NEXT: s_waitcnt vmcnt(0)400; GFX10-NEXT: buffer_gl1_inv401; GFX10-NEXT: buffer_gl0_inv402; GFX10-NEXT: global_store_dword v1, v0, s[0:1]403; GFX10-NEXT: s_endpgm404;405; GFX11-LABEL: global_atomic_inc_ret_i32:406; GFX11: ; %bb.0:407; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0408; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0409; GFX11-NEXT: s_waitcnt lgkmcnt(0)410; GFX11-NEXT: global_atomic_inc_u32 v0, v1, v0, s[2:3] glc411; GFX11-NEXT: s_waitcnt vmcnt(0)412; GFX11-NEXT: buffer_gl1_inv413; GFX11-NEXT: buffer_gl0_inv414; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]415; GFX11-NEXT: s_endpgm416;417; GFX12-LABEL: global_atomic_inc_ret_i32:418; GFX12: ; %bb.0:419; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x0420; GFX12-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0421; GFX12-NEXT: s_wait_kmcnt 0x0422; GFX12-NEXT: global_atomic_inc_u32 v0, v1, v0, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV423; GFX12-NEXT: s_wait_loadcnt 0x0424; GFX12-NEXT: global_inv scope:SCOPE_DEV425; GFX12-NEXT: global_store_b32 v1, v0, s[0:1]426; GFX12-NEXT: s_endpgm427 %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1428 store i32 %result, ptr addrspace(1) %out, align 4429 ret void430}431 432define amdgpu_kernel void @global_atomic_inc_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #1 {433; CI-LABEL: global_atomic_inc_ret_i32_offset:434; CI: ; %bb.0:435; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0436; CI-NEXT: s_add_i32 s12, s12, s17437; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8438; CI-NEXT: s_mov_b32 flat_scratch_lo, s13439; CI-NEXT: v_mov_b32_e32 v2, 42440; CI-NEXT: s_waitcnt lgkmcnt(0)441; CI-NEXT: s_add_u32 s2, s2, 16442; CI-NEXT: s_addc_u32 s3, s3, 0443; CI-NEXT: v_mov_b32_e32 v0, s2444; CI-NEXT: v_mov_b32_e32 v1, s3445; CI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc446; CI-NEXT: s_waitcnt vmcnt(0)447; CI-NEXT: buffer_wbinvl1_vol448; CI-NEXT: v_mov_b32_e32 v0, s0449; CI-NEXT: v_mov_b32_e32 v1, s1450; CI-NEXT: flat_store_dword v[0:1], v2451; CI-NEXT: s_endpgm452;453; VI-LABEL: global_atomic_inc_ret_i32_offset:454; VI: ; %bb.0:455; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0456; VI-NEXT: s_add_i32 s12, s12, s17457; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8458; VI-NEXT: s_mov_b32 flat_scratch_lo, s13459; VI-NEXT: v_mov_b32_e32 v2, 42460; VI-NEXT: s_waitcnt lgkmcnt(0)461; VI-NEXT: s_add_u32 s2, s2, 16462; VI-NEXT: s_addc_u32 s3, s3, 0463; VI-NEXT: v_mov_b32_e32 v0, s2464; VI-NEXT: v_mov_b32_e32 v1, s3465; VI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc466; VI-NEXT: s_waitcnt vmcnt(0)467; VI-NEXT: buffer_wbinvl1_vol468; VI-NEXT: v_mov_b32_e32 v0, s0469; VI-NEXT: v_mov_b32_e32 v1, s1470; VI-NEXT: flat_store_dword v[0:1], v2471; VI-NEXT: s_endpgm472;473; GFX9-LABEL: global_atomic_inc_ret_i32_offset:474; GFX9: ; %bb.0:475; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0476; GFX9-NEXT: v_mov_b32_e32 v0, 42477; GFX9-NEXT: v_mov_b32_e32 v1, 0478; GFX9-NEXT: s_waitcnt lgkmcnt(0)479; GFX9-NEXT: global_atomic_inc v0, v1, v0, s[2:3] offset:16 glc480; GFX9-NEXT: s_waitcnt vmcnt(0)481; GFX9-NEXT: buffer_wbinvl1_vol482; GFX9-NEXT: global_store_dword v1, v0, s[0:1]483; GFX9-NEXT: s_endpgm484;485; GFX10-LABEL: global_atomic_inc_ret_i32_offset:486; GFX10: ; %bb.0:487; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0488; GFX10-NEXT: v_mov_b32_e32 v0, 42489; GFX10-NEXT: v_mov_b32_e32 v1, 0490; GFX10-NEXT: s_waitcnt lgkmcnt(0)491; GFX10-NEXT: global_atomic_inc v0, v1, v0, s[2:3] offset:16 glc492; GFX10-NEXT: s_waitcnt vmcnt(0)493; GFX10-NEXT: buffer_gl1_inv494; GFX10-NEXT: buffer_gl0_inv495; GFX10-NEXT: global_store_dword v1, v0, s[0:1]496; GFX10-NEXT: s_endpgm497;498; GFX11-LABEL: global_atomic_inc_ret_i32_offset:499; GFX11: ; %bb.0:500; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0501; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0502; GFX11-NEXT: s_waitcnt lgkmcnt(0)503; GFX11-NEXT: global_atomic_inc_u32 v0, v1, v0, s[2:3] offset:16 glc504; GFX11-NEXT: s_waitcnt vmcnt(0)505; GFX11-NEXT: buffer_gl1_inv506; GFX11-NEXT: buffer_gl0_inv507; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]508; GFX11-NEXT: s_endpgm509;510; GFX12-LABEL: global_atomic_inc_ret_i32_offset:511; GFX12: ; %bb.0:512; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x0513; GFX12-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0514; GFX12-NEXT: s_wait_kmcnt 0x0515; GFX12-NEXT: global_atomic_inc_u32 v0, v1, v0, s[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV516; GFX12-NEXT: s_wait_loadcnt 0x0517; GFX12-NEXT: global_inv scope:SCOPE_DEV518; GFX12-NEXT: global_store_b32 v1, v0, s[0:1]519; GFX12-NEXT: s_endpgm520 %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4521 %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1522 store i32 %result, ptr addrspace(1) %out, align 4523 ret void524}525 526define amdgpu_kernel void @global_atomic_inc_ret_i32_offset_sistem(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #1 {527; CI-LABEL: global_atomic_inc_ret_i32_offset_sistem:528; CI: ; %bb.0:529; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0530; CI-NEXT: s_add_i32 s12, s12, s17531; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8532; CI-NEXT: s_mov_b32 flat_scratch_lo, s13533; CI-NEXT: v_mov_b32_e32 v2, 42534; CI-NEXT: s_waitcnt lgkmcnt(0)535; CI-NEXT: s_add_u32 s2, s2, 16536; CI-NEXT: s_addc_u32 s3, s3, 0537; CI-NEXT: v_mov_b32_e32 v0, s2538; CI-NEXT: v_mov_b32_e32 v1, s3539; CI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc540; CI-NEXT: s_waitcnt vmcnt(0)541; CI-NEXT: buffer_wbinvl1_vol542; CI-NEXT: v_mov_b32_e32 v0, s0543; CI-NEXT: v_mov_b32_e32 v1, s1544; CI-NEXT: flat_store_dword v[0:1], v2545; CI-NEXT: s_endpgm546;547; VI-LABEL: global_atomic_inc_ret_i32_offset_sistem:548; VI: ; %bb.0:549; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0550; VI-NEXT: s_add_i32 s12, s12, s17551; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8552; VI-NEXT: s_mov_b32 flat_scratch_lo, s13553; VI-NEXT: v_mov_b32_e32 v2, 42554; VI-NEXT: s_waitcnt lgkmcnt(0)555; VI-NEXT: s_add_u32 s2, s2, 16556; VI-NEXT: s_addc_u32 s3, s3, 0557; VI-NEXT: v_mov_b32_e32 v0, s2558; VI-NEXT: v_mov_b32_e32 v1, s3559; VI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc560; VI-NEXT: s_waitcnt vmcnt(0)561; VI-NEXT: buffer_wbinvl1_vol562; VI-NEXT: v_mov_b32_e32 v0, s0563; VI-NEXT: v_mov_b32_e32 v1, s1564; VI-NEXT: flat_store_dword v[0:1], v2565; VI-NEXT: s_endpgm566;567; GFX9-LABEL: global_atomic_inc_ret_i32_offset_sistem:568; GFX9: ; %bb.0:569; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0570; GFX9-NEXT: v_mov_b32_e32 v0, 42571; GFX9-NEXT: v_mov_b32_e32 v1, 0572; GFX9-NEXT: s_waitcnt lgkmcnt(0)573; GFX9-NEXT: global_atomic_inc v0, v1, v0, s[2:3] offset:16 glc574; GFX9-NEXT: s_waitcnt vmcnt(0)575; GFX9-NEXT: buffer_wbinvl1_vol576; GFX9-NEXT: global_store_dword v1, v0, s[0:1]577; GFX9-NEXT: s_endpgm578;579; GFX10-LABEL: global_atomic_inc_ret_i32_offset_sistem:580; GFX10: ; %bb.0:581; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0582; GFX10-NEXT: v_mov_b32_e32 v0, 42583; GFX10-NEXT: v_mov_b32_e32 v1, 0584; GFX10-NEXT: s_waitcnt lgkmcnt(0)585; GFX10-NEXT: global_atomic_inc v0, v1, v0, s[2:3] offset:16 glc586; GFX10-NEXT: s_waitcnt vmcnt(0)587; GFX10-NEXT: buffer_gl1_inv588; GFX10-NEXT: buffer_gl0_inv589; GFX10-NEXT: global_store_dword v1, v0, s[0:1]590; GFX10-NEXT: s_endpgm591;592; GFX11-LABEL: global_atomic_inc_ret_i32_offset_sistem:593; GFX11: ; %bb.0:594; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0595; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0596; GFX11-NEXT: s_waitcnt lgkmcnt(0)597; GFX11-NEXT: global_atomic_inc_u32 v0, v1, v0, s[2:3] offset:16 glc598; GFX11-NEXT: s_waitcnt vmcnt(0)599; GFX11-NEXT: buffer_gl1_inv600; GFX11-NEXT: buffer_gl0_inv601; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]602; GFX11-NEXT: s_endpgm603;604; GFX12-LABEL: global_atomic_inc_ret_i32_offset_sistem:605; GFX12: ; %bb.0:606; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x0607; GFX12-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0608; GFX12-NEXT: global_wb scope:SCOPE_SYS609; GFX12-NEXT: s_wait_storecnt 0x0610; GFX12-NEXT: s_wait_kmcnt 0x0611; GFX12-NEXT: global_atomic_inc_u32 v0, v1, v0, s[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS612; GFX12-NEXT: s_wait_loadcnt 0x0613; GFX12-NEXT: global_inv scope:SCOPE_SYS614; GFX12-NEXT: global_store_b32 v1, v0, s[0:1]615; GFX12-NEXT: s_endpgm616 %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4617 %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 seq_cst, align 4, !amdgpu.no.remote.memory !1618 store i32 %result, ptr addrspace(1) %out, align 4619 ret void620}621 622define amdgpu_kernel void @global_atomic_inc_noret_i32(ptr addrspace(1) %ptr) #1 {623; CI-LABEL: global_atomic_inc_noret_i32:624; CI: ; %bb.0:625; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0626; CI-NEXT: s_add_i32 s12, s12, s17627; CI-NEXT: s_mov_b32 flat_scratch_lo, s13628; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8629; CI-NEXT: v_mov_b32_e32 v2, 42630; CI-NEXT: s_waitcnt lgkmcnt(0)631; CI-NEXT: v_mov_b32_e32 v0, s0632; CI-NEXT: v_mov_b32_e32 v1, s1633; CI-NEXT: flat_atomic_inc v[0:1], v2634; CI-NEXT: s_waitcnt vmcnt(0)635; CI-NEXT: buffer_wbinvl1_vol636; CI-NEXT: s_endpgm637;638; VI-LABEL: global_atomic_inc_noret_i32:639; VI: ; %bb.0:640; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0641; VI-NEXT: s_add_i32 s12, s12, s17642; VI-NEXT: s_mov_b32 flat_scratch_lo, s13643; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8644; VI-NEXT: v_mov_b32_e32 v2, 42645; VI-NEXT: s_waitcnt lgkmcnt(0)646; VI-NEXT: v_mov_b32_e32 v0, s0647; VI-NEXT: v_mov_b32_e32 v1, s1648; VI-NEXT: flat_atomic_inc v[0:1], v2649; VI-NEXT: s_waitcnt vmcnt(0)650; VI-NEXT: buffer_wbinvl1_vol651; VI-NEXT: s_endpgm652;653; GFX9-LABEL: global_atomic_inc_noret_i32:654; GFX9: ; %bb.0:655; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0656; GFX9-NEXT: v_mov_b32_e32 v0, 42657; GFX9-NEXT: v_mov_b32_e32 v1, 0658; GFX9-NEXT: s_waitcnt lgkmcnt(0)659; GFX9-NEXT: global_atomic_inc v1, v0, s[0:1]660; GFX9-NEXT: s_waitcnt vmcnt(0)661; GFX9-NEXT: buffer_wbinvl1_vol662; GFX9-NEXT: s_endpgm663;664; GFX10-LABEL: global_atomic_inc_noret_i32:665; GFX10: ; %bb.0:666; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0667; GFX10-NEXT: v_mov_b32_e32 v0, 42668; GFX10-NEXT: v_mov_b32_e32 v1, 0669; GFX10-NEXT: s_waitcnt lgkmcnt(0)670; GFX10-NEXT: global_atomic_inc v1, v0, s[0:1]671; GFX10-NEXT: s_waitcnt_vscnt null, 0x0672; GFX10-NEXT: buffer_gl1_inv673; GFX10-NEXT: buffer_gl0_inv674; GFX10-NEXT: s_endpgm675;676; GFX11-LABEL: global_atomic_inc_noret_i32:677; GFX11: ; %bb.0:678; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0679; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0680; GFX11-NEXT: s_waitcnt lgkmcnt(0)681; GFX11-NEXT: global_atomic_inc_u32 v1, v0, s[0:1]682; GFX11-NEXT: s_waitcnt_vscnt null, 0x0683; GFX11-NEXT: buffer_gl1_inv684; GFX11-NEXT: buffer_gl0_inv685; GFX11-NEXT: s_endpgm686;687; GFX12-LABEL: global_atomic_inc_noret_i32:688; GFX12: ; %bb.0:689; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x0690; GFX12-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0691; GFX12-NEXT: s_wait_kmcnt 0x0692; GFX12-NEXT: global_atomic_inc_u32 v1, v0, s[0:1] scope:SCOPE_DEV693; GFX12-NEXT: s_wait_storecnt 0x0694; GFX12-NEXT: global_inv scope:SCOPE_DEV695; GFX12-NEXT: s_endpgm696 %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1697 ret void698}699 700define amdgpu_kernel void @global_atomic_inc_noret_i32_offset(ptr addrspace(1) %ptr) #1 {701; CI-LABEL: global_atomic_inc_noret_i32_offset:702; CI: ; %bb.0:703; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0704; CI-NEXT: s_add_i32 s12, s12, s17705; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8706; CI-NEXT: s_mov_b32 flat_scratch_lo, s13707; CI-NEXT: v_mov_b32_e32 v2, 42708; CI-NEXT: s_waitcnt lgkmcnt(0)709; CI-NEXT: s_add_u32 s0, s0, 16710; CI-NEXT: s_addc_u32 s1, s1, 0711; CI-NEXT: v_mov_b32_e32 v0, s0712; CI-NEXT: v_mov_b32_e32 v1, s1713; CI-NEXT: flat_atomic_inc v[0:1], v2714; CI-NEXT: s_waitcnt vmcnt(0)715; CI-NEXT: buffer_wbinvl1_vol716; CI-NEXT: s_endpgm717;718; VI-LABEL: global_atomic_inc_noret_i32_offset:719; VI: ; %bb.0:720; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0721; VI-NEXT: s_add_i32 s12, s12, s17722; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8723; VI-NEXT: s_mov_b32 flat_scratch_lo, s13724; VI-NEXT: v_mov_b32_e32 v2, 42725; VI-NEXT: s_waitcnt lgkmcnt(0)726; VI-NEXT: s_add_u32 s0, s0, 16727; VI-NEXT: s_addc_u32 s1, s1, 0728; VI-NEXT: v_mov_b32_e32 v0, s0729; VI-NEXT: v_mov_b32_e32 v1, s1730; VI-NEXT: flat_atomic_inc v[0:1], v2731; VI-NEXT: s_waitcnt vmcnt(0)732; VI-NEXT: buffer_wbinvl1_vol733; VI-NEXT: s_endpgm734;735; GFX9-LABEL: global_atomic_inc_noret_i32_offset:736; GFX9: ; %bb.0:737; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0738; GFX9-NEXT: v_mov_b32_e32 v0, 42739; GFX9-NEXT: v_mov_b32_e32 v1, 0740; GFX9-NEXT: s_waitcnt lgkmcnt(0)741; GFX9-NEXT: global_atomic_inc v1, v0, s[0:1] offset:16742; GFX9-NEXT: s_waitcnt vmcnt(0)743; GFX9-NEXT: buffer_wbinvl1_vol744; GFX9-NEXT: s_endpgm745;746; GFX10-LABEL: global_atomic_inc_noret_i32_offset:747; GFX10: ; %bb.0:748; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0749; GFX10-NEXT: v_mov_b32_e32 v0, 42750; GFX10-NEXT: v_mov_b32_e32 v1, 0751; GFX10-NEXT: s_waitcnt lgkmcnt(0)752; GFX10-NEXT: global_atomic_inc v1, v0, s[0:1] offset:16753; GFX10-NEXT: s_waitcnt_vscnt null, 0x0754; GFX10-NEXT: buffer_gl1_inv755; GFX10-NEXT: buffer_gl0_inv756; GFX10-NEXT: s_endpgm757;758; GFX11-LABEL: global_atomic_inc_noret_i32_offset:759; GFX11: ; %bb.0:760; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0761; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0762; GFX11-NEXT: s_waitcnt lgkmcnt(0)763; GFX11-NEXT: global_atomic_inc_u32 v1, v0, s[0:1] offset:16764; GFX11-NEXT: s_waitcnt_vscnt null, 0x0765; GFX11-NEXT: buffer_gl1_inv766; GFX11-NEXT: buffer_gl0_inv767; GFX11-NEXT: s_endpgm768;769; GFX12-LABEL: global_atomic_inc_noret_i32_offset:770; GFX12: ; %bb.0:771; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x0772; GFX12-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0773; GFX12-NEXT: s_wait_kmcnt 0x0774; GFX12-NEXT: global_atomic_inc_u32 v1, v0, s[0:1] offset:16 scope:SCOPE_DEV775; GFX12-NEXT: s_wait_storecnt 0x0776; GFX12-NEXT: global_inv scope:SCOPE_DEV777; GFX12-NEXT: s_endpgm778 %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4779 %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1780 ret void781}782 783define amdgpu_kernel void @global_atomic_inc_noret_i32_offset_system(ptr addrspace(1) %ptr) #1 {784; CI-LABEL: global_atomic_inc_noret_i32_offset_system:785; CI: ; %bb.0:786; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0787; CI-NEXT: s_add_i32 s12, s12, s17788; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8789; CI-NEXT: s_mov_b32 flat_scratch_lo, s13790; CI-NEXT: v_mov_b32_e32 v2, 42791; CI-NEXT: s_waitcnt lgkmcnt(0)792; CI-NEXT: s_add_u32 s0, s0, 16793; CI-NEXT: s_addc_u32 s1, s1, 0794; CI-NEXT: v_mov_b32_e32 v0, s0795; CI-NEXT: v_mov_b32_e32 v1, s1796; CI-NEXT: flat_atomic_inc v[0:1], v2797; CI-NEXT: s_waitcnt vmcnt(0)798; CI-NEXT: buffer_wbinvl1_vol799; CI-NEXT: s_endpgm800;801; VI-LABEL: global_atomic_inc_noret_i32_offset_system:802; VI: ; %bb.0:803; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0804; VI-NEXT: s_add_i32 s12, s12, s17805; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8806; VI-NEXT: s_mov_b32 flat_scratch_lo, s13807; VI-NEXT: v_mov_b32_e32 v2, 42808; VI-NEXT: s_waitcnt lgkmcnt(0)809; VI-NEXT: s_add_u32 s0, s0, 16810; VI-NEXT: s_addc_u32 s1, s1, 0811; VI-NEXT: v_mov_b32_e32 v0, s0812; VI-NEXT: v_mov_b32_e32 v1, s1813; VI-NEXT: flat_atomic_inc v[0:1], v2814; VI-NEXT: s_waitcnt vmcnt(0)815; VI-NEXT: buffer_wbinvl1_vol816; VI-NEXT: s_endpgm817;818; GFX9-LABEL: global_atomic_inc_noret_i32_offset_system:819; GFX9: ; %bb.0:820; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0821; GFX9-NEXT: v_mov_b32_e32 v0, 42822; GFX9-NEXT: v_mov_b32_e32 v1, 0823; GFX9-NEXT: s_waitcnt lgkmcnt(0)824; GFX9-NEXT: global_atomic_inc v1, v0, s[0:1] offset:16825; GFX9-NEXT: s_waitcnt vmcnt(0)826; GFX9-NEXT: buffer_wbinvl1_vol827; GFX9-NEXT: s_endpgm828;829; GFX10-LABEL: global_atomic_inc_noret_i32_offset_system:830; GFX10: ; %bb.0:831; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0832; GFX10-NEXT: v_mov_b32_e32 v0, 42833; GFX10-NEXT: v_mov_b32_e32 v1, 0834; GFX10-NEXT: s_waitcnt lgkmcnt(0)835; GFX10-NEXT: global_atomic_inc v1, v0, s[0:1] offset:16836; GFX10-NEXT: s_waitcnt_vscnt null, 0x0837; GFX10-NEXT: buffer_gl1_inv838; GFX10-NEXT: buffer_gl0_inv839; GFX10-NEXT: s_endpgm840;841; GFX11-LABEL: global_atomic_inc_noret_i32_offset_system:842; GFX11: ; %bb.0:843; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0844; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0845; GFX11-NEXT: s_waitcnt lgkmcnt(0)846; GFX11-NEXT: global_atomic_inc_u32 v1, v0, s[0:1] offset:16847; GFX11-NEXT: s_waitcnt_vscnt null, 0x0848; GFX11-NEXT: buffer_gl1_inv849; GFX11-NEXT: buffer_gl0_inv850; GFX11-NEXT: s_endpgm851;852; GFX12-LABEL: global_atomic_inc_noret_i32_offset_system:853; GFX12: ; %bb.0:854; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x0855; GFX12-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0856; GFX12-NEXT: global_wb scope:SCOPE_SYS857; GFX12-NEXT: s_wait_storecnt 0x0858; GFX12-NEXT: s_wait_kmcnt 0x0859; GFX12-NEXT: global_atomic_inc_u32 v1, v0, s[0:1] offset:16 scope:SCOPE_SYS860; GFX12-NEXT: s_wait_storecnt 0x0861; GFX12-NEXT: global_inv scope:SCOPE_SYS862; GFX12-NEXT: s_endpgm863 %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4864 %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 seq_cst, align 4, !amdgpu.no.remote.memory !1865 ret void866}867 868define amdgpu_kernel void @global_atomic_inc_ret_i32_offset_addr64(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #1 {869; CI-LABEL: global_atomic_inc_ret_i32_offset_addr64:870; CI: ; %bb.0:871; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0872; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v0873; CI-NEXT: s_add_i32 s12, s12, s17874; CI-NEXT: s_mov_b32 flat_scratch_lo, s13875; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8876; CI-NEXT: s_waitcnt lgkmcnt(0)877; CI-NEXT: v_mov_b32_e32 v0, s2878; CI-NEXT: v_mov_b32_e32 v1, s3879; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2880; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc881; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v0882; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc883; CI-NEXT: v_mov_b32_e32 v3, 42884; CI-NEXT: flat_atomic_inc v3, v[0:1], v3 glc885; CI-NEXT: s_waitcnt vmcnt(0)886; CI-NEXT: buffer_wbinvl1_vol887; CI-NEXT: v_mov_b32_e32 v0, s0888; CI-NEXT: v_mov_b32_e32 v1, s1889; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2890; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc891; CI-NEXT: flat_store_dword v[0:1], v3892; CI-NEXT: s_endpgm893;894; VI-LABEL: global_atomic_inc_ret_i32_offset_addr64:895; VI: ; %bb.0:896; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0897; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0898; VI-NEXT: s_add_i32 s12, s12, s17899; VI-NEXT: s_mov_b32 flat_scratch_lo, s13900; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8901; VI-NEXT: s_waitcnt lgkmcnt(0)902; VI-NEXT: v_mov_b32_e32 v0, s2903; VI-NEXT: v_mov_b32_e32 v1, s3904; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2905; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc906; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v0907; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc908; VI-NEXT: v_mov_b32_e32 v3, 42909; VI-NEXT: flat_atomic_inc v3, v[0:1], v3 glc910; VI-NEXT: s_waitcnt vmcnt(0)911; VI-NEXT: buffer_wbinvl1_vol912; VI-NEXT: v_mov_b32_e32 v0, s0913; VI-NEXT: v_mov_b32_e32 v1, s1914; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2915; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc916; VI-NEXT: flat_store_dword v[0:1], v3917; VI-NEXT: s_endpgm918;919; GFX9-LABEL: global_atomic_inc_ret_i32_offset_addr64:920; GFX9: ; %bb.0:921; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0922; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0923; GFX9-NEXT: v_mov_b32_e32 v1, 42924; GFX9-NEXT: s_waitcnt lgkmcnt(0)925; GFX9-NEXT: global_atomic_inc v1, v0, v1, s[2:3] offset:20 glc926; GFX9-NEXT: s_waitcnt vmcnt(0)927; GFX9-NEXT: buffer_wbinvl1_vol928; GFX9-NEXT: global_store_dword v0, v1, s[0:1]929; GFX9-NEXT: s_endpgm930;931; GFX10-LABEL: global_atomic_inc_ret_i32_offset_addr64:932; GFX10: ; %bb.0:933; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0934; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0935; GFX10-NEXT: v_mov_b32_e32 v1, 42936; GFX10-NEXT: s_waitcnt lgkmcnt(0)937; GFX10-NEXT: global_atomic_inc v1, v0, v1, s[2:3] offset:20 glc938; GFX10-NEXT: s_waitcnt vmcnt(0)939; GFX10-NEXT: buffer_gl1_inv940; GFX10-NEXT: buffer_gl0_inv941; GFX10-NEXT: global_store_dword v0, v1, s[0:1]942; GFX10-NEXT: s_endpgm943;944; GFX11-LABEL: global_atomic_inc_ret_i32_offset_addr64:945; GFX11: ; %bb.0:946; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0947; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_and_b32 v0, 0x3ff, v0948; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)949; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0950; GFX11-NEXT: s_waitcnt lgkmcnt(0)951; GFX11-NEXT: global_atomic_inc_u32 v1, v0, v1, s[2:3] offset:20 glc952; GFX11-NEXT: s_waitcnt vmcnt(0)953; GFX11-NEXT: buffer_gl1_inv954; GFX11-NEXT: buffer_gl0_inv955; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]956; GFX11-NEXT: s_endpgm957;958; GFX12-LABEL: global_atomic_inc_ret_i32_offset_addr64:959; GFX12: ; %bb.0:960; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x0961; GFX12-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_and_b32 v0, 0x3ff, v0962; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)963; GFX12-NEXT: v_lshlrev_b32_e32 v0, 2, v0964; GFX12-NEXT: s_wait_kmcnt 0x0965; GFX12-NEXT: global_atomic_inc_u32 v1, v0, v1, s[2:3] offset:20 th:TH_ATOMIC_RETURN scope:SCOPE_DEV966; GFX12-NEXT: s_wait_loadcnt 0x0967; GFX12-NEXT: global_inv scope:SCOPE_DEV968; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]969; GFX12-NEXT: s_endpgm970 %id = call i32 @llvm.amdgcn.workitem.id.x()971 %gep.tid = getelementptr i32, ptr addrspace(1) %ptr, i32 %id972 %out.gep = getelementptr i32, ptr addrspace(1) %out, i32 %id973 %gep = getelementptr i32, ptr addrspace(1) %gep.tid, i32 5974 %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1975 store i32 %result, ptr addrspace(1) %out.gep, align 4976 ret void977}978 979define amdgpu_kernel void @global_atomic_inc_noret_i32_offset_addr64(ptr addrspace(1) %ptr) #1 {980; CI-LABEL: global_atomic_inc_noret_i32_offset_addr64:981; CI: ; %bb.0:982; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0983; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v0984; CI-NEXT: s_add_i32 s12, s12, s17985; CI-NEXT: s_mov_b32 flat_scratch_lo, s13986; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8987; CI-NEXT: s_waitcnt lgkmcnt(0)988; CI-NEXT: v_mov_b32_e32 v0, s0989; CI-NEXT: v_mov_b32_e32 v1, s1990; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v2991; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc992; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v0993; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc994; CI-NEXT: v_mov_b32_e32 v2, 42995; CI-NEXT: flat_atomic_inc v[0:1], v2996; CI-NEXT: s_waitcnt vmcnt(0)997; CI-NEXT: buffer_wbinvl1_vol998; CI-NEXT: s_endpgm999;1000; VI-LABEL: global_atomic_inc_noret_i32_offset_addr64:1001; VI: ; %bb.0:1002; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01003; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v01004; VI-NEXT: s_add_i32 s12, s12, s171005; VI-NEXT: s_mov_b32 flat_scratch_lo, s131006; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81007; VI-NEXT: s_waitcnt lgkmcnt(0)1008; VI-NEXT: v_mov_b32_e32 v0, s01009; VI-NEXT: v_mov_b32_e32 v1, s11010; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v21011; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1012; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v01013; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1014; VI-NEXT: v_mov_b32_e32 v2, 421015; VI-NEXT: flat_atomic_inc v[0:1], v21016; VI-NEXT: s_waitcnt vmcnt(0)1017; VI-NEXT: buffer_wbinvl1_vol1018; VI-NEXT: s_endpgm1019;1020; GFX9-LABEL: global_atomic_inc_noret_i32_offset_addr64:1021; GFX9: ; %bb.0:1022; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01023; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v01024; GFX9-NEXT: v_mov_b32_e32 v1, 421025; GFX9-NEXT: s_waitcnt lgkmcnt(0)1026; GFX9-NEXT: global_atomic_inc v0, v1, s[0:1] offset:201027; GFX9-NEXT: s_waitcnt vmcnt(0)1028; GFX9-NEXT: buffer_wbinvl1_vol1029; GFX9-NEXT: s_endpgm1030;1031; GFX10-LABEL: global_atomic_inc_noret_i32_offset_addr64:1032; GFX10: ; %bb.0:1033; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01034; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v01035; GFX10-NEXT: v_mov_b32_e32 v1, 421036; GFX10-NEXT: s_waitcnt lgkmcnt(0)1037; GFX10-NEXT: global_atomic_inc v0, v1, s[0:1] offset:201038; GFX10-NEXT: s_waitcnt_vscnt null, 0x01039; GFX10-NEXT: buffer_gl1_inv1040; GFX10-NEXT: buffer_gl0_inv1041; GFX10-NEXT: s_endpgm1042;1043; GFX11-LABEL: global_atomic_inc_noret_i32_offset_addr64:1044; GFX11: ; %bb.0:1045; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x01046; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_and_b32 v0, 0x3ff, v01047; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)1048; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v01049; GFX11-NEXT: s_waitcnt lgkmcnt(0)1050; GFX11-NEXT: global_atomic_inc_u32 v0, v1, s[0:1] offset:201051; GFX11-NEXT: s_waitcnt_vscnt null, 0x01052; GFX11-NEXT: buffer_gl1_inv1053; GFX11-NEXT: buffer_gl0_inv1054; GFX11-NEXT: s_endpgm1055;1056; GFX12-LABEL: global_atomic_inc_noret_i32_offset_addr64:1057; GFX12: ; %bb.0:1058; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x01059; GFX12-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_and_b32 v0, 0x3ff, v01060; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)1061; GFX12-NEXT: v_lshlrev_b32_e32 v0, 2, v01062; GFX12-NEXT: s_wait_kmcnt 0x01063; GFX12-NEXT: global_atomic_inc_u32 v0, v1, s[0:1] offset:20 scope:SCOPE_DEV1064; GFX12-NEXT: s_wait_storecnt 0x01065; GFX12-NEXT: global_inv scope:SCOPE_DEV1066; GFX12-NEXT: s_endpgm1067 %id = call i32 @llvm.amdgcn.workitem.id.x()1068 %gep.tid = getelementptr i32, ptr addrspace(1) %ptr, i32 %id1069 %gep = getelementptr i32, ptr addrspace(1) %gep.tid, i32 51070 %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !11071 ret void1072}1073 1074define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i32(ptr addrspace(1) %out, ptr addrspace(1) %add_use) #1 {1075; CI-LABEL: atomic_inc_shl_base_lds_0_i32:1076; CI: ; %bb.0:1077; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01078; CI-NEXT: v_lshlrev_b32_e32 v1, 2, v01079; CI-NEXT: v_mov_b32_e32 v2, 91080; CI-NEXT: s_mov_b32 m0, -11081; CI-NEXT: s_waitcnt lgkmcnt(0)1082; CI-NEXT: ds_inc_rtn_u32 v2, v1, v2 offset:81083; CI-NEXT: s_waitcnt lgkmcnt(0)1084; CI-NEXT: s_add_i32 s12, s12, s171085; CI-NEXT: v_add_i32_e32 v3, vcc, 2, v01086; CI-NEXT: v_mov_b32_e32 v0, s21087; CI-NEXT: s_mov_b32 flat_scratch_lo, s131088; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81089; CI-NEXT: v_mov_b32_e32 v1, s31090; CI-NEXT: flat_store_dword v[0:1], v31091; CI-NEXT: v_mov_b32_e32 v0, s01092; CI-NEXT: v_mov_b32_e32 v1, s11093; CI-NEXT: flat_store_dword v[0:1], v21094; CI-NEXT: s_endpgm1095;1096; VI-LABEL: atomic_inc_shl_base_lds_0_i32:1097; VI: ; %bb.0:1098; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01099; VI-NEXT: v_lshlrev_b32_e32 v1, 2, v01100; VI-NEXT: v_mov_b32_e32 v2, 91101; VI-NEXT: s_mov_b32 m0, -11102; VI-NEXT: s_waitcnt lgkmcnt(0)1103; VI-NEXT: ds_inc_rtn_u32 v2, v1, v2 offset:81104; VI-NEXT: s_waitcnt lgkmcnt(0)1105; VI-NEXT: s_add_i32 s12, s12, s171106; VI-NEXT: v_add_u32_e32 v3, vcc, 2, v01107; VI-NEXT: v_mov_b32_e32 v0, s21108; VI-NEXT: s_mov_b32 flat_scratch_lo, s131109; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81110; VI-NEXT: v_mov_b32_e32 v1, s31111; VI-NEXT: flat_store_dword v[0:1], v31112; VI-NEXT: v_mov_b32_e32 v0, s01113; VI-NEXT: v_mov_b32_e32 v1, s11114; VI-NEXT: flat_store_dword v[0:1], v21115; VI-NEXT: s_endpgm1116;1117; GFX9-LABEL: atomic_inc_shl_base_lds_0_i32:1118; GFX9: ; %bb.0:1119; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01120; GFX9-NEXT: v_lshlrev_b32_e32 v1, 2, v01121; GFX9-NEXT: v_mov_b32_e32 v2, 91122; GFX9-NEXT: s_waitcnt lgkmcnt(0)1123; GFX9-NEXT: ds_inc_rtn_u32 v1, v1, v2 offset:81124; GFX9-NEXT: s_waitcnt lgkmcnt(0)1125; GFX9-NEXT: v_add_u32_e32 v0, 2, v01126; GFX9-NEXT: v_mov_b32_e32 v2, 01127; GFX9-NEXT: global_store_dword v2, v0, s[2:3]1128; GFX9-NEXT: global_store_dword v2, v1, s[0:1]1129; GFX9-NEXT: s_endpgm1130;1131; GFX10-LABEL: atomic_inc_shl_base_lds_0_i32:1132; GFX10: ; %bb.0:1133; GFX10-NEXT: v_lshlrev_b32_e32 v1, 2, v01134; GFX10-NEXT: v_mov_b32_e32 v2, 91135; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01136; GFX10-NEXT: v_add_nc_u32_e32 v0, 2, v01137; GFX10-NEXT: s_waitcnt lgkmcnt(0)1138; GFX10-NEXT: ds_inc_rtn_u32 v1, v1, v2 offset:81139; GFX10-NEXT: s_waitcnt lgkmcnt(0)1140; GFX10-NEXT: buffer_gl0_inv1141; GFX10-NEXT: v_mov_b32_e32 v2, 01142; GFX10-NEXT: global_store_dword v2, v0, s[2:3]1143; GFX10-NEXT: global_store_dword v2, v1, s[0:1]1144; GFX10-NEXT: s_endpgm1145;1146; GFX11-LABEL: atomic_inc_shl_base_lds_0_i32:1147; GFX11: ; %bb.0:1148; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v01149; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x01150; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)1151; GFX11-NEXT: v_dual_mov_b32 v2, 9 :: v_dual_lshlrev_b32 v1, 2, v01152; GFX11-NEXT: v_add_nc_u32_e32 v0, 2, v01153; GFX11-NEXT: s_waitcnt lgkmcnt(0)1154; GFX11-NEXT: ds_inc_rtn_u32 v1, v1, v2 offset:81155; GFX11-NEXT: s_waitcnt lgkmcnt(0)1156; GFX11-NEXT: buffer_gl0_inv1157; GFX11-NEXT: v_mov_b32_e32 v2, 01158; GFX11-NEXT: s_clause 0x11159; GFX11-NEXT: global_store_b32 v2, v0, s[2:3]1160; GFX11-NEXT: global_store_b32 v2, v1, s[0:1]1161; GFX11-NEXT: s_endpgm1162;1163; GFX12-LABEL: atomic_inc_shl_base_lds_0_i32:1164; GFX12: ; %bb.0:1165; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v01166; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x01167; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)1168; GFX12-NEXT: v_dual_mov_b32 v2, 9 :: v_dual_lshlrev_b32 v1, 2, v01169; GFX12-NEXT: v_add_nc_u32_e32 v0, 2, v01170; GFX12-NEXT: ds_inc_rtn_u32 v1, v1, v2 offset:81171; GFX12-NEXT: s_wait_dscnt 0x01172; GFX12-NEXT: global_inv scope:SCOPE_SE1173; GFX12-NEXT: v_mov_b32_e32 v2, 01174; GFX12-NEXT: s_wait_kmcnt 0x01175; GFX12-NEXT: s_clause 0x11176; GFX12-NEXT: global_store_b32 v2, v0, s[2:3]1177; GFX12-NEXT: global_store_b32 v2, v1, s[0:1]1178; GFX12-NEXT: s_endpgm1179 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #21180 %idx.0 = add nsw i32 %tid.x, 21181 %arrayidx0 = getelementptr inbounds [512 x i32], ptr addrspace(3) @lds0, i32 0, i32 %idx.01182 %result = atomicrmw uinc_wrap ptr addrspace(3) %arrayidx0, i32 9 syncscope("agent") seq_cst, align 41183 store i32 %idx.0, ptr addrspace(1) %add_use, align 41184 store i32 %result, ptr addrspace(1) %out, align 41185 ret void1186}1187 1188define amdgpu_kernel void @lds_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {1189; CI-LABEL: lds_atomic_inc_ret_i64:1190; CI: ; %bb.0:1191; CI-NEXT: s_load_dword s2, s[8:9], 0x21192; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01193; CI-NEXT: v_mov_b32_e32 v0, 421194; CI-NEXT: v_mov_b32_e32 v1, 01195; CI-NEXT: s_mov_b32 m0, -11196; CI-NEXT: s_waitcnt lgkmcnt(0)1197; CI-NEXT: v_mov_b32_e32 v2, s21198; CI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1]1199; CI-NEXT: s_waitcnt lgkmcnt(0)1200; CI-NEXT: s_add_i32 s12, s12, s171201; CI-NEXT: v_mov_b32_e32 v3, s11202; CI-NEXT: s_mov_b32 flat_scratch_lo, s131203; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81204; CI-NEXT: v_mov_b32_e32 v2, s01205; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1206; CI-NEXT: s_endpgm1207;1208; VI-LABEL: lds_atomic_inc_ret_i64:1209; VI: ; %bb.0:1210; VI-NEXT: s_load_dword s2, s[8:9], 0x81211; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01212; VI-NEXT: v_mov_b32_e32 v0, 421213; VI-NEXT: v_mov_b32_e32 v1, 01214; VI-NEXT: s_mov_b32 m0, -11215; VI-NEXT: s_waitcnt lgkmcnt(0)1216; VI-NEXT: v_mov_b32_e32 v2, s21217; VI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1]1218; VI-NEXT: s_waitcnt lgkmcnt(0)1219; VI-NEXT: s_add_i32 s12, s12, s171220; VI-NEXT: v_mov_b32_e32 v3, s11221; VI-NEXT: s_mov_b32 flat_scratch_lo, s131222; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81223; VI-NEXT: v_mov_b32_e32 v2, s01224; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1225; VI-NEXT: s_endpgm1226;1227; GFX9-LABEL: lds_atomic_inc_ret_i64:1228; GFX9: ; %bb.0:1229; GFX9-NEXT: s_load_dword s2, s[8:9], 0x81230; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01231; GFX9-NEXT: v_mov_b32_e32 v0, 421232; GFX9-NEXT: v_mov_b32_e32 v1, 01233; GFX9-NEXT: s_waitcnt lgkmcnt(0)1234; GFX9-NEXT: v_mov_b32_e32 v2, s21235; GFX9-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1]1236; GFX9-NEXT: s_waitcnt lgkmcnt(0)1237; GFX9-NEXT: v_mov_b32_e32 v2, 01238; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]1239; GFX9-NEXT: s_endpgm1240;1241; GFX10-LABEL: lds_atomic_inc_ret_i64:1242; GFX10: ; %bb.0:1243; GFX10-NEXT: s_load_dword s0, s[8:9], 0x81244; GFX10-NEXT: v_mov_b32_e32 v0, 421245; GFX10-NEXT: v_mov_b32_e32 v1, 01246; GFX10-NEXT: s_waitcnt lgkmcnt(0)1247; GFX10-NEXT: v_mov_b32_e32 v2, s01248; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01249; GFX10-NEXT: s_waitcnt lgkmcnt(0)1250; GFX10-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1]1251; GFX10-NEXT: s_waitcnt lgkmcnt(0)1252; GFX10-NEXT: buffer_gl0_inv1253; GFX10-NEXT: v_mov_b32_e32 v2, 01254; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]1255; GFX10-NEXT: s_endpgm1256;1257; GFX11-LABEL: lds_atomic_inc_ret_i64:1258; GFX11: ; %bb.0:1259; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x81260; GFX11-NEXT: v_mov_b32_e32 v0, 421261; GFX11-NEXT: s_waitcnt lgkmcnt(0)1262; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s01263; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x01264; GFX11-NEXT: s_waitcnt lgkmcnt(0)1265; GFX11-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1]1266; GFX11-NEXT: s_waitcnt lgkmcnt(0)1267; GFX11-NEXT: buffer_gl0_inv1268; GFX11-NEXT: v_mov_b32_e32 v2, 01269; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]1270; GFX11-NEXT: s_endpgm1271;1272; GFX12-LABEL: lds_atomic_inc_ret_i64:1273; GFX12: ; %bb.0:1274; GFX12-NEXT: s_load_b96 s[0:2], s[4:5], 0x01275; GFX12-NEXT: v_mov_b32_e32 v0, 421276; GFX12-NEXT: s_wait_kmcnt 0x01277; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s21278; GFX12-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1]1279; GFX12-NEXT: s_wait_dscnt 0x01280; GFX12-NEXT: global_inv scope:SCOPE_SE1281; GFX12-NEXT: v_mov_b32_e32 v2, 01282; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]1283; GFX12-NEXT: s_endpgm1284 %result = atomicrmw uinc_wrap ptr addrspace(3) %ptr, i64 42 syncscope("agent") seq_cst, align 81285 store i64 %result, ptr addrspace(1) %out, align 41286 ret void1287}1288 1289define amdgpu_kernel void @lds_atomic_inc_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {1290; CI-LABEL: lds_atomic_inc_ret_i64_offset:1291; CI: ; %bb.0:1292; CI-NEXT: s_load_dword s2, s[8:9], 0x21293; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01294; CI-NEXT: v_mov_b32_e32 v0, 421295; CI-NEXT: v_mov_b32_e32 v1, 01296; CI-NEXT: s_mov_b32 m0, -11297; CI-NEXT: s_waitcnt lgkmcnt(0)1298; CI-NEXT: v_mov_b32_e32 v2, s21299; CI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:321300; CI-NEXT: s_waitcnt lgkmcnt(0)1301; CI-NEXT: s_add_i32 s12, s12, s171302; CI-NEXT: v_mov_b32_e32 v3, s11303; CI-NEXT: s_mov_b32 flat_scratch_lo, s131304; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81305; CI-NEXT: v_mov_b32_e32 v2, s01306; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1307; CI-NEXT: s_endpgm1308;1309; VI-LABEL: lds_atomic_inc_ret_i64_offset:1310; VI: ; %bb.0:1311; VI-NEXT: s_load_dword s2, s[8:9], 0x81312; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01313; VI-NEXT: v_mov_b32_e32 v0, 421314; VI-NEXT: v_mov_b32_e32 v1, 01315; VI-NEXT: s_mov_b32 m0, -11316; VI-NEXT: s_waitcnt lgkmcnt(0)1317; VI-NEXT: v_mov_b32_e32 v2, s21318; VI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:321319; VI-NEXT: s_waitcnt lgkmcnt(0)1320; VI-NEXT: s_add_i32 s12, s12, s171321; VI-NEXT: v_mov_b32_e32 v3, s11322; VI-NEXT: s_mov_b32 flat_scratch_lo, s131323; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81324; VI-NEXT: v_mov_b32_e32 v2, s01325; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1326; VI-NEXT: s_endpgm1327;1328; GFX9-LABEL: lds_atomic_inc_ret_i64_offset:1329; GFX9: ; %bb.0:1330; GFX9-NEXT: s_load_dword s2, s[8:9], 0x81331; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01332; GFX9-NEXT: v_mov_b32_e32 v0, 421333; GFX9-NEXT: v_mov_b32_e32 v1, 01334; GFX9-NEXT: s_waitcnt lgkmcnt(0)1335; GFX9-NEXT: v_mov_b32_e32 v2, s21336; GFX9-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:321337; GFX9-NEXT: s_waitcnt lgkmcnt(0)1338; GFX9-NEXT: v_mov_b32_e32 v2, 01339; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]1340; GFX9-NEXT: s_endpgm1341;1342; GFX10-LABEL: lds_atomic_inc_ret_i64_offset:1343; GFX10: ; %bb.0:1344; GFX10-NEXT: s_load_dword s0, s[8:9], 0x81345; GFX10-NEXT: v_mov_b32_e32 v0, 421346; GFX10-NEXT: v_mov_b32_e32 v1, 01347; GFX10-NEXT: s_waitcnt lgkmcnt(0)1348; GFX10-NEXT: v_mov_b32_e32 v2, s01349; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01350; GFX10-NEXT: s_waitcnt lgkmcnt(0)1351; GFX10-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:321352; GFX10-NEXT: s_waitcnt lgkmcnt(0)1353; GFX10-NEXT: buffer_gl0_inv1354; GFX10-NEXT: v_mov_b32_e32 v2, 01355; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]1356; GFX10-NEXT: s_endpgm1357;1358; GFX11-LABEL: lds_atomic_inc_ret_i64_offset:1359; GFX11: ; %bb.0:1360; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x81361; GFX11-NEXT: v_mov_b32_e32 v0, 421362; GFX11-NEXT: s_waitcnt lgkmcnt(0)1363; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s01364; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x01365; GFX11-NEXT: s_waitcnt lgkmcnt(0)1366; GFX11-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:321367; GFX11-NEXT: s_waitcnt lgkmcnt(0)1368; GFX11-NEXT: buffer_gl0_inv1369; GFX11-NEXT: v_mov_b32_e32 v2, 01370; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]1371; GFX11-NEXT: s_endpgm1372;1373; GFX12-LABEL: lds_atomic_inc_ret_i64_offset:1374; GFX12: ; %bb.0:1375; GFX12-NEXT: s_load_b96 s[0:2], s[4:5], 0x01376; GFX12-NEXT: v_mov_b32_e32 v0, 421377; GFX12-NEXT: s_wait_kmcnt 0x01378; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s21379; GFX12-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:321380; GFX12-NEXT: s_wait_dscnt 0x01381; GFX12-NEXT: global_inv scope:SCOPE_SE1382; GFX12-NEXT: v_mov_b32_e32 v2, 01383; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]1384; GFX12-NEXT: s_endpgm1385 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 41386 %result = atomicrmw uinc_wrap ptr addrspace(3) %gep, i64 42 syncscope("agent") seq_cst, align 81387 store i64 %result, ptr addrspace(1) %out, align 41388 ret void1389}1390 1391define amdgpu_kernel void @lds_atomic_inc_noret_i64(ptr addrspace(3) %ptr) #1 {1392; CI-LABEL: lds_atomic_inc_noret_i64:1393; CI: ; %bb.0:1394; CI-NEXT: s_load_dword s0, s[8:9], 0x01395; CI-NEXT: v_mov_b32_e32 v0, 421396; CI-NEXT: v_mov_b32_e32 v1, 01397; CI-NEXT: s_mov_b32 m0, -11398; CI-NEXT: s_waitcnt lgkmcnt(0)1399; CI-NEXT: v_mov_b32_e32 v2, s01400; CI-NEXT: ds_inc_u64 v2, v[0:1]1401; CI-NEXT: s_waitcnt lgkmcnt(0)1402; CI-NEXT: s_endpgm1403;1404; VI-LABEL: lds_atomic_inc_noret_i64:1405; VI: ; %bb.0:1406; VI-NEXT: s_load_dword s0, s[8:9], 0x01407; VI-NEXT: v_mov_b32_e32 v0, 421408; VI-NEXT: v_mov_b32_e32 v1, 01409; VI-NEXT: s_mov_b32 m0, -11410; VI-NEXT: s_waitcnt lgkmcnt(0)1411; VI-NEXT: v_mov_b32_e32 v2, s01412; VI-NEXT: ds_inc_u64 v2, v[0:1]1413; VI-NEXT: s_waitcnt lgkmcnt(0)1414; VI-NEXT: s_endpgm1415;1416; GFX9-LABEL: lds_atomic_inc_noret_i64:1417; GFX9: ; %bb.0:1418; GFX9-NEXT: s_load_dword s0, s[8:9], 0x01419; GFX9-NEXT: v_mov_b32_e32 v0, 421420; GFX9-NEXT: v_mov_b32_e32 v1, 01421; GFX9-NEXT: s_waitcnt lgkmcnt(0)1422; GFX9-NEXT: v_mov_b32_e32 v2, s01423; GFX9-NEXT: ds_inc_u64 v2, v[0:1]1424; GFX9-NEXT: s_waitcnt lgkmcnt(0)1425; GFX9-NEXT: s_endpgm1426;1427; GFX10-LABEL: lds_atomic_inc_noret_i64:1428; GFX10: ; %bb.0:1429; GFX10-NEXT: s_load_dword s0, s[8:9], 0x01430; GFX10-NEXT: v_mov_b32_e32 v0, 421431; GFX10-NEXT: v_mov_b32_e32 v1, 01432; GFX10-NEXT: s_waitcnt lgkmcnt(0)1433; GFX10-NEXT: v_mov_b32_e32 v2, s01434; GFX10-NEXT: ds_inc_u64 v2, v[0:1]1435; GFX10-NEXT: s_waitcnt lgkmcnt(0)1436; GFX10-NEXT: buffer_gl0_inv1437; GFX10-NEXT: s_endpgm1438;1439; GFX11-LABEL: lds_atomic_inc_noret_i64:1440; GFX11: ; %bb.0:1441; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x01442; GFX11-NEXT: v_mov_b32_e32 v0, 421443; GFX11-NEXT: s_waitcnt lgkmcnt(0)1444; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s01445; GFX11-NEXT: ds_inc_u64 v2, v[0:1]1446; GFX11-NEXT: s_waitcnt lgkmcnt(0)1447; GFX11-NEXT: buffer_gl0_inv1448; GFX11-NEXT: s_endpgm1449;1450; GFX12-LABEL: lds_atomic_inc_noret_i64:1451; GFX12: ; %bb.0:1452; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x01453; GFX12-NEXT: v_mov_b32_e32 v0, 421454; GFX12-NEXT: s_wait_kmcnt 0x01455; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s01456; GFX12-NEXT: ds_inc_u64 v2, v[0:1]1457; GFX12-NEXT: s_wait_dscnt 0x01458; GFX12-NEXT: global_inv scope:SCOPE_SE1459; GFX12-NEXT: s_endpgm1460 %result = atomicrmw uinc_wrap ptr addrspace(3) %ptr, i64 42 syncscope("agent") seq_cst, align 81461 ret void1462}1463 1464define amdgpu_kernel void @lds_atomic_inc_noret_i64_offset(ptr addrspace(3) %ptr) #1 {1465; CI-LABEL: lds_atomic_inc_noret_i64_offset:1466; CI: ; %bb.0:1467; CI-NEXT: s_load_dword s0, s[8:9], 0x01468; CI-NEXT: v_mov_b32_e32 v0, 421469; CI-NEXT: v_mov_b32_e32 v1, 01470; CI-NEXT: s_mov_b32 m0, -11471; CI-NEXT: s_waitcnt lgkmcnt(0)1472; CI-NEXT: v_mov_b32_e32 v2, s01473; CI-NEXT: ds_inc_u64 v2, v[0:1] offset:321474; CI-NEXT: s_waitcnt lgkmcnt(0)1475; CI-NEXT: s_endpgm1476;1477; VI-LABEL: lds_atomic_inc_noret_i64_offset:1478; VI: ; %bb.0:1479; VI-NEXT: s_load_dword s0, s[8:9], 0x01480; VI-NEXT: v_mov_b32_e32 v0, 421481; VI-NEXT: v_mov_b32_e32 v1, 01482; VI-NEXT: s_mov_b32 m0, -11483; VI-NEXT: s_waitcnt lgkmcnt(0)1484; VI-NEXT: v_mov_b32_e32 v2, s01485; VI-NEXT: ds_inc_u64 v2, v[0:1] offset:321486; VI-NEXT: s_waitcnt lgkmcnt(0)1487; VI-NEXT: s_endpgm1488;1489; GFX9-LABEL: lds_atomic_inc_noret_i64_offset:1490; GFX9: ; %bb.0:1491; GFX9-NEXT: s_load_dword s0, s[8:9], 0x01492; GFX9-NEXT: v_mov_b32_e32 v0, 421493; GFX9-NEXT: v_mov_b32_e32 v1, 01494; GFX9-NEXT: s_waitcnt lgkmcnt(0)1495; GFX9-NEXT: v_mov_b32_e32 v2, s01496; GFX9-NEXT: ds_inc_u64 v2, v[0:1] offset:321497; GFX9-NEXT: s_waitcnt lgkmcnt(0)1498; GFX9-NEXT: s_endpgm1499;1500; GFX10-LABEL: lds_atomic_inc_noret_i64_offset:1501; GFX10: ; %bb.0:1502; GFX10-NEXT: s_load_dword s0, s[8:9], 0x01503; GFX10-NEXT: v_mov_b32_e32 v0, 421504; GFX10-NEXT: v_mov_b32_e32 v1, 01505; GFX10-NEXT: s_waitcnt lgkmcnt(0)1506; GFX10-NEXT: v_mov_b32_e32 v2, s01507; GFX10-NEXT: ds_inc_u64 v2, v[0:1] offset:321508; GFX10-NEXT: s_waitcnt lgkmcnt(0)1509; GFX10-NEXT: buffer_gl0_inv1510; GFX10-NEXT: s_endpgm1511;1512; GFX11-LABEL: lds_atomic_inc_noret_i64_offset:1513; GFX11: ; %bb.0:1514; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x01515; GFX11-NEXT: v_mov_b32_e32 v0, 421516; GFX11-NEXT: s_waitcnt lgkmcnt(0)1517; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s01518; GFX11-NEXT: ds_inc_u64 v2, v[0:1] offset:321519; GFX11-NEXT: s_waitcnt lgkmcnt(0)1520; GFX11-NEXT: buffer_gl0_inv1521; GFX11-NEXT: s_endpgm1522;1523; GFX12-LABEL: lds_atomic_inc_noret_i64_offset:1524; GFX12: ; %bb.0:1525; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x01526; GFX12-NEXT: v_mov_b32_e32 v0, 421527; GFX12-NEXT: s_wait_kmcnt 0x01528; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s01529; GFX12-NEXT: ds_inc_u64 v2, v[0:1] offset:321530; GFX12-NEXT: s_wait_dscnt 0x01531; GFX12-NEXT: global_inv scope:SCOPE_SE1532; GFX12-NEXT: s_endpgm1533 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 41534 %result = atomicrmw uinc_wrap ptr addrspace(3) %gep, i64 42 syncscope("agent") seq_cst, align 81535 ret void1536}1537 1538define amdgpu_kernel void @global_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #1 {1539; CI-LABEL: global_atomic_inc_ret_i64:1540; CI: ; %bb.0:1541; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01542; CI-NEXT: s_add_i32 s12, s12, s171543; CI-NEXT: v_mov_b32_e32 v0, 421544; CI-NEXT: s_mov_b32 flat_scratch_lo, s131545; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81546; CI-NEXT: s_waitcnt lgkmcnt(0)1547; CI-NEXT: v_mov_b32_e32 v2, s21548; CI-NEXT: v_mov_b32_e32 v1, 01549; CI-NEXT: v_mov_b32_e32 v3, s31550; CI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc1551; CI-NEXT: s_waitcnt vmcnt(0)1552; CI-NEXT: buffer_wbinvl1_vol1553; CI-NEXT: v_mov_b32_e32 v3, s11554; CI-NEXT: v_mov_b32_e32 v2, s01555; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1556; CI-NEXT: s_endpgm1557;1558; VI-LABEL: global_atomic_inc_ret_i64:1559; VI: ; %bb.0:1560; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01561; VI-NEXT: s_add_i32 s12, s12, s171562; VI-NEXT: v_mov_b32_e32 v0, 421563; VI-NEXT: s_mov_b32 flat_scratch_lo, s131564; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81565; VI-NEXT: s_waitcnt lgkmcnt(0)1566; VI-NEXT: v_mov_b32_e32 v2, s21567; VI-NEXT: v_mov_b32_e32 v1, 01568; VI-NEXT: v_mov_b32_e32 v3, s31569; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc1570; VI-NEXT: s_waitcnt vmcnt(0)1571; VI-NEXT: buffer_wbinvl1_vol1572; VI-NEXT: v_mov_b32_e32 v3, s11573; VI-NEXT: v_mov_b32_e32 v2, s01574; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1575; VI-NEXT: s_endpgm1576;1577; GFX9-LABEL: global_atomic_inc_ret_i64:1578; GFX9: ; %bb.0:1579; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01580; GFX9-NEXT: v_mov_b32_e32 v0, 421581; GFX9-NEXT: v_mov_b32_e32 v1, 01582; GFX9-NEXT: v_mov_b32_e32 v2, 01583; GFX9-NEXT: s_waitcnt lgkmcnt(0)1584; GFX9-NEXT: global_atomic_inc_x2 v[0:1], v2, v[0:1], s[2:3] glc1585; GFX9-NEXT: s_waitcnt vmcnt(0)1586; GFX9-NEXT: buffer_wbinvl1_vol1587; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]1588; GFX9-NEXT: s_endpgm1589;1590; GFX10-LABEL: global_atomic_inc_ret_i64:1591; GFX10: ; %bb.0:1592; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01593; GFX10-NEXT: v_mov_b32_e32 v0, 421594; GFX10-NEXT: v_mov_b32_e32 v1, 01595; GFX10-NEXT: v_mov_b32_e32 v2, 01596; GFX10-NEXT: s_waitcnt lgkmcnt(0)1597; GFX10-NEXT: global_atomic_inc_x2 v[0:1], v2, v[0:1], s[2:3] glc1598; GFX10-NEXT: s_waitcnt vmcnt(0)1599; GFX10-NEXT: buffer_gl1_inv1600; GFX10-NEXT: buffer_gl0_inv1601; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]1602; GFX10-NEXT: s_endpgm1603;1604; GFX11-LABEL: global_atomic_inc_ret_i64:1605; GFX11: ; %bb.0:1606; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x01607; GFX11-NEXT: v_mov_b32_e32 v0, 421608; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 01609; GFX11-NEXT: s_waitcnt lgkmcnt(0)1610; GFX11-NEXT: global_atomic_inc_u64 v[0:1], v2, v[0:1], s[2:3] glc1611; GFX11-NEXT: s_waitcnt vmcnt(0)1612; GFX11-NEXT: buffer_gl1_inv1613; GFX11-NEXT: buffer_gl0_inv1614; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]1615; GFX11-NEXT: s_endpgm1616;1617; GFX12-LABEL: global_atomic_inc_ret_i64:1618; GFX12: ; %bb.0:1619; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x01620; GFX12-NEXT: v_mov_b32_e32 v0, 421621; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 01622; GFX12-NEXT: s_wait_kmcnt 0x01623; GFX12-NEXT: global_atomic_inc_u64 v[0:1], v2, v[0:1], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV1624; GFX12-NEXT: s_wait_loadcnt 0x01625; GFX12-NEXT: global_inv scope:SCOPE_DEV1626; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]1627; GFX12-NEXT: s_endpgm1628 %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !11629 store i64 %result, ptr addrspace(1) %out, align 41630 ret void1631}1632 1633define amdgpu_kernel void @global_atomic_inc_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #1 {1634; CI-LABEL: global_atomic_inc_ret_i64_offset:1635; CI: ; %bb.0:1636; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01637; CI-NEXT: s_add_i32 s12, s12, s171638; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81639; CI-NEXT: v_mov_b32_e32 v0, 421640; CI-NEXT: s_mov_b32 flat_scratch_lo, s131641; CI-NEXT: s_waitcnt lgkmcnt(0)1642; CI-NEXT: s_add_u32 s2, s2, 321643; CI-NEXT: s_addc_u32 s3, s3, 01644; CI-NEXT: v_mov_b32_e32 v2, s21645; CI-NEXT: v_mov_b32_e32 v1, 01646; CI-NEXT: v_mov_b32_e32 v3, s31647; CI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc1648; CI-NEXT: s_waitcnt vmcnt(0)1649; CI-NEXT: buffer_wbinvl1_vol1650; CI-NEXT: v_mov_b32_e32 v3, s11651; CI-NEXT: v_mov_b32_e32 v2, s01652; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1653; CI-NEXT: s_endpgm1654;1655; VI-LABEL: global_atomic_inc_ret_i64_offset:1656; VI: ; %bb.0:1657; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01658; VI-NEXT: s_add_i32 s12, s12, s171659; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81660; VI-NEXT: v_mov_b32_e32 v0, 421661; VI-NEXT: s_mov_b32 flat_scratch_lo, s131662; VI-NEXT: s_waitcnt lgkmcnt(0)1663; VI-NEXT: s_add_u32 s2, s2, 321664; VI-NEXT: s_addc_u32 s3, s3, 01665; VI-NEXT: v_mov_b32_e32 v2, s21666; VI-NEXT: v_mov_b32_e32 v1, 01667; VI-NEXT: v_mov_b32_e32 v3, s31668; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc1669; VI-NEXT: s_waitcnt vmcnt(0)1670; VI-NEXT: buffer_wbinvl1_vol1671; VI-NEXT: v_mov_b32_e32 v3, s11672; VI-NEXT: v_mov_b32_e32 v2, s01673; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1674; VI-NEXT: s_endpgm1675;1676; GFX9-LABEL: global_atomic_inc_ret_i64_offset:1677; GFX9: ; %bb.0:1678; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01679; GFX9-NEXT: v_mov_b32_e32 v0, 421680; GFX9-NEXT: v_mov_b32_e32 v1, 01681; GFX9-NEXT: v_mov_b32_e32 v2, 01682; GFX9-NEXT: s_waitcnt lgkmcnt(0)1683; GFX9-NEXT: global_atomic_inc_x2 v[0:1], v2, v[0:1], s[2:3] offset:32 glc1684; GFX9-NEXT: s_waitcnt vmcnt(0)1685; GFX9-NEXT: buffer_wbinvl1_vol1686; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]1687; GFX9-NEXT: s_endpgm1688;1689; GFX10-LABEL: global_atomic_inc_ret_i64_offset:1690; GFX10: ; %bb.0:1691; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01692; GFX10-NEXT: v_mov_b32_e32 v0, 421693; GFX10-NEXT: v_mov_b32_e32 v1, 01694; GFX10-NEXT: v_mov_b32_e32 v2, 01695; GFX10-NEXT: s_waitcnt lgkmcnt(0)1696; GFX10-NEXT: global_atomic_inc_x2 v[0:1], v2, v[0:1], s[2:3] offset:32 glc1697; GFX10-NEXT: s_waitcnt vmcnt(0)1698; GFX10-NEXT: buffer_gl1_inv1699; GFX10-NEXT: buffer_gl0_inv1700; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]1701; GFX10-NEXT: s_endpgm1702;1703; GFX11-LABEL: global_atomic_inc_ret_i64_offset:1704; GFX11: ; %bb.0:1705; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x01706; GFX11-NEXT: v_mov_b32_e32 v0, 421707; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 01708; GFX11-NEXT: s_waitcnt lgkmcnt(0)1709; GFX11-NEXT: global_atomic_inc_u64 v[0:1], v2, v[0:1], s[2:3] offset:32 glc1710; GFX11-NEXT: s_waitcnt vmcnt(0)1711; GFX11-NEXT: buffer_gl1_inv1712; GFX11-NEXT: buffer_gl0_inv1713; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]1714; GFX11-NEXT: s_endpgm1715;1716; GFX12-LABEL: global_atomic_inc_ret_i64_offset:1717; GFX12: ; %bb.0:1718; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x01719; GFX12-NEXT: v_mov_b32_e32 v0, 421720; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 01721; GFX12-NEXT: s_wait_kmcnt 0x01722; GFX12-NEXT: global_atomic_inc_u64 v[0:1], v2, v[0:1], s[2:3] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV1723; GFX12-NEXT: s_wait_loadcnt 0x01724; GFX12-NEXT: global_inv scope:SCOPE_DEV1725; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]1726; GFX12-NEXT: s_endpgm1727 %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 41728 %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !11729 store i64 %result, ptr addrspace(1) %out, align 41730 ret void1731}1732 1733define amdgpu_kernel void @global_atomic_inc_ret_i64_offset_system(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #1 {1734; CI-LABEL: global_atomic_inc_ret_i64_offset_system:1735; CI: ; %bb.0:1736; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01737; CI-NEXT: s_add_i32 s12, s12, s171738; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81739; CI-NEXT: v_mov_b32_e32 v0, 421740; CI-NEXT: s_mov_b32 flat_scratch_lo, s131741; CI-NEXT: s_waitcnt lgkmcnt(0)1742; CI-NEXT: s_add_u32 s2, s2, 321743; CI-NEXT: s_addc_u32 s3, s3, 01744; CI-NEXT: v_mov_b32_e32 v2, s21745; CI-NEXT: v_mov_b32_e32 v1, 01746; CI-NEXT: v_mov_b32_e32 v3, s31747; CI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc1748; CI-NEXT: s_waitcnt vmcnt(0)1749; CI-NEXT: buffer_wbinvl1_vol1750; CI-NEXT: v_mov_b32_e32 v3, s11751; CI-NEXT: v_mov_b32_e32 v2, s01752; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1753; CI-NEXT: s_endpgm1754;1755; VI-LABEL: global_atomic_inc_ret_i64_offset_system:1756; VI: ; %bb.0:1757; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01758; VI-NEXT: s_add_i32 s12, s12, s171759; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81760; VI-NEXT: v_mov_b32_e32 v0, 421761; VI-NEXT: s_mov_b32 flat_scratch_lo, s131762; VI-NEXT: s_waitcnt lgkmcnt(0)1763; VI-NEXT: s_add_u32 s2, s2, 321764; VI-NEXT: s_addc_u32 s3, s3, 01765; VI-NEXT: v_mov_b32_e32 v2, s21766; VI-NEXT: v_mov_b32_e32 v1, 01767; VI-NEXT: v_mov_b32_e32 v3, s31768; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc1769; VI-NEXT: s_waitcnt vmcnt(0)1770; VI-NEXT: buffer_wbinvl1_vol1771; VI-NEXT: v_mov_b32_e32 v3, s11772; VI-NEXT: v_mov_b32_e32 v2, s01773; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]1774; VI-NEXT: s_endpgm1775;1776; GFX9-LABEL: global_atomic_inc_ret_i64_offset_system:1777; GFX9: ; %bb.0:1778; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01779; GFX9-NEXT: v_mov_b32_e32 v0, 421780; GFX9-NEXT: v_mov_b32_e32 v1, 01781; GFX9-NEXT: v_mov_b32_e32 v2, 01782; GFX9-NEXT: s_waitcnt lgkmcnt(0)1783; GFX9-NEXT: global_atomic_inc_x2 v[0:1], v2, v[0:1], s[2:3] offset:32 glc1784; GFX9-NEXT: s_waitcnt vmcnt(0)1785; GFX9-NEXT: buffer_wbinvl1_vol1786; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]1787; GFX9-NEXT: s_endpgm1788;1789; GFX10-LABEL: global_atomic_inc_ret_i64_offset_system:1790; GFX10: ; %bb.0:1791; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01792; GFX10-NEXT: v_mov_b32_e32 v0, 421793; GFX10-NEXT: v_mov_b32_e32 v1, 01794; GFX10-NEXT: v_mov_b32_e32 v2, 01795; GFX10-NEXT: s_waitcnt lgkmcnt(0)1796; GFX10-NEXT: global_atomic_inc_x2 v[0:1], v2, v[0:1], s[2:3] offset:32 glc1797; GFX10-NEXT: s_waitcnt vmcnt(0)1798; GFX10-NEXT: buffer_gl1_inv1799; GFX10-NEXT: buffer_gl0_inv1800; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]1801; GFX10-NEXT: s_endpgm1802;1803; GFX11-LABEL: global_atomic_inc_ret_i64_offset_system:1804; GFX11: ; %bb.0:1805; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x01806; GFX11-NEXT: v_mov_b32_e32 v0, 421807; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 01808; GFX11-NEXT: s_waitcnt lgkmcnt(0)1809; GFX11-NEXT: global_atomic_inc_u64 v[0:1], v2, v[0:1], s[2:3] offset:32 glc1810; GFX11-NEXT: s_waitcnt vmcnt(0)1811; GFX11-NEXT: buffer_gl1_inv1812; GFX11-NEXT: buffer_gl0_inv1813; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]1814; GFX11-NEXT: s_endpgm1815;1816; GFX12-LABEL: global_atomic_inc_ret_i64_offset_system:1817; GFX12: ; %bb.0:1818; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x01819; GFX12-NEXT: v_mov_b32_e32 v0, 421820; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 01821; GFX12-NEXT: global_wb scope:SCOPE_SYS1822; GFX12-NEXT: s_wait_storecnt 0x01823; GFX12-NEXT: s_wait_kmcnt 0x01824; GFX12-NEXT: global_atomic_inc_u64 v[0:1], v2, v[0:1], s[2:3] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_SYS1825; GFX12-NEXT: s_wait_loadcnt 0x01826; GFX12-NEXT: global_inv scope:SCOPE_SYS1827; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]1828; GFX12-NEXT: s_endpgm1829 %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 41830 %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 seq_cst, align 8, !amdgpu.no.remote.memory !11831 store i64 %result, ptr addrspace(1) %out, align 41832 ret void1833}1834 1835define amdgpu_kernel void @global_atomic_inc_noret_i64(ptr addrspace(1) %ptr) #1 {1836; CI-LABEL: global_atomic_inc_noret_i64:1837; CI: ; %bb.0:1838; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01839; CI-NEXT: s_add_i32 s12, s12, s171840; CI-NEXT: v_mov_b32_e32 v0, 421841; CI-NEXT: s_mov_b32 flat_scratch_lo, s131842; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81843; CI-NEXT: s_waitcnt lgkmcnt(0)1844; CI-NEXT: v_mov_b32_e32 v3, s11845; CI-NEXT: v_mov_b32_e32 v1, 01846; CI-NEXT: v_mov_b32_e32 v2, s01847; CI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]1848; CI-NEXT: s_waitcnt vmcnt(0)1849; CI-NEXT: buffer_wbinvl1_vol1850; CI-NEXT: s_endpgm1851;1852; VI-LABEL: global_atomic_inc_noret_i64:1853; VI: ; %bb.0:1854; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01855; VI-NEXT: s_add_i32 s12, s12, s171856; VI-NEXT: v_mov_b32_e32 v0, 421857; VI-NEXT: s_mov_b32 flat_scratch_lo, s131858; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81859; VI-NEXT: s_waitcnt lgkmcnt(0)1860; VI-NEXT: v_mov_b32_e32 v3, s11861; VI-NEXT: v_mov_b32_e32 v1, 01862; VI-NEXT: v_mov_b32_e32 v2, s01863; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]1864; VI-NEXT: s_waitcnt vmcnt(0)1865; VI-NEXT: buffer_wbinvl1_vol1866; VI-NEXT: s_endpgm1867;1868; GFX9-LABEL: global_atomic_inc_noret_i64:1869; GFX9: ; %bb.0:1870; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01871; GFX9-NEXT: v_mov_b32_e32 v0, 421872; GFX9-NEXT: v_mov_b32_e32 v1, 01873; GFX9-NEXT: v_mov_b32_e32 v2, 01874; GFX9-NEXT: s_waitcnt lgkmcnt(0)1875; GFX9-NEXT: global_atomic_inc_x2 v2, v[0:1], s[0:1]1876; GFX9-NEXT: s_waitcnt vmcnt(0)1877; GFX9-NEXT: buffer_wbinvl1_vol1878; GFX9-NEXT: s_endpgm1879;1880; GFX10-LABEL: global_atomic_inc_noret_i64:1881; GFX10: ; %bb.0:1882; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01883; GFX10-NEXT: v_mov_b32_e32 v0, 421884; GFX10-NEXT: v_mov_b32_e32 v1, 01885; GFX10-NEXT: v_mov_b32_e32 v2, 01886; GFX10-NEXT: s_waitcnt lgkmcnt(0)1887; GFX10-NEXT: global_atomic_inc_x2 v2, v[0:1], s[0:1]1888; GFX10-NEXT: s_waitcnt_vscnt null, 0x01889; GFX10-NEXT: buffer_gl1_inv1890; GFX10-NEXT: buffer_gl0_inv1891; GFX10-NEXT: s_endpgm1892;1893; GFX11-LABEL: global_atomic_inc_noret_i64:1894; GFX11: ; %bb.0:1895; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x01896; GFX11-NEXT: v_mov_b32_e32 v0, 421897; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 01898; GFX11-NEXT: s_waitcnt lgkmcnt(0)1899; GFX11-NEXT: global_atomic_inc_u64 v2, v[0:1], s[0:1]1900; GFX11-NEXT: s_waitcnt_vscnt null, 0x01901; GFX11-NEXT: buffer_gl1_inv1902; GFX11-NEXT: buffer_gl0_inv1903; GFX11-NEXT: s_endpgm1904;1905; GFX12-LABEL: global_atomic_inc_noret_i64:1906; GFX12: ; %bb.0:1907; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x01908; GFX12-NEXT: v_mov_b32_e32 v0, 421909; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 01910; GFX12-NEXT: s_wait_kmcnt 0x01911; GFX12-NEXT: global_atomic_inc_u64 v2, v[0:1], s[0:1] scope:SCOPE_DEV1912; GFX12-NEXT: s_wait_storecnt 0x01913; GFX12-NEXT: global_inv scope:SCOPE_DEV1914; GFX12-NEXT: s_endpgm1915 %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !11916 ret void1917}1918 1919define amdgpu_kernel void @global_atomic_inc_noret_i64_offset(ptr addrspace(1) %ptr) #1 {1920; CI-LABEL: global_atomic_inc_noret_i64_offset:1921; CI: ; %bb.0:1922; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01923; CI-NEXT: s_add_i32 s12, s12, s171924; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81925; CI-NEXT: v_mov_b32_e32 v0, 421926; CI-NEXT: s_mov_b32 flat_scratch_lo, s131927; CI-NEXT: s_waitcnt lgkmcnt(0)1928; CI-NEXT: s_add_u32 s0, s0, 321929; CI-NEXT: s_addc_u32 s1, s1, 01930; CI-NEXT: v_mov_b32_e32 v3, s11931; CI-NEXT: v_mov_b32_e32 v1, 01932; CI-NEXT: v_mov_b32_e32 v2, s01933; CI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]1934; CI-NEXT: s_waitcnt vmcnt(0)1935; CI-NEXT: buffer_wbinvl1_vol1936; CI-NEXT: s_endpgm1937;1938; VI-LABEL: global_atomic_inc_noret_i64_offset:1939; VI: ; %bb.0:1940; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01941; VI-NEXT: s_add_i32 s12, s12, s171942; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81943; VI-NEXT: v_mov_b32_e32 v0, 421944; VI-NEXT: s_mov_b32 flat_scratch_lo, s131945; VI-NEXT: s_waitcnt lgkmcnt(0)1946; VI-NEXT: s_add_u32 s0, s0, 321947; VI-NEXT: s_addc_u32 s1, s1, 01948; VI-NEXT: v_mov_b32_e32 v3, s11949; VI-NEXT: v_mov_b32_e32 v1, 01950; VI-NEXT: v_mov_b32_e32 v2, s01951; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]1952; VI-NEXT: s_waitcnt vmcnt(0)1953; VI-NEXT: buffer_wbinvl1_vol1954; VI-NEXT: s_endpgm1955;1956; GFX9-LABEL: global_atomic_inc_noret_i64_offset:1957; GFX9: ; %bb.0:1958; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01959; GFX9-NEXT: v_mov_b32_e32 v0, 421960; GFX9-NEXT: v_mov_b32_e32 v1, 01961; GFX9-NEXT: v_mov_b32_e32 v2, 01962; GFX9-NEXT: s_waitcnt lgkmcnt(0)1963; GFX9-NEXT: global_atomic_inc_x2 v2, v[0:1], s[0:1] offset:321964; GFX9-NEXT: s_waitcnt vmcnt(0)1965; GFX9-NEXT: buffer_wbinvl1_vol1966; GFX9-NEXT: s_endpgm1967;1968; GFX10-LABEL: global_atomic_inc_noret_i64_offset:1969; GFX10: ; %bb.0:1970; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01971; GFX10-NEXT: v_mov_b32_e32 v0, 421972; GFX10-NEXT: v_mov_b32_e32 v1, 01973; GFX10-NEXT: v_mov_b32_e32 v2, 01974; GFX10-NEXT: s_waitcnt lgkmcnt(0)1975; GFX10-NEXT: global_atomic_inc_x2 v2, v[0:1], s[0:1] offset:321976; GFX10-NEXT: s_waitcnt_vscnt null, 0x01977; GFX10-NEXT: buffer_gl1_inv1978; GFX10-NEXT: buffer_gl0_inv1979; GFX10-NEXT: s_endpgm1980;1981; GFX11-LABEL: global_atomic_inc_noret_i64_offset:1982; GFX11: ; %bb.0:1983; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x01984; GFX11-NEXT: v_mov_b32_e32 v0, 421985; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 01986; GFX11-NEXT: s_waitcnt lgkmcnt(0)1987; GFX11-NEXT: global_atomic_inc_u64 v2, v[0:1], s[0:1] offset:321988; GFX11-NEXT: s_waitcnt_vscnt null, 0x01989; GFX11-NEXT: buffer_gl1_inv1990; GFX11-NEXT: buffer_gl0_inv1991; GFX11-NEXT: s_endpgm1992;1993; GFX12-LABEL: global_atomic_inc_noret_i64_offset:1994; GFX12: ; %bb.0:1995; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x01996; GFX12-NEXT: v_mov_b32_e32 v0, 421997; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 01998; GFX12-NEXT: s_wait_kmcnt 0x01999; GFX12-NEXT: global_atomic_inc_u64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_DEV2000; GFX12-NEXT: s_wait_storecnt 0x02001; GFX12-NEXT: global_inv scope:SCOPE_DEV2002; GFX12-NEXT: s_endpgm2003 %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 42004 %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !12005 ret void2006}2007 2008define amdgpu_kernel void @global_atomic_inc_noret_i64_offset_system(ptr addrspace(1) %ptr) #1 {2009; CI-LABEL: global_atomic_inc_noret_i64_offset_system:2010; CI: ; %bb.0:2011; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02012; CI-NEXT: s_add_i32 s12, s12, s172013; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82014; CI-NEXT: v_mov_b32_e32 v0, 422015; CI-NEXT: s_mov_b32 flat_scratch_lo, s132016; CI-NEXT: s_waitcnt lgkmcnt(0)2017; CI-NEXT: s_add_u32 s0, s0, 322018; CI-NEXT: s_addc_u32 s1, s1, 02019; CI-NEXT: v_mov_b32_e32 v3, s12020; CI-NEXT: v_mov_b32_e32 v1, 02021; CI-NEXT: v_mov_b32_e32 v2, s02022; CI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]2023; CI-NEXT: s_waitcnt vmcnt(0)2024; CI-NEXT: buffer_wbinvl1_vol2025; CI-NEXT: s_endpgm2026;2027; VI-LABEL: global_atomic_inc_noret_i64_offset_system:2028; VI: ; %bb.0:2029; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02030; VI-NEXT: s_add_i32 s12, s12, s172031; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82032; VI-NEXT: v_mov_b32_e32 v0, 422033; VI-NEXT: s_mov_b32 flat_scratch_lo, s132034; VI-NEXT: s_waitcnt lgkmcnt(0)2035; VI-NEXT: s_add_u32 s0, s0, 322036; VI-NEXT: s_addc_u32 s1, s1, 02037; VI-NEXT: v_mov_b32_e32 v3, s12038; VI-NEXT: v_mov_b32_e32 v1, 02039; VI-NEXT: v_mov_b32_e32 v2, s02040; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]2041; VI-NEXT: s_waitcnt vmcnt(0)2042; VI-NEXT: buffer_wbinvl1_vol2043; VI-NEXT: s_endpgm2044;2045; GFX9-LABEL: global_atomic_inc_noret_i64_offset_system:2046; GFX9: ; %bb.0:2047; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02048; GFX9-NEXT: v_mov_b32_e32 v0, 422049; GFX9-NEXT: v_mov_b32_e32 v1, 02050; GFX9-NEXT: v_mov_b32_e32 v2, 02051; GFX9-NEXT: s_waitcnt lgkmcnt(0)2052; GFX9-NEXT: global_atomic_inc_x2 v2, v[0:1], s[0:1] offset:322053; GFX9-NEXT: s_waitcnt vmcnt(0)2054; GFX9-NEXT: buffer_wbinvl1_vol2055; GFX9-NEXT: s_endpgm2056;2057; GFX10-LABEL: global_atomic_inc_noret_i64_offset_system:2058; GFX10: ; %bb.0:2059; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02060; GFX10-NEXT: v_mov_b32_e32 v0, 422061; GFX10-NEXT: v_mov_b32_e32 v1, 02062; GFX10-NEXT: v_mov_b32_e32 v2, 02063; GFX10-NEXT: s_waitcnt lgkmcnt(0)2064; GFX10-NEXT: global_atomic_inc_x2 v2, v[0:1], s[0:1] offset:322065; GFX10-NEXT: s_waitcnt_vscnt null, 0x02066; GFX10-NEXT: buffer_gl1_inv2067; GFX10-NEXT: buffer_gl0_inv2068; GFX10-NEXT: s_endpgm2069;2070; GFX11-LABEL: global_atomic_inc_noret_i64_offset_system:2071; GFX11: ; %bb.0:2072; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x02073; GFX11-NEXT: v_mov_b32_e32 v0, 422074; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 02075; GFX11-NEXT: s_waitcnt lgkmcnt(0)2076; GFX11-NEXT: global_atomic_inc_u64 v2, v[0:1], s[0:1] offset:322077; GFX11-NEXT: s_waitcnt_vscnt null, 0x02078; GFX11-NEXT: buffer_gl1_inv2079; GFX11-NEXT: buffer_gl0_inv2080; GFX11-NEXT: s_endpgm2081;2082; GFX12-LABEL: global_atomic_inc_noret_i64_offset_system:2083; GFX12: ; %bb.0:2084; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x02085; GFX12-NEXT: v_mov_b32_e32 v0, 422086; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 02087; GFX12-NEXT: global_wb scope:SCOPE_SYS2088; GFX12-NEXT: s_wait_storecnt 0x02089; GFX12-NEXT: s_wait_kmcnt 0x02090; GFX12-NEXT: global_atomic_inc_u64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_SYS2091; GFX12-NEXT: s_wait_storecnt 0x02092; GFX12-NEXT: global_inv scope:SCOPE_SYS2093; GFX12-NEXT: s_endpgm2094 %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 42095 %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 seq_cst, align 8, !amdgpu.no.remote.memory !12096 ret void2097}2098 2099define amdgpu_kernel void @global_atomic_inc_ret_i64_offset_addr64(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #1 {2100; CI-LABEL: global_atomic_inc_ret_i64_offset_addr64:2101; CI: ; %bb.0:2102; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02103; CI-NEXT: v_lshlrev_b32_e32 v4, 3, v02104; CI-NEXT: s_add_i32 s12, s12, s172105; CI-NEXT: s_mov_b32 flat_scratch_lo, s132106; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82107; CI-NEXT: s_waitcnt lgkmcnt(0)2108; CI-NEXT: v_mov_b32_e32 v0, s22109; CI-NEXT: v_mov_b32_e32 v1, s32110; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v42111; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc2112; CI-NEXT: v_mov_b32_e32 v0, 422113; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v22114; CI-NEXT: v_mov_b32_e32 v1, 02115; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc2116; CI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc2117; CI-NEXT: s_waitcnt vmcnt(0)2118; CI-NEXT: buffer_wbinvl1_vol2119; CI-NEXT: v_mov_b32_e32 v3, s12120; CI-NEXT: v_mov_b32_e32 v2, s02121; CI-NEXT: v_add_i32_e32 v2, vcc, v2, v42122; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc2123; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]2124; CI-NEXT: s_endpgm2125;2126; VI-LABEL: global_atomic_inc_ret_i64_offset_addr64:2127; VI: ; %bb.0:2128; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02129; VI-NEXT: v_lshlrev_b32_e32 v4, 3, v02130; VI-NEXT: s_add_i32 s12, s12, s172131; VI-NEXT: s_mov_b32 flat_scratch_lo, s132132; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82133; VI-NEXT: s_waitcnt lgkmcnt(0)2134; VI-NEXT: v_mov_b32_e32 v0, s22135; VI-NEXT: v_mov_b32_e32 v1, s32136; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v42137; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc2138; VI-NEXT: v_mov_b32_e32 v0, 422139; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v22140; VI-NEXT: v_mov_b32_e32 v1, 02141; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc2142; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc2143; VI-NEXT: s_waitcnt vmcnt(0)2144; VI-NEXT: buffer_wbinvl1_vol2145; VI-NEXT: v_mov_b32_e32 v3, s12146; VI-NEXT: v_mov_b32_e32 v2, s02147; VI-NEXT: v_add_u32_e32 v2, vcc, v2, v42148; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc2149; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]2150; VI-NEXT: s_endpgm2151;2152; GFX9-LABEL: global_atomic_inc_ret_i64_offset_addr64:2153; GFX9: ; %bb.0:2154; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02155; GFX9-NEXT: v_mov_b32_e32 v1, 422156; GFX9-NEXT: v_mov_b32_e32 v2, 02157; GFX9-NEXT: v_lshlrev_b32_e32 v3, 3, v02158; GFX9-NEXT: s_waitcnt lgkmcnt(0)2159; GFX9-NEXT: global_atomic_inc_x2 v[0:1], v3, v[1:2], s[2:3] offset:40 glc2160; GFX9-NEXT: s_waitcnt vmcnt(0)2161; GFX9-NEXT: buffer_wbinvl1_vol2162; GFX9-NEXT: global_store_dwordx2 v3, v[0:1], s[0:1]2163; GFX9-NEXT: s_endpgm2164;2165; GFX10-LABEL: global_atomic_inc_ret_i64_offset_addr64:2166; GFX10: ; %bb.0:2167; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02168; GFX10-NEXT: v_mov_b32_e32 v1, 422169; GFX10-NEXT: v_mov_b32_e32 v2, 02170; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v02171; GFX10-NEXT: s_waitcnt lgkmcnt(0)2172; GFX10-NEXT: global_atomic_inc_x2 v[0:1], v3, v[1:2], s[2:3] offset:40 glc2173; GFX10-NEXT: s_waitcnt vmcnt(0)2174; GFX10-NEXT: buffer_gl1_inv2175; GFX10-NEXT: buffer_gl0_inv2176; GFX10-NEXT: global_store_dwordx2 v3, v[0:1], s[0:1]2177; GFX10-NEXT: s_endpgm2178;2179; GFX11-LABEL: global_atomic_inc_ret_i64_offset_addr64:2180; GFX11: ; %bb.0:2181; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x02182; GFX11-NEXT: v_and_b32_e32 v2, 0x3ff, v02183; GFX11-NEXT: v_mov_b32_e32 v0, 422184; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)2185; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v2, 3, v22186; GFX11-NEXT: s_waitcnt lgkmcnt(0)2187; GFX11-NEXT: global_atomic_inc_u64 v[0:1], v2, v[0:1], s[2:3] offset:40 glc2188; GFX11-NEXT: s_waitcnt vmcnt(0)2189; GFX11-NEXT: buffer_gl1_inv2190; GFX11-NEXT: buffer_gl0_inv2191; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]2192; GFX11-NEXT: s_endpgm2193;2194; GFX12-LABEL: global_atomic_inc_ret_i64_offset_addr64:2195; GFX12: ; %bb.0:2196; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x02197; GFX12-NEXT: v_and_b32_e32 v2, 0x3ff, v02198; GFX12-NEXT: v_mov_b32_e32 v0, 422199; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)2200; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v2, 3, v22201; GFX12-NEXT: s_wait_kmcnt 0x02202; GFX12-NEXT: global_atomic_inc_u64 v[0:1], v2, v[0:1], s[2:3] offset:40 th:TH_ATOMIC_RETURN scope:SCOPE_DEV2203; GFX12-NEXT: s_wait_loadcnt 0x02204; GFX12-NEXT: global_inv scope:SCOPE_DEV2205; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]2206; GFX12-NEXT: s_endpgm2207 %id = call i32 @llvm.amdgcn.workitem.id.x()2208 %gep.tid = getelementptr i64, ptr addrspace(1) %ptr, i32 %id2209 %out.gep = getelementptr i64, ptr addrspace(1) %out, i32 %id2210 %gep = getelementptr i64, ptr addrspace(1) %gep.tid, i32 52211 %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !12212 store i64 %result, ptr addrspace(1) %out.gep, align 42213 ret void2214}2215 2216define amdgpu_kernel void @global_atomic_inc_noret_i64_offset_addr64(ptr addrspace(1) %ptr) #1 {2217; CI-LABEL: global_atomic_inc_noret_i64_offset_addr64:2218; CI: ; %bb.0:2219; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02220; CI-NEXT: v_lshlrev_b32_e32 v2, 3, v02221; CI-NEXT: s_add_i32 s12, s12, s172222; CI-NEXT: s_mov_b32 flat_scratch_lo, s132223; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82224; CI-NEXT: s_waitcnt lgkmcnt(0)2225; CI-NEXT: v_mov_b32_e32 v0, s02226; CI-NEXT: v_mov_b32_e32 v1, s12227; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v22228; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc2229; CI-NEXT: v_mov_b32_e32 v0, 422230; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v22231; CI-NEXT: v_mov_b32_e32 v1, 02232; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc2233; CI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]2234; CI-NEXT: s_waitcnt vmcnt(0)2235; CI-NEXT: buffer_wbinvl1_vol2236; CI-NEXT: s_endpgm2237;2238; VI-LABEL: global_atomic_inc_noret_i64_offset_addr64:2239; VI: ; %bb.0:2240; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02241; VI-NEXT: v_lshlrev_b32_e32 v2, 3, v02242; VI-NEXT: s_add_i32 s12, s12, s172243; VI-NEXT: s_mov_b32 flat_scratch_lo, s132244; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82245; VI-NEXT: s_waitcnt lgkmcnt(0)2246; VI-NEXT: v_mov_b32_e32 v0, s02247; VI-NEXT: v_mov_b32_e32 v1, s12248; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v22249; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc2250; VI-NEXT: v_mov_b32_e32 v0, 422251; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v22252; VI-NEXT: v_mov_b32_e32 v1, 02253; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc2254; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]2255; VI-NEXT: s_waitcnt vmcnt(0)2256; VI-NEXT: buffer_wbinvl1_vol2257; VI-NEXT: s_endpgm2258;2259; GFX9-LABEL: global_atomic_inc_noret_i64_offset_addr64:2260; GFX9: ; %bb.0:2261; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02262; GFX9-NEXT: v_mov_b32_e32 v1, 422263; GFX9-NEXT: v_mov_b32_e32 v2, 02264; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v02265; GFX9-NEXT: s_waitcnt lgkmcnt(0)2266; GFX9-NEXT: global_atomic_inc_x2 v0, v[1:2], s[0:1] offset:402267; GFX9-NEXT: s_waitcnt vmcnt(0)2268; GFX9-NEXT: buffer_wbinvl1_vol2269; GFX9-NEXT: s_endpgm2270;2271; GFX10-LABEL: global_atomic_inc_noret_i64_offset_addr64:2272; GFX10: ; %bb.0:2273; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02274; GFX10-NEXT: v_mov_b32_e32 v1, 422275; GFX10-NEXT: v_mov_b32_e32 v2, 02276; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v02277; GFX10-NEXT: s_waitcnt lgkmcnt(0)2278; GFX10-NEXT: global_atomic_inc_x2 v0, v[1:2], s[0:1] offset:402279; GFX10-NEXT: s_waitcnt_vscnt null, 0x02280; GFX10-NEXT: buffer_gl1_inv2281; GFX10-NEXT: buffer_gl0_inv2282; GFX10-NEXT: s_endpgm2283;2284; GFX11-LABEL: global_atomic_inc_noret_i64_offset_addr64:2285; GFX11: ; %bb.0:2286; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x02287; GFX11-NEXT: v_and_b32_e32 v2, 0x3ff, v02288; GFX11-NEXT: v_mov_b32_e32 v0, 422289; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)2290; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v2, 3, v22291; GFX11-NEXT: s_waitcnt lgkmcnt(0)2292; GFX11-NEXT: global_atomic_inc_u64 v2, v[0:1], s[0:1] offset:402293; GFX11-NEXT: s_waitcnt_vscnt null, 0x02294; GFX11-NEXT: buffer_gl1_inv2295; GFX11-NEXT: buffer_gl0_inv2296; GFX11-NEXT: s_endpgm2297;2298; GFX12-LABEL: global_atomic_inc_noret_i64_offset_addr64:2299; GFX12: ; %bb.0:2300; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x02301; GFX12-NEXT: v_and_b32_e32 v2, 0x3ff, v02302; GFX12-NEXT: v_mov_b32_e32 v0, 422303; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)2304; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v2, 3, v22305; GFX12-NEXT: s_wait_kmcnt 0x02306; GFX12-NEXT: global_atomic_inc_u64 v2, v[0:1], s[0:1] offset:40 scope:SCOPE_DEV2307; GFX12-NEXT: s_wait_storecnt 0x02308; GFX12-NEXT: global_inv scope:SCOPE_DEV2309; GFX12-NEXT: s_endpgm2310 %id = call i32 @llvm.amdgcn.workitem.id.x()2311 %gep.tid = getelementptr i64, ptr addrspace(1) %ptr, i32 %id2312 %gep = getelementptr i64, ptr addrspace(1) %gep.tid, i32 52313 %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !12314 ret void2315}2316 2317define amdgpu_kernel void @flat_atomic_inc_ret_i32(ptr %out, ptr %ptr) #1 {2318; CI-LABEL: flat_atomic_inc_ret_i32:2319; CI: ; %bb.0:2320; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02321; CI-NEXT: s_add_i32 s12, s12, s172322; CI-NEXT: s_mov_b32 flat_scratch_lo, s132323; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82324; CI-NEXT: v_mov_b32_e32 v2, 422325; CI-NEXT: s_waitcnt lgkmcnt(0)2326; CI-NEXT: v_mov_b32_e32 v0, s22327; CI-NEXT: v_mov_b32_e32 v1, s32328; CI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc2329; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2330; CI-NEXT: buffer_wbinvl1_vol2331; CI-NEXT: v_mov_b32_e32 v0, s02332; CI-NEXT: v_mov_b32_e32 v1, s12333; CI-NEXT: flat_store_dword v[0:1], v22334; CI-NEXT: s_endpgm2335;2336; VI-LABEL: flat_atomic_inc_ret_i32:2337; VI: ; %bb.0:2338; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02339; VI-NEXT: s_add_i32 s12, s12, s172340; VI-NEXT: s_mov_b32 flat_scratch_lo, s132341; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82342; VI-NEXT: v_mov_b32_e32 v2, 422343; VI-NEXT: s_waitcnt lgkmcnt(0)2344; VI-NEXT: v_mov_b32_e32 v0, s22345; VI-NEXT: v_mov_b32_e32 v1, s32346; VI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc2347; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2348; VI-NEXT: buffer_wbinvl1_vol2349; VI-NEXT: v_mov_b32_e32 v0, s02350; VI-NEXT: v_mov_b32_e32 v1, s12351; VI-NEXT: flat_store_dword v[0:1], v22352; VI-NEXT: s_endpgm2353;2354; GFX9-LABEL: flat_atomic_inc_ret_i32:2355; GFX9: ; %bb.0:2356; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02357; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s172358; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 02359; GFX9-NEXT: v_mov_b32_e32 v2, 422360; GFX9-NEXT: s_waitcnt lgkmcnt(0)2361; GFX9-NEXT: v_mov_b32_e32 v0, s22362; GFX9-NEXT: v_mov_b32_e32 v1, s32363; GFX9-NEXT: flat_atomic_inc v2, v[0:1], v2 glc2364; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2365; GFX9-NEXT: buffer_wbinvl1_vol2366; GFX9-NEXT: v_mov_b32_e32 v0, s02367; GFX9-NEXT: v_mov_b32_e32 v1, s12368; GFX9-NEXT: flat_store_dword v[0:1], v22369; GFX9-NEXT: s_endpgm2370;2371; GFX10-LABEL: flat_atomic_inc_ret_i32:2372; GFX10: ; %bb.0:2373; GFX10-NEXT: s_add_u32 s12, s12, s172374; GFX10-NEXT: s_addc_u32 s13, s13, 02375; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s122376; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s132377; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02378; GFX10-NEXT: v_mov_b32_e32 v2, 422379; GFX10-NEXT: s_waitcnt lgkmcnt(0)2380; GFX10-NEXT: v_mov_b32_e32 v0, s22381; GFX10-NEXT: v_mov_b32_e32 v1, s32382; GFX10-NEXT: flat_atomic_inc v2, v[0:1], v2 glc2383; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2384; GFX10-NEXT: buffer_gl1_inv2385; GFX10-NEXT: buffer_gl0_inv2386; GFX10-NEXT: v_mov_b32_e32 v0, s02387; GFX10-NEXT: v_mov_b32_e32 v1, s12388; GFX10-NEXT: flat_store_dword v[0:1], v22389; GFX10-NEXT: s_endpgm2390;2391; GFX11-LABEL: flat_atomic_inc_ret_i32:2392; GFX11: ; %bb.0:2393; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x02394; GFX11-NEXT: v_mov_b32_e32 v2, 422395; GFX11-NEXT: s_waitcnt lgkmcnt(0)2396; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s32397; GFX11-NEXT: flat_atomic_inc_u32 v2, v[0:1], v2 glc2398; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2399; GFX11-NEXT: buffer_gl1_inv2400; GFX11-NEXT: buffer_gl0_inv2401; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12402; GFX11-NEXT: flat_store_b32 v[0:1], v22403; GFX11-NEXT: s_endpgm2404;2405; GFX12-LABEL: flat_atomic_inc_ret_i32:2406; GFX12: ; %bb.0:2407; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x02408; GFX12-NEXT: v_mov_b32_e32 v2, 422409; GFX12-NEXT: s_wait_kmcnt 0x02410; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s32411; GFX12-NEXT: flat_atomic_inc_u32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV2412; GFX12-NEXT: s_wait_loadcnt_dscnt 0x02413; GFX12-NEXT: global_inv scope:SCOPE_DEV2414; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12415; GFX12-NEXT: flat_store_b32 v[0:1], v22416; GFX12-NEXT: s_endpgm2417 %result = atomicrmw uinc_wrap ptr %ptr, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !12418 store i32 %result, ptr %out, align 42419 ret void2420}2421 2422define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset(ptr %out, ptr %ptr) #1 {2423; CI-LABEL: flat_atomic_inc_ret_i32_offset:2424; CI: ; %bb.0:2425; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02426; CI-NEXT: s_add_i32 s12, s12, s172427; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82428; CI-NEXT: s_mov_b32 flat_scratch_lo, s132429; CI-NEXT: v_mov_b32_e32 v2, 422430; CI-NEXT: s_waitcnt lgkmcnt(0)2431; CI-NEXT: s_add_u32 s2, s2, 162432; CI-NEXT: s_addc_u32 s3, s3, 02433; CI-NEXT: v_mov_b32_e32 v0, s22434; CI-NEXT: v_mov_b32_e32 v1, s32435; CI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc2436; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2437; CI-NEXT: buffer_wbinvl1_vol2438; CI-NEXT: v_mov_b32_e32 v0, s02439; CI-NEXT: v_mov_b32_e32 v1, s12440; CI-NEXT: flat_store_dword v[0:1], v22441; CI-NEXT: s_endpgm2442;2443; VI-LABEL: flat_atomic_inc_ret_i32_offset:2444; VI: ; %bb.0:2445; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02446; VI-NEXT: s_add_i32 s12, s12, s172447; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82448; VI-NEXT: s_mov_b32 flat_scratch_lo, s132449; VI-NEXT: v_mov_b32_e32 v2, 422450; VI-NEXT: s_waitcnt lgkmcnt(0)2451; VI-NEXT: s_add_u32 s2, s2, 162452; VI-NEXT: s_addc_u32 s3, s3, 02453; VI-NEXT: v_mov_b32_e32 v0, s22454; VI-NEXT: v_mov_b32_e32 v1, s32455; VI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc2456; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2457; VI-NEXT: buffer_wbinvl1_vol2458; VI-NEXT: v_mov_b32_e32 v0, s02459; VI-NEXT: v_mov_b32_e32 v1, s12460; VI-NEXT: flat_store_dword v[0:1], v22461; VI-NEXT: s_endpgm2462;2463; GFX9-LABEL: flat_atomic_inc_ret_i32_offset:2464; GFX9: ; %bb.0:2465; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02466; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s172467; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 02468; GFX9-NEXT: v_mov_b32_e32 v2, 422469; GFX9-NEXT: s_waitcnt lgkmcnt(0)2470; GFX9-NEXT: v_mov_b32_e32 v0, s22471; GFX9-NEXT: v_mov_b32_e32 v1, s32472; GFX9-NEXT: flat_atomic_inc v2, v[0:1], v2 offset:16 glc2473; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2474; GFX9-NEXT: buffer_wbinvl1_vol2475; GFX9-NEXT: v_mov_b32_e32 v0, s02476; GFX9-NEXT: v_mov_b32_e32 v1, s12477; GFX9-NEXT: flat_store_dword v[0:1], v22478; GFX9-NEXT: s_endpgm2479;2480; GFX10-LABEL: flat_atomic_inc_ret_i32_offset:2481; GFX10: ; %bb.0:2482; GFX10-NEXT: s_add_u32 s12, s12, s172483; GFX10-NEXT: s_addc_u32 s13, s13, 02484; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s122485; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s132486; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02487; GFX10-NEXT: v_mov_b32_e32 v2, 422488; GFX10-NEXT: s_waitcnt lgkmcnt(0)2489; GFX10-NEXT: s_add_u32 s2, s2, 162490; GFX10-NEXT: s_addc_u32 s3, s3, 02491; GFX10-NEXT: v_mov_b32_e32 v0, s22492; GFX10-NEXT: v_mov_b32_e32 v1, s32493; GFX10-NEXT: flat_atomic_inc v2, v[0:1], v2 glc2494; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2495; GFX10-NEXT: buffer_gl1_inv2496; GFX10-NEXT: buffer_gl0_inv2497; GFX10-NEXT: v_mov_b32_e32 v0, s02498; GFX10-NEXT: v_mov_b32_e32 v1, s12499; GFX10-NEXT: flat_store_dword v[0:1], v22500; GFX10-NEXT: s_endpgm2501;2502; GFX11-LABEL: flat_atomic_inc_ret_i32_offset:2503; GFX11: ; %bb.0:2504; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x02505; GFX11-NEXT: v_mov_b32_e32 v2, 422506; GFX11-NEXT: s_waitcnt lgkmcnt(0)2507; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s32508; GFX11-NEXT: flat_atomic_inc_u32 v2, v[0:1], v2 offset:16 glc2509; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2510; GFX11-NEXT: buffer_gl1_inv2511; GFX11-NEXT: buffer_gl0_inv2512; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12513; GFX11-NEXT: flat_store_b32 v[0:1], v22514; GFX11-NEXT: s_endpgm2515;2516; GFX12-LABEL: flat_atomic_inc_ret_i32_offset:2517; GFX12: ; %bb.0:2518; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x02519; GFX12-NEXT: v_mov_b32_e32 v2, 422520; GFX12-NEXT: s_wait_kmcnt 0x02521; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s32522; GFX12-NEXT: flat_atomic_inc_u32 v2, v[0:1], v2 offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV2523; GFX12-NEXT: s_wait_loadcnt_dscnt 0x02524; GFX12-NEXT: global_inv scope:SCOPE_DEV2525; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12526; GFX12-NEXT: flat_store_b32 v[0:1], v22527; GFX12-NEXT: s_endpgm2528 %gep = getelementptr inbounds i32, ptr %ptr, i32 42529 %result = atomicrmw uinc_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !12530 store i32 %result, ptr %out, align 42531 ret void2532}2533 2534define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_system(ptr %out, ptr %ptr) #1 {2535; CI-LABEL: flat_atomic_inc_ret_i32_offset_system:2536; CI: ; %bb.0:2537; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02538; CI-NEXT: s_add_i32 s12, s12, s172539; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82540; CI-NEXT: s_mov_b32 flat_scratch_lo, s132541; CI-NEXT: v_mov_b32_e32 v2, 422542; CI-NEXT: s_waitcnt lgkmcnt(0)2543; CI-NEXT: s_add_u32 s2, s2, 162544; CI-NEXT: s_addc_u32 s3, s3, 02545; CI-NEXT: v_mov_b32_e32 v0, s22546; CI-NEXT: v_mov_b32_e32 v1, s32547; CI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc2548; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2549; CI-NEXT: buffer_wbinvl1_vol2550; CI-NEXT: v_mov_b32_e32 v0, s02551; CI-NEXT: v_mov_b32_e32 v1, s12552; CI-NEXT: flat_store_dword v[0:1], v22553; CI-NEXT: s_endpgm2554;2555; VI-LABEL: flat_atomic_inc_ret_i32_offset_system:2556; VI: ; %bb.0:2557; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02558; VI-NEXT: s_add_i32 s12, s12, s172559; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82560; VI-NEXT: s_mov_b32 flat_scratch_lo, s132561; VI-NEXT: v_mov_b32_e32 v2, 422562; VI-NEXT: s_waitcnt lgkmcnt(0)2563; VI-NEXT: s_add_u32 s2, s2, 162564; VI-NEXT: s_addc_u32 s3, s3, 02565; VI-NEXT: v_mov_b32_e32 v0, s22566; VI-NEXT: v_mov_b32_e32 v1, s32567; VI-NEXT: flat_atomic_inc v2, v[0:1], v2 glc2568; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2569; VI-NEXT: buffer_wbinvl1_vol2570; VI-NEXT: v_mov_b32_e32 v0, s02571; VI-NEXT: v_mov_b32_e32 v1, s12572; VI-NEXT: flat_store_dword v[0:1], v22573; VI-NEXT: s_endpgm2574;2575; GFX9-LABEL: flat_atomic_inc_ret_i32_offset_system:2576; GFX9: ; %bb.0:2577; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02578; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s172579; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 02580; GFX9-NEXT: v_mov_b32_e32 v2, 422581; GFX9-NEXT: s_waitcnt lgkmcnt(0)2582; GFX9-NEXT: v_mov_b32_e32 v0, s22583; GFX9-NEXT: v_mov_b32_e32 v1, s32584; GFX9-NEXT: flat_atomic_inc v2, v[0:1], v2 offset:16 glc2585; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2586; GFX9-NEXT: buffer_wbinvl1_vol2587; GFX9-NEXT: v_mov_b32_e32 v0, s02588; GFX9-NEXT: v_mov_b32_e32 v1, s12589; GFX9-NEXT: flat_store_dword v[0:1], v22590; GFX9-NEXT: s_endpgm2591;2592; GFX10-LABEL: flat_atomic_inc_ret_i32_offset_system:2593; GFX10: ; %bb.0:2594; GFX10-NEXT: s_add_u32 s12, s12, s172595; GFX10-NEXT: s_addc_u32 s13, s13, 02596; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s122597; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s132598; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02599; GFX10-NEXT: v_mov_b32_e32 v2, 422600; GFX10-NEXT: s_waitcnt lgkmcnt(0)2601; GFX10-NEXT: s_add_u32 s2, s2, 162602; GFX10-NEXT: s_addc_u32 s3, s3, 02603; GFX10-NEXT: v_mov_b32_e32 v0, s22604; GFX10-NEXT: v_mov_b32_e32 v1, s32605; GFX10-NEXT: flat_atomic_inc v2, v[0:1], v2 glc2606; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2607; GFX10-NEXT: buffer_gl1_inv2608; GFX10-NEXT: buffer_gl0_inv2609; GFX10-NEXT: v_mov_b32_e32 v0, s02610; GFX10-NEXT: v_mov_b32_e32 v1, s12611; GFX10-NEXT: flat_store_dword v[0:1], v22612; GFX10-NEXT: s_endpgm2613;2614; GFX11-LABEL: flat_atomic_inc_ret_i32_offset_system:2615; GFX11: ; %bb.0:2616; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x02617; GFX11-NEXT: v_mov_b32_e32 v2, 422618; GFX11-NEXT: s_waitcnt lgkmcnt(0)2619; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s32620; GFX11-NEXT: flat_atomic_inc_u32 v2, v[0:1], v2 offset:16 glc2621; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2622; GFX11-NEXT: buffer_gl1_inv2623; GFX11-NEXT: buffer_gl0_inv2624; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12625; GFX11-NEXT: flat_store_b32 v[0:1], v22626; GFX11-NEXT: s_endpgm2627;2628; GFX12-LABEL: flat_atomic_inc_ret_i32_offset_system:2629; GFX12: ; %bb.0:2630; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x02631; GFX12-NEXT: v_mov_b32_e32 v2, 422632; GFX12-NEXT: s_wait_kmcnt 0x02633; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s32634; GFX12-NEXT: global_wb scope:SCOPE_SYS2635; GFX12-NEXT: s_wait_storecnt 0x02636; GFX12-NEXT: flat_atomic_inc_u32 v2, v[0:1], v2 offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS2637; GFX12-NEXT: s_wait_loadcnt_dscnt 0x02638; GFX12-NEXT: global_inv scope:SCOPE_SYS2639; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12640; GFX12-NEXT: flat_store_b32 v[0:1], v22641; GFX12-NEXT: s_endpgm2642 %gep = getelementptr inbounds i32, ptr %ptr, i32 42643 %result = atomicrmw uinc_wrap ptr %gep, i32 42 seq_cst, align 4, !amdgpu.no.remote.memory !12644 store i32 %result, ptr %out, align 42645 ret void2646}2647 2648define amdgpu_kernel void @flat_atomic_inc_noret_i32(ptr %ptr) #1 {2649; CI-LABEL: flat_atomic_inc_noret_i32:2650; CI: ; %bb.0:2651; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02652; CI-NEXT: s_add_i32 s12, s12, s172653; CI-NEXT: s_mov_b32 flat_scratch_lo, s132654; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82655; CI-NEXT: v_mov_b32_e32 v2, 422656; CI-NEXT: s_waitcnt lgkmcnt(0)2657; CI-NEXT: v_mov_b32_e32 v0, s02658; CI-NEXT: v_mov_b32_e32 v1, s12659; CI-NEXT: flat_atomic_inc v[0:1], v22660; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2661; CI-NEXT: buffer_wbinvl1_vol2662; CI-NEXT: s_endpgm2663;2664; VI-LABEL: flat_atomic_inc_noret_i32:2665; VI: ; %bb.0:2666; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02667; VI-NEXT: s_add_i32 s12, s12, s172668; VI-NEXT: s_mov_b32 flat_scratch_lo, s132669; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82670; VI-NEXT: v_mov_b32_e32 v2, 422671; VI-NEXT: s_waitcnt lgkmcnt(0)2672; VI-NEXT: v_mov_b32_e32 v0, s02673; VI-NEXT: v_mov_b32_e32 v1, s12674; VI-NEXT: flat_atomic_inc v[0:1], v22675; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2676; VI-NEXT: buffer_wbinvl1_vol2677; VI-NEXT: s_endpgm2678;2679; GFX9-LABEL: flat_atomic_inc_noret_i32:2680; GFX9: ; %bb.0:2681; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02682; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s172683; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 02684; GFX9-NEXT: v_mov_b32_e32 v2, 422685; GFX9-NEXT: s_waitcnt lgkmcnt(0)2686; GFX9-NEXT: v_mov_b32_e32 v0, s02687; GFX9-NEXT: v_mov_b32_e32 v1, s12688; GFX9-NEXT: flat_atomic_inc v[0:1], v22689; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2690; GFX9-NEXT: buffer_wbinvl1_vol2691; GFX9-NEXT: s_endpgm2692;2693; GFX10-LABEL: flat_atomic_inc_noret_i32:2694; GFX10: ; %bb.0:2695; GFX10-NEXT: s_add_u32 s12, s12, s172696; GFX10-NEXT: s_addc_u32 s13, s13, 02697; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s122698; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s132699; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02700; GFX10-NEXT: v_mov_b32_e32 v2, 422701; GFX10-NEXT: s_waitcnt lgkmcnt(0)2702; GFX10-NEXT: v_mov_b32_e32 v0, s02703; GFX10-NEXT: v_mov_b32_e32 v1, s12704; GFX10-NEXT: flat_atomic_inc v[0:1], v22705; GFX10-NEXT: s_waitcnt lgkmcnt(0)2706; GFX10-NEXT: s_waitcnt_vscnt null, 0x02707; GFX10-NEXT: buffer_gl1_inv2708; GFX10-NEXT: buffer_gl0_inv2709; GFX10-NEXT: s_endpgm2710;2711; GFX11-LABEL: flat_atomic_inc_noret_i32:2712; GFX11: ; %bb.0:2713; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x02714; GFX11-NEXT: v_mov_b32_e32 v2, 422715; GFX11-NEXT: s_waitcnt lgkmcnt(0)2716; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12717; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v22718; GFX11-NEXT: s_waitcnt lgkmcnt(0)2719; GFX11-NEXT: s_waitcnt_vscnt null, 0x02720; GFX11-NEXT: buffer_gl1_inv2721; GFX11-NEXT: buffer_gl0_inv2722; GFX11-NEXT: s_endpgm2723;2724; GFX12-LABEL: flat_atomic_inc_noret_i32:2725; GFX12: ; %bb.0:2726; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x02727; GFX12-NEXT: v_mov_b32_e32 v2, 422728; GFX12-NEXT: s_wait_kmcnt 0x02729; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12730; GFX12-NEXT: flat_atomic_inc_u32 v[0:1], v2 scope:SCOPE_DEV2731; GFX12-NEXT: s_wait_storecnt_dscnt 0x02732; GFX12-NEXT: global_inv scope:SCOPE_DEV2733; GFX12-NEXT: s_endpgm2734 %result = atomicrmw uinc_wrap ptr %ptr, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !12735 ret void2736}2737 2738define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset(ptr %ptr) #1 {2739; CI-LABEL: flat_atomic_inc_noret_i32_offset:2740; CI: ; %bb.0:2741; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02742; CI-NEXT: s_add_i32 s12, s12, s172743; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82744; CI-NEXT: s_mov_b32 flat_scratch_lo, s132745; CI-NEXT: v_mov_b32_e32 v2, 422746; CI-NEXT: s_waitcnt lgkmcnt(0)2747; CI-NEXT: s_add_u32 s0, s0, 162748; CI-NEXT: s_addc_u32 s1, s1, 02749; CI-NEXT: v_mov_b32_e32 v0, s02750; CI-NEXT: v_mov_b32_e32 v1, s12751; CI-NEXT: flat_atomic_inc v[0:1], v22752; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2753; CI-NEXT: buffer_wbinvl1_vol2754; CI-NEXT: s_endpgm2755;2756; VI-LABEL: flat_atomic_inc_noret_i32_offset:2757; VI: ; %bb.0:2758; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02759; VI-NEXT: s_add_i32 s12, s12, s172760; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82761; VI-NEXT: s_mov_b32 flat_scratch_lo, s132762; VI-NEXT: v_mov_b32_e32 v2, 422763; VI-NEXT: s_waitcnt lgkmcnt(0)2764; VI-NEXT: s_add_u32 s0, s0, 162765; VI-NEXT: s_addc_u32 s1, s1, 02766; VI-NEXT: v_mov_b32_e32 v0, s02767; VI-NEXT: v_mov_b32_e32 v1, s12768; VI-NEXT: flat_atomic_inc v[0:1], v22769; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2770; VI-NEXT: buffer_wbinvl1_vol2771; VI-NEXT: s_endpgm2772;2773; GFX9-LABEL: flat_atomic_inc_noret_i32_offset:2774; GFX9: ; %bb.0:2775; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02776; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s172777; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 02778; GFX9-NEXT: v_mov_b32_e32 v2, 422779; GFX9-NEXT: s_waitcnt lgkmcnt(0)2780; GFX9-NEXT: v_mov_b32_e32 v0, s02781; GFX9-NEXT: v_mov_b32_e32 v1, s12782; GFX9-NEXT: flat_atomic_inc v[0:1], v2 offset:162783; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2784; GFX9-NEXT: buffer_wbinvl1_vol2785; GFX9-NEXT: s_endpgm2786;2787; GFX10-LABEL: flat_atomic_inc_noret_i32_offset:2788; GFX10: ; %bb.0:2789; GFX10-NEXT: s_add_u32 s12, s12, s172790; GFX10-NEXT: s_addc_u32 s13, s13, 02791; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s122792; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s132793; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02794; GFX10-NEXT: v_mov_b32_e32 v2, 422795; GFX10-NEXT: s_waitcnt lgkmcnt(0)2796; GFX10-NEXT: s_add_u32 s0, s0, 162797; GFX10-NEXT: s_addc_u32 s1, s1, 02798; GFX10-NEXT: v_mov_b32_e32 v0, s02799; GFX10-NEXT: v_mov_b32_e32 v1, s12800; GFX10-NEXT: flat_atomic_inc v[0:1], v22801; GFX10-NEXT: s_waitcnt lgkmcnt(0)2802; GFX10-NEXT: s_waitcnt_vscnt null, 0x02803; GFX10-NEXT: buffer_gl1_inv2804; GFX10-NEXT: buffer_gl0_inv2805; GFX10-NEXT: s_endpgm2806;2807; GFX11-LABEL: flat_atomic_inc_noret_i32_offset:2808; GFX11: ; %bb.0:2809; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x02810; GFX11-NEXT: v_mov_b32_e32 v2, 422811; GFX11-NEXT: s_waitcnt lgkmcnt(0)2812; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12813; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v2 offset:162814; GFX11-NEXT: s_waitcnt lgkmcnt(0)2815; GFX11-NEXT: s_waitcnt_vscnt null, 0x02816; GFX11-NEXT: buffer_gl1_inv2817; GFX11-NEXT: buffer_gl0_inv2818; GFX11-NEXT: s_endpgm2819;2820; GFX12-LABEL: flat_atomic_inc_noret_i32_offset:2821; GFX12: ; %bb.0:2822; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x02823; GFX12-NEXT: v_mov_b32_e32 v2, 422824; GFX12-NEXT: s_wait_kmcnt 0x02825; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12826; GFX12-NEXT: flat_atomic_inc_u32 v[0:1], v2 offset:16 scope:SCOPE_DEV2827; GFX12-NEXT: s_wait_storecnt_dscnt 0x02828; GFX12-NEXT: global_inv scope:SCOPE_DEV2829; GFX12-NEXT: s_endpgm2830 %gep = getelementptr inbounds i32, ptr %ptr, i32 42831 %result = atomicrmw uinc_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !12832 ret void2833}2834 2835define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_system(ptr %ptr) #1 {2836; CI-LABEL: flat_atomic_inc_noret_i32_offset_system:2837; CI: ; %bb.0:2838; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02839; CI-NEXT: s_add_i32 s12, s12, s172840; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82841; CI-NEXT: s_mov_b32 flat_scratch_lo, s132842; CI-NEXT: v_mov_b32_e32 v2, 422843; CI-NEXT: s_waitcnt lgkmcnt(0)2844; CI-NEXT: s_add_u32 s0, s0, 162845; CI-NEXT: s_addc_u32 s1, s1, 02846; CI-NEXT: v_mov_b32_e32 v0, s02847; CI-NEXT: v_mov_b32_e32 v1, s12848; CI-NEXT: flat_atomic_inc v[0:1], v22849; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2850; CI-NEXT: buffer_wbinvl1_vol2851; CI-NEXT: s_endpgm2852;2853; VI-LABEL: flat_atomic_inc_noret_i32_offset_system:2854; VI: ; %bb.0:2855; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02856; VI-NEXT: s_add_i32 s12, s12, s172857; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82858; VI-NEXT: s_mov_b32 flat_scratch_lo, s132859; VI-NEXT: v_mov_b32_e32 v2, 422860; VI-NEXT: s_waitcnt lgkmcnt(0)2861; VI-NEXT: s_add_u32 s0, s0, 162862; VI-NEXT: s_addc_u32 s1, s1, 02863; VI-NEXT: v_mov_b32_e32 v0, s02864; VI-NEXT: v_mov_b32_e32 v1, s12865; VI-NEXT: flat_atomic_inc v[0:1], v22866; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2867; VI-NEXT: buffer_wbinvl1_vol2868; VI-NEXT: s_endpgm2869;2870; GFX9-LABEL: flat_atomic_inc_noret_i32_offset_system:2871; GFX9: ; %bb.0:2872; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02873; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s172874; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 02875; GFX9-NEXT: v_mov_b32_e32 v2, 422876; GFX9-NEXT: s_waitcnt lgkmcnt(0)2877; GFX9-NEXT: v_mov_b32_e32 v0, s02878; GFX9-NEXT: v_mov_b32_e32 v1, s12879; GFX9-NEXT: flat_atomic_inc v[0:1], v2 offset:162880; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2881; GFX9-NEXT: buffer_wbinvl1_vol2882; GFX9-NEXT: s_endpgm2883;2884; GFX10-LABEL: flat_atomic_inc_noret_i32_offset_system:2885; GFX10: ; %bb.0:2886; GFX10-NEXT: s_add_u32 s12, s12, s172887; GFX10-NEXT: s_addc_u32 s13, s13, 02888; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s122889; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s132890; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x02891; GFX10-NEXT: v_mov_b32_e32 v2, 422892; GFX10-NEXT: s_waitcnt lgkmcnt(0)2893; GFX10-NEXT: s_add_u32 s0, s0, 162894; GFX10-NEXT: s_addc_u32 s1, s1, 02895; GFX10-NEXT: v_mov_b32_e32 v0, s02896; GFX10-NEXT: v_mov_b32_e32 v1, s12897; GFX10-NEXT: flat_atomic_inc v[0:1], v22898; GFX10-NEXT: s_waitcnt lgkmcnt(0)2899; GFX10-NEXT: s_waitcnt_vscnt null, 0x02900; GFX10-NEXT: buffer_gl1_inv2901; GFX10-NEXT: buffer_gl0_inv2902; GFX10-NEXT: s_endpgm2903;2904; GFX11-LABEL: flat_atomic_inc_noret_i32_offset_system:2905; GFX11: ; %bb.0:2906; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x02907; GFX11-NEXT: v_mov_b32_e32 v2, 422908; GFX11-NEXT: s_waitcnt lgkmcnt(0)2909; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12910; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v2 offset:162911; GFX11-NEXT: s_waitcnt lgkmcnt(0)2912; GFX11-NEXT: s_waitcnt_vscnt null, 0x02913; GFX11-NEXT: buffer_gl1_inv2914; GFX11-NEXT: buffer_gl0_inv2915; GFX11-NEXT: s_endpgm2916;2917; GFX12-LABEL: flat_atomic_inc_noret_i32_offset_system:2918; GFX12: ; %bb.0:2919; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x02920; GFX12-NEXT: v_mov_b32_e32 v2, 422921; GFX12-NEXT: s_wait_kmcnt 0x02922; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12923; GFX12-NEXT: global_wb scope:SCOPE_SYS2924; GFX12-NEXT: s_wait_storecnt 0x02925; GFX12-NEXT: flat_atomic_inc_u32 v[0:1], v2 offset:16 scope:SCOPE_SYS2926; GFX12-NEXT: s_wait_storecnt_dscnt 0x02927; GFX12-NEXT: global_inv scope:SCOPE_SYS2928; GFX12-NEXT: s_endpgm2929 %gep = getelementptr inbounds i32, ptr %ptr, i32 42930 %result = atomicrmw uinc_wrap ptr %gep, i32 42 seq_cst, align 4, !amdgpu.no.remote.memory !12931 ret void2932}2933 2934define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_addr64(ptr %out, ptr %ptr) #1 {2935; CI-LABEL: flat_atomic_inc_ret_i32_offset_addr64:2936; CI: ; %bb.0:2937; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02938; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v02939; CI-NEXT: s_add_i32 s12, s12, s172940; CI-NEXT: s_mov_b32 flat_scratch_lo, s132941; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82942; CI-NEXT: s_waitcnt lgkmcnt(0)2943; CI-NEXT: v_mov_b32_e32 v0, s22944; CI-NEXT: v_mov_b32_e32 v1, s32945; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v22946; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2947; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v02948; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2949; CI-NEXT: v_mov_b32_e32 v3, 422950; CI-NEXT: flat_atomic_inc v3, v[0:1], v3 glc2951; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2952; CI-NEXT: buffer_wbinvl1_vol2953; CI-NEXT: v_mov_b32_e32 v0, s02954; CI-NEXT: v_mov_b32_e32 v1, s12955; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v22956; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2957; CI-NEXT: flat_store_dword v[0:1], v32958; CI-NEXT: s_endpgm2959;2960; VI-LABEL: flat_atomic_inc_ret_i32_offset_addr64:2961; VI: ; %bb.0:2962; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02963; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v02964; VI-NEXT: s_add_i32 s12, s12, s172965; VI-NEXT: s_mov_b32 flat_scratch_lo, s132966; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 82967; VI-NEXT: s_waitcnt lgkmcnt(0)2968; VI-NEXT: v_mov_b32_e32 v0, s22969; VI-NEXT: v_mov_b32_e32 v1, s32970; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v22971; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2972; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v02973; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2974; VI-NEXT: v_mov_b32_e32 v3, 422975; VI-NEXT: flat_atomic_inc v3, v[0:1], v3 glc2976; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2977; VI-NEXT: buffer_wbinvl1_vol2978; VI-NEXT: v_mov_b32_e32 v0, s02979; VI-NEXT: v_mov_b32_e32 v1, s12980; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v22981; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2982; VI-NEXT: flat_store_dword v[0:1], v32983; VI-NEXT: s_endpgm2984;2985; GFX9-LABEL: flat_atomic_inc_ret_i32_offset_addr64:2986; GFX9: ; %bb.0:2987; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x02988; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v02989; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s172990; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 02991; GFX9-NEXT: v_mov_b32_e32 v3, 422992; GFX9-NEXT: s_waitcnt lgkmcnt(0)2993; GFX9-NEXT: v_mov_b32_e32 v0, s22994; GFX9-NEXT: v_mov_b32_e32 v1, s32995; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v22996; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc2997; GFX9-NEXT: flat_atomic_inc v3, v[0:1], v3 offset:20 glc2998; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2999; GFX9-NEXT: buffer_wbinvl1_vol3000; GFX9-NEXT: v_mov_b32_e32 v0, s03001; GFX9-NEXT: v_mov_b32_e32 v1, s13002; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v23003; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc3004; GFX9-NEXT: flat_store_dword v[0:1], v33005; GFX9-NEXT: s_endpgm3006;3007; GFX10-LABEL: flat_atomic_inc_ret_i32_offset_addr64:3008; GFX10: ; %bb.0:3009; GFX10-NEXT: s_add_u32 s12, s12, s173010; GFX10-NEXT: s_addc_u32 s13, s13, 03011; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s123012; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s133013; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03014; GFX10-NEXT: v_lshlrev_b32_e32 v2, 2, v03015; GFX10-NEXT: v_mov_b32_e32 v3, 423016; GFX10-NEXT: s_waitcnt lgkmcnt(0)3017; GFX10-NEXT: v_mov_b32_e32 v0, s23018; GFX10-NEXT: v_mov_b32_e32 v1, s33019; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v23020; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo3021; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, 203022; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo3023; GFX10-NEXT: flat_atomic_inc v3, v[0:1], v3 glc3024; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3025; GFX10-NEXT: buffer_gl1_inv3026; GFX10-NEXT: buffer_gl0_inv3027; GFX10-NEXT: v_mov_b32_e32 v0, s03028; GFX10-NEXT: v_mov_b32_e32 v1, s13029; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v23030; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo3031; GFX10-NEXT: flat_store_dword v[0:1], v33032; GFX10-NEXT: s_endpgm3033;3034; GFX11-LABEL: flat_atomic_inc_ret_i32_offset_addr64:3035; GFX11: ; %bb.0:3036; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x03037; GFX11-NEXT: v_dual_mov_b32 v3, 42 :: v_dual_and_b32 v0, 0x3ff, v03038; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)3039; GFX11-NEXT: v_lshlrev_b32_e32 v2, 2, v03040; GFX11-NEXT: s_waitcnt lgkmcnt(0)3041; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s33042; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v23043; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)3044; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo3045; GFX11-NEXT: flat_atomic_inc_u32 v3, v[0:1], v3 offset:20 glc3046; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3047; GFX11-NEXT: buffer_gl1_inv3048; GFX11-NEXT: buffer_gl0_inv3049; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13050; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3051; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v23052; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo3053; GFX11-NEXT: flat_store_b32 v[0:1], v33054; GFX11-NEXT: s_endpgm3055;3056; GFX12-LABEL: flat_atomic_inc_ret_i32_offset_addr64:3057; GFX12: ; %bb.0:3058; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x03059; GFX12-NEXT: v_dual_mov_b32 v3, 42 :: v_dual_and_b32 v0, 0x3ff, v03060; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)3061; GFX12-NEXT: v_lshlrev_b32_e32 v2, 2, v03062; GFX12-NEXT: s_wait_kmcnt 0x03063; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s33064; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v23065; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)3066; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo3067; GFX12-NEXT: flat_atomic_inc_u32 v3, v[0:1], v3 offset:20 th:TH_ATOMIC_RETURN scope:SCOPE_DEV3068; GFX12-NEXT: s_wait_loadcnt_dscnt 0x03069; GFX12-NEXT: global_inv scope:SCOPE_DEV3070; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13071; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v23072; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)3073; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)3074; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo3075; GFX12-NEXT: flat_store_b32 v[0:1], v33076; GFX12-NEXT: s_endpgm3077 %id = call i32 @llvm.amdgcn.workitem.id.x()3078 %gep.tid = getelementptr i32, ptr %ptr, i32 %id3079 %out.gep = getelementptr i32, ptr %out, i32 %id3080 %gep = getelementptr inbounds i32, ptr %gep.tid, i32 53081 %result = atomicrmw uinc_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !13082 store i32 %result, ptr %out.gep, align 43083 ret void3084}3085 3086define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_addr64(ptr %ptr) #1 {3087; CI-LABEL: flat_atomic_inc_noret_i32_offset_addr64:3088; CI: ; %bb.0:3089; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03090; CI-NEXT: v_lshlrev_b32_e32 v2, 2, v03091; CI-NEXT: s_add_i32 s12, s12, s173092; CI-NEXT: s_mov_b32 flat_scratch_lo, s133093; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83094; CI-NEXT: s_waitcnt lgkmcnt(0)3095; CI-NEXT: v_mov_b32_e32 v0, s03096; CI-NEXT: v_mov_b32_e32 v1, s13097; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v23098; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3099; CI-NEXT: v_add_i32_e32 v0, vcc, 20, v03100; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3101; CI-NEXT: v_mov_b32_e32 v2, 423102; CI-NEXT: flat_atomic_inc v[0:1], v23103; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3104; CI-NEXT: buffer_wbinvl1_vol3105; CI-NEXT: s_endpgm3106;3107; VI-LABEL: flat_atomic_inc_noret_i32_offset_addr64:3108; VI: ; %bb.0:3109; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03110; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v03111; VI-NEXT: s_add_i32 s12, s12, s173112; VI-NEXT: s_mov_b32 flat_scratch_lo, s133113; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83114; VI-NEXT: s_waitcnt lgkmcnt(0)3115; VI-NEXT: v_mov_b32_e32 v0, s03116; VI-NEXT: v_mov_b32_e32 v1, s13117; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v23118; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3119; VI-NEXT: v_add_u32_e32 v0, vcc, 20, v03120; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3121; VI-NEXT: v_mov_b32_e32 v2, 423122; VI-NEXT: flat_atomic_inc v[0:1], v23123; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3124; VI-NEXT: buffer_wbinvl1_vol3125; VI-NEXT: s_endpgm3126;3127; GFX9-LABEL: flat_atomic_inc_noret_i32_offset_addr64:3128; GFX9: ; %bb.0:3129; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03130; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v03131; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s173132; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 03133; GFX9-NEXT: s_waitcnt lgkmcnt(0)3134; GFX9-NEXT: v_mov_b32_e32 v0, s03135; GFX9-NEXT: v_mov_b32_e32 v1, s13136; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v23137; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc3138; GFX9-NEXT: v_mov_b32_e32 v2, 423139; GFX9-NEXT: flat_atomic_inc v[0:1], v2 offset:203140; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3141; GFX9-NEXT: buffer_wbinvl1_vol3142; GFX9-NEXT: s_endpgm3143;3144; GFX10-LABEL: flat_atomic_inc_noret_i32_offset_addr64:3145; GFX10: ; %bb.0:3146; GFX10-NEXT: s_add_u32 s12, s12, s173147; GFX10-NEXT: s_addc_u32 s13, s13, 03148; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s123149; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s133150; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03151; GFX10-NEXT: v_lshlrev_b32_e32 v2, 2, v03152; GFX10-NEXT: s_waitcnt lgkmcnt(0)3153; GFX10-NEXT: v_mov_b32_e32 v0, s03154; GFX10-NEXT: v_mov_b32_e32 v1, s13155; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v23156; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo3157; GFX10-NEXT: v_mov_b32_e32 v2, 423158; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, 203159; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo3160; GFX10-NEXT: flat_atomic_inc v[0:1], v23161; GFX10-NEXT: s_waitcnt lgkmcnt(0)3162; GFX10-NEXT: s_waitcnt_vscnt null, 0x03163; GFX10-NEXT: buffer_gl1_inv3164; GFX10-NEXT: buffer_gl0_inv3165; GFX10-NEXT: s_endpgm3166;3167; GFX11-LABEL: flat_atomic_inc_noret_i32_offset_addr64:3168; GFX11: ; %bb.0:3169; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x03170; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v03171; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)3172; GFX11-NEXT: v_lshlrev_b32_e32 v2, 2, v03173; GFX11-NEXT: s_waitcnt lgkmcnt(0)3174; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13175; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v23176; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)3177; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo3178; GFX11-NEXT: v_mov_b32_e32 v2, 423179; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v2 offset:203180; GFX11-NEXT: s_waitcnt lgkmcnt(0)3181; GFX11-NEXT: s_waitcnt_vscnt null, 0x03182; GFX11-NEXT: buffer_gl1_inv3183; GFX11-NEXT: buffer_gl0_inv3184; GFX11-NEXT: s_endpgm3185;3186; GFX12-LABEL: flat_atomic_inc_noret_i32_offset_addr64:3187; GFX12: ; %bb.0:3188; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x03189; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v03190; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)3191; GFX12-NEXT: v_lshlrev_b32_e32 v2, 2, v03192; GFX12-NEXT: s_wait_kmcnt 0x03193; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13194; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v23195; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)3196; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo3197; GFX12-NEXT: v_mov_b32_e32 v2, 423198; GFX12-NEXT: flat_atomic_inc_u32 v[0:1], v2 offset:20 scope:SCOPE_DEV3199; GFX12-NEXT: s_wait_storecnt_dscnt 0x03200; GFX12-NEXT: global_inv scope:SCOPE_DEV3201; GFX12-NEXT: s_endpgm3202 %id = call i32 @llvm.amdgcn.workitem.id.x()3203 %gep.tid = getelementptr i32, ptr %ptr, i32 %id3204 %gep = getelementptr inbounds i32, ptr %gep.tid, i32 53205 %result = atomicrmw uinc_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !13206 ret void3207}3208 3209define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i64(ptr addrspace(1) %out, ptr addrspace(1) %add_use) #1 {3210; CI-LABEL: atomic_inc_shl_base_lds_0_i64:3211; CI: ; %bb.0:3212; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03213; CI-NEXT: v_mov_b32_e32 v1, 93214; CI-NEXT: v_lshlrev_b32_e32 v3, 3, v03215; CI-NEXT: v_mov_b32_e32 v2, 03216; CI-NEXT: s_mov_b32 m0, -13217; CI-NEXT: s_waitcnt lgkmcnt(0)3218; CI-NEXT: ds_inc_rtn_u64 v[1:2], v3, v[1:2] offset:163219; CI-NEXT: s_waitcnt lgkmcnt(0)3220; CI-NEXT: s_add_i32 s12, s12, s173221; CI-NEXT: v_mov_b32_e32 v4, s33222; CI-NEXT: s_mov_b32 flat_scratch_lo, s133223; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83224; CI-NEXT: v_add_i32_e32 v0, vcc, 2, v03225; CI-NEXT: v_mov_b32_e32 v3, s23226; CI-NEXT: flat_store_dword v[3:4], v03227; CI-NEXT: v_mov_b32_e32 v4, s13228; CI-NEXT: v_mov_b32_e32 v3, s03229; CI-NEXT: flat_store_dwordx2 v[3:4], v[1:2]3230; CI-NEXT: s_endpgm3231;3232; VI-LABEL: atomic_inc_shl_base_lds_0_i64:3233; VI: ; %bb.0:3234; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03235; VI-NEXT: v_mov_b32_e32 v1, 93236; VI-NEXT: v_lshlrev_b32_e32 v3, 3, v03237; VI-NEXT: v_mov_b32_e32 v2, 03238; VI-NEXT: s_mov_b32 m0, -13239; VI-NEXT: s_waitcnt lgkmcnt(0)3240; VI-NEXT: ds_inc_rtn_u64 v[1:2], v3, v[1:2] offset:163241; VI-NEXT: s_waitcnt lgkmcnt(0)3242; VI-NEXT: s_add_i32 s12, s12, s173243; VI-NEXT: v_mov_b32_e32 v4, s33244; VI-NEXT: s_mov_b32 flat_scratch_lo, s133245; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83246; VI-NEXT: v_add_u32_e32 v0, vcc, 2, v03247; VI-NEXT: v_mov_b32_e32 v3, s23248; VI-NEXT: flat_store_dword v[3:4], v03249; VI-NEXT: v_mov_b32_e32 v4, s13250; VI-NEXT: v_mov_b32_e32 v3, s03251; VI-NEXT: flat_store_dwordx2 v[3:4], v[1:2]3252; VI-NEXT: s_endpgm3253;3254; GFX9-LABEL: atomic_inc_shl_base_lds_0_i64:3255; GFX9: ; %bb.0:3256; GFX9-NEXT: v_mov_b32_e32 v1, 93257; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03258; GFX9-NEXT: v_mov_b32_e32 v2, 03259; GFX9-NEXT: v_lshlrev_b32_e32 v3, 3, v03260; GFX9-NEXT: s_waitcnt lgkmcnt(0)3261; GFX9-NEXT: ds_inc_rtn_u64 v[1:2], v3, v[1:2] offset:163262; GFX9-NEXT: s_waitcnt lgkmcnt(0)3263; GFX9-NEXT: v_add_u32_e32 v0, 2, v03264; GFX9-NEXT: v_mov_b32_e32 v3, 03265; GFX9-NEXT: global_store_dword v3, v0, s[2:3]3266; GFX9-NEXT: global_store_dwordx2 v3, v[1:2], s[0:1]3267; GFX9-NEXT: s_endpgm3268;3269; GFX10-LABEL: atomic_inc_shl_base_lds_0_i64:3270; GFX10: ; %bb.0:3271; GFX10-NEXT: v_mov_b32_e32 v1, 93272; GFX10-NEXT: v_mov_b32_e32 v2, 03273; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v03274; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03275; GFX10-NEXT: v_add_nc_u32_e32 v0, 2, v03276; GFX10-NEXT: s_waitcnt lgkmcnt(0)3277; GFX10-NEXT: ds_inc_rtn_u64 v[1:2], v3, v[1:2] offset:163278; GFX10-NEXT: s_waitcnt lgkmcnt(0)3279; GFX10-NEXT: buffer_gl0_inv3280; GFX10-NEXT: v_mov_b32_e32 v3, 03281; GFX10-NEXT: global_store_dword v3, v0, s[2:3]3282; GFX10-NEXT: global_store_dwordx2 v3, v[1:2], s[0:1]3283; GFX10-NEXT: s_endpgm3284;3285; GFX11-LABEL: atomic_inc_shl_base_lds_0_i64:3286; GFX11: ; %bb.0:3287; GFX11-NEXT: v_and_b32_e32 v2, 0x3ff, v03288; GFX11-NEXT: v_mov_b32_e32 v0, 93289; GFX11-NEXT: v_mov_b32_e32 v1, 03290; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x03291; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)3292; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v23293; GFX11-NEXT: v_add_nc_u32_e32 v2, 2, v23294; GFX11-NEXT: s_waitcnt lgkmcnt(0)3295; GFX11-NEXT: ds_inc_rtn_u64 v[0:1], v3, v[0:1] offset:163296; GFX11-NEXT: s_waitcnt lgkmcnt(0)3297; GFX11-NEXT: buffer_gl0_inv3298; GFX11-NEXT: v_mov_b32_e32 v3, 03299; GFX11-NEXT: s_clause 0x13300; GFX11-NEXT: global_store_b32 v3, v2, s[2:3]3301; GFX11-NEXT: global_store_b64 v3, v[0:1], s[0:1]3302; GFX11-NEXT: s_endpgm3303;3304; GFX12-LABEL: atomic_inc_shl_base_lds_0_i64:3305; GFX12: ; %bb.0:3306; GFX12-NEXT: v_and_b32_e32 v2, 0x3ff, v03307; GFX12-NEXT: v_mov_b32_e32 v0, 93308; GFX12-NEXT: v_mov_b32_e32 v1, 03309; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x03310; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)3311; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v23312; GFX12-NEXT: v_add_nc_u32_e32 v2, 2, v23313; GFX12-NEXT: ds_inc_rtn_u64 v[0:1], v3, v[0:1] offset:163314; GFX12-NEXT: s_wait_dscnt 0x03315; GFX12-NEXT: global_inv scope:SCOPE_SE3316; GFX12-NEXT: v_mov_b32_e32 v3, 03317; GFX12-NEXT: s_wait_kmcnt 0x03318; GFX12-NEXT: s_clause 0x13319; GFX12-NEXT: global_store_b32 v3, v2, s[2:3]3320; GFX12-NEXT: global_store_b64 v3, v[0:1], s[0:1]3321; GFX12-NEXT: s_endpgm3322 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #23323 %idx.0 = add nsw i32 %tid.x, 23324 %arrayidx0 = getelementptr inbounds [512 x i64], ptr addrspace(3) @lds1, i32 0, i32 %idx.03325 %result = atomicrmw uinc_wrap ptr addrspace(3) %arrayidx0, i64 9 syncscope("agent") seq_cst, align 83326 store i32 %idx.0, ptr addrspace(1) %add_use, align 43327 store i64 %result, ptr addrspace(1) %out, align 43328 ret void3329}3330 3331define amdgpu_kernel void @flat_atomic_inc_ret_i64(ptr %out, ptr %ptr) #1 {3332; CI-LABEL: flat_atomic_inc_ret_i64:3333; CI: ; %bb.0:3334; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03335; CI-NEXT: s_add_i32 s12, s12, s173336; CI-NEXT: v_mov_b32_e32 v0, 423337; CI-NEXT: s_mov_b32 flat_scratch_lo, s133338; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83339; CI-NEXT: s_waitcnt lgkmcnt(0)3340; CI-NEXT: v_mov_b32_e32 v2, s23341; CI-NEXT: v_mov_b32_e32 v1, 03342; CI-NEXT: v_mov_b32_e32 v3, s33343; CI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc3344; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3345; CI-NEXT: buffer_wbinvl1_vol3346; CI-NEXT: v_mov_b32_e32 v3, s13347; CI-NEXT: v_mov_b32_e32 v2, s03348; CI-NEXT: s_add_u32 s0, s0, 43349; CI-NEXT: s_addc_u32 s1, s1, 03350; CI-NEXT: v_mov_b32_e32 v5, s13351; CI-NEXT: v_mov_b32_e32 v4, s03352; CI-NEXT: flat_store_dword v[2:3], v03353; CI-NEXT: flat_store_dword v[4:5], v13354; CI-NEXT: s_endpgm3355;3356; VI-LABEL: flat_atomic_inc_ret_i64:3357; VI: ; %bb.0:3358; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03359; VI-NEXT: s_add_i32 s12, s12, s173360; VI-NEXT: v_mov_b32_e32 v0, 423361; VI-NEXT: s_mov_b32 flat_scratch_lo, s133362; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83363; VI-NEXT: s_waitcnt lgkmcnt(0)3364; VI-NEXT: v_mov_b32_e32 v2, s23365; VI-NEXT: v_mov_b32_e32 v1, 03366; VI-NEXT: v_mov_b32_e32 v3, s33367; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc3368; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3369; VI-NEXT: buffer_wbinvl1_vol3370; VI-NEXT: v_mov_b32_e32 v3, s13371; VI-NEXT: v_mov_b32_e32 v2, s03372; VI-NEXT: s_add_u32 s0, s0, 43373; VI-NEXT: s_addc_u32 s1, s1, 03374; VI-NEXT: v_mov_b32_e32 v5, s13375; VI-NEXT: v_mov_b32_e32 v4, s03376; VI-NEXT: flat_store_dword v[2:3], v03377; VI-NEXT: flat_store_dword v[4:5], v13378; VI-NEXT: s_endpgm3379;3380; GFX9-LABEL: flat_atomic_inc_ret_i64:3381; GFX9: ; %bb.0:3382; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03383; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s173384; GFX9-NEXT: v_mov_b32_e32 v0, 423385; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 03386; GFX9-NEXT: v_mov_b32_e32 v1, 03387; GFX9-NEXT: s_waitcnt lgkmcnt(0)3388; GFX9-NEXT: v_mov_b32_e32 v2, s23389; GFX9-NEXT: v_mov_b32_e32 v3, s33390; GFX9-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc3391; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3392; GFX9-NEXT: buffer_wbinvl1_vol3393; GFX9-NEXT: v_mov_b32_e32 v3, s13394; GFX9-NEXT: v_mov_b32_e32 v2, s03395; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]3396; GFX9-NEXT: s_endpgm3397;3398; GFX10-LABEL: flat_atomic_inc_ret_i64:3399; GFX10: ; %bb.0:3400; GFX10-NEXT: s_add_u32 s12, s12, s173401; GFX10-NEXT: s_addc_u32 s13, s13, 03402; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s123403; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s133404; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03405; GFX10-NEXT: v_mov_b32_e32 v0, 423406; GFX10-NEXT: v_mov_b32_e32 v1, 03407; GFX10-NEXT: s_waitcnt lgkmcnt(0)3408; GFX10-NEXT: v_mov_b32_e32 v2, s23409; GFX10-NEXT: v_mov_b32_e32 v3, s33410; GFX10-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc3411; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3412; GFX10-NEXT: buffer_gl1_inv3413; GFX10-NEXT: buffer_gl0_inv3414; GFX10-NEXT: v_mov_b32_e32 v3, s13415; GFX10-NEXT: v_mov_b32_e32 v2, s03416; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]3417; GFX10-NEXT: s_endpgm3418;3419; GFX11-LABEL: flat_atomic_inc_ret_i64:3420; GFX11: ; %bb.0:3421; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x03422; GFX11-NEXT: v_mov_b32_e32 v0, 423423; GFX11-NEXT: s_waitcnt lgkmcnt(0)3424; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s23425; GFX11-NEXT: v_mov_b32_e32 v3, s33426; GFX11-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3], v[0:1] glc3427; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3428; GFX11-NEXT: buffer_gl1_inv3429; GFX11-NEXT: buffer_gl0_inv3430; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s03431; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]3432; GFX11-NEXT: s_endpgm3433;3434; GFX12-LABEL: flat_atomic_inc_ret_i64:3435; GFX12: ; %bb.0:3436; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x03437; GFX12-NEXT: v_mov_b32_e32 v0, 423438; GFX12-NEXT: s_wait_kmcnt 0x03439; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s23440; GFX12-NEXT: v_mov_b32_e32 v3, s33441; GFX12-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3], v[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV3442; GFX12-NEXT: s_wait_loadcnt_dscnt 0x03443; GFX12-NEXT: global_inv scope:SCOPE_DEV3444; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s03445; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]3446; GFX12-NEXT: s_endpgm3447 %result = atomicrmw uinc_wrap ptr %ptr, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !13448 store i64 %result, ptr %out, align 43449 ret void3450}3451 3452define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset(ptr %out, ptr %ptr) #1 {3453; CI-LABEL: flat_atomic_inc_ret_i64_offset:3454; CI: ; %bb.0:3455; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03456; CI-NEXT: s_add_i32 s12, s12, s173457; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83458; CI-NEXT: v_mov_b32_e32 v0, 423459; CI-NEXT: s_mov_b32 flat_scratch_lo, s133460; CI-NEXT: s_waitcnt lgkmcnt(0)3461; CI-NEXT: s_add_u32 s2, s2, 323462; CI-NEXT: s_addc_u32 s3, s3, 03463; CI-NEXT: v_mov_b32_e32 v2, s23464; CI-NEXT: v_mov_b32_e32 v1, 03465; CI-NEXT: v_mov_b32_e32 v3, s33466; CI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc3467; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3468; CI-NEXT: buffer_wbinvl1_vol3469; CI-NEXT: v_mov_b32_e32 v3, s13470; CI-NEXT: v_mov_b32_e32 v2, s03471; CI-NEXT: s_add_u32 s0, s0, 43472; CI-NEXT: s_addc_u32 s1, s1, 03473; CI-NEXT: v_mov_b32_e32 v5, s13474; CI-NEXT: v_mov_b32_e32 v4, s03475; CI-NEXT: flat_store_dword v[2:3], v03476; CI-NEXT: flat_store_dword v[4:5], v13477; CI-NEXT: s_endpgm3478;3479; VI-LABEL: flat_atomic_inc_ret_i64_offset:3480; VI: ; %bb.0:3481; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03482; VI-NEXT: s_add_i32 s12, s12, s173483; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83484; VI-NEXT: v_mov_b32_e32 v0, 423485; VI-NEXT: s_mov_b32 flat_scratch_lo, s133486; VI-NEXT: s_waitcnt lgkmcnt(0)3487; VI-NEXT: s_add_u32 s2, s2, 323488; VI-NEXT: s_addc_u32 s3, s3, 03489; VI-NEXT: v_mov_b32_e32 v2, s23490; VI-NEXT: v_mov_b32_e32 v1, 03491; VI-NEXT: v_mov_b32_e32 v3, s33492; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc3493; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3494; VI-NEXT: buffer_wbinvl1_vol3495; VI-NEXT: v_mov_b32_e32 v3, s13496; VI-NEXT: v_mov_b32_e32 v2, s03497; VI-NEXT: s_add_u32 s0, s0, 43498; VI-NEXT: s_addc_u32 s1, s1, 03499; VI-NEXT: v_mov_b32_e32 v5, s13500; VI-NEXT: v_mov_b32_e32 v4, s03501; VI-NEXT: flat_store_dword v[2:3], v03502; VI-NEXT: flat_store_dword v[4:5], v13503; VI-NEXT: s_endpgm3504;3505; GFX9-LABEL: flat_atomic_inc_ret_i64_offset:3506; GFX9: ; %bb.0:3507; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03508; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s173509; GFX9-NEXT: v_mov_b32_e32 v0, 423510; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 03511; GFX9-NEXT: v_mov_b32_e32 v1, 03512; GFX9-NEXT: s_waitcnt lgkmcnt(0)3513; GFX9-NEXT: v_mov_b32_e32 v2, s23514; GFX9-NEXT: v_mov_b32_e32 v3, s33515; GFX9-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] offset:32 glc3516; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3517; GFX9-NEXT: buffer_wbinvl1_vol3518; GFX9-NEXT: v_mov_b32_e32 v3, s13519; GFX9-NEXT: v_mov_b32_e32 v2, s03520; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]3521; GFX9-NEXT: s_endpgm3522;3523; GFX10-LABEL: flat_atomic_inc_ret_i64_offset:3524; GFX10: ; %bb.0:3525; GFX10-NEXT: s_add_u32 s12, s12, s173526; GFX10-NEXT: s_addc_u32 s13, s13, 03527; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s123528; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s133529; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03530; GFX10-NEXT: v_mov_b32_e32 v0, 423531; GFX10-NEXT: v_mov_b32_e32 v1, 03532; GFX10-NEXT: s_waitcnt lgkmcnt(0)3533; GFX10-NEXT: s_add_u32 s2, s2, 323534; GFX10-NEXT: s_addc_u32 s3, s3, 03535; GFX10-NEXT: v_mov_b32_e32 v2, s23536; GFX10-NEXT: v_mov_b32_e32 v3, s33537; GFX10-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc3538; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3539; GFX10-NEXT: buffer_gl1_inv3540; GFX10-NEXT: buffer_gl0_inv3541; GFX10-NEXT: v_mov_b32_e32 v3, s13542; GFX10-NEXT: v_mov_b32_e32 v2, s03543; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]3544; GFX10-NEXT: s_endpgm3545;3546; GFX11-LABEL: flat_atomic_inc_ret_i64_offset:3547; GFX11: ; %bb.0:3548; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x03549; GFX11-NEXT: v_mov_b32_e32 v0, 423550; GFX11-NEXT: s_waitcnt lgkmcnt(0)3551; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s23552; GFX11-NEXT: v_mov_b32_e32 v3, s33553; GFX11-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3], v[0:1] offset:32 glc3554; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3555; GFX11-NEXT: buffer_gl1_inv3556; GFX11-NEXT: buffer_gl0_inv3557; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s03558; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]3559; GFX11-NEXT: s_endpgm3560;3561; GFX12-LABEL: flat_atomic_inc_ret_i64_offset:3562; GFX12: ; %bb.0:3563; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x03564; GFX12-NEXT: v_mov_b32_e32 v0, 423565; GFX12-NEXT: s_wait_kmcnt 0x03566; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s23567; GFX12-NEXT: v_mov_b32_e32 v3, s33568; GFX12-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3], v[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV3569; GFX12-NEXT: s_wait_loadcnt_dscnt 0x03570; GFX12-NEXT: global_inv scope:SCOPE_DEV3571; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s03572; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]3573; GFX12-NEXT: s_endpgm3574 %gep = getelementptr inbounds i64, ptr %ptr, i32 43575 %result = atomicrmw uinc_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !13576 store i64 %result, ptr %out, align 43577 ret void3578}3579 3580define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_system(ptr %out, ptr %ptr) #1 {3581; CI-LABEL: flat_atomic_inc_ret_i64_offset_system:3582; CI: ; %bb.0:3583; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03584; CI-NEXT: s_add_i32 s12, s12, s173585; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83586; CI-NEXT: v_mov_b32_e32 v0, 423587; CI-NEXT: s_mov_b32 flat_scratch_lo, s133588; CI-NEXT: s_waitcnt lgkmcnt(0)3589; CI-NEXT: s_add_u32 s2, s2, 323590; CI-NEXT: s_addc_u32 s3, s3, 03591; CI-NEXT: v_mov_b32_e32 v2, s23592; CI-NEXT: v_mov_b32_e32 v1, 03593; CI-NEXT: v_mov_b32_e32 v3, s33594; CI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc3595; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3596; CI-NEXT: buffer_wbinvl1_vol3597; CI-NEXT: v_mov_b32_e32 v3, s13598; CI-NEXT: v_mov_b32_e32 v2, s03599; CI-NEXT: s_add_u32 s0, s0, 43600; CI-NEXT: s_addc_u32 s1, s1, 03601; CI-NEXT: v_mov_b32_e32 v5, s13602; CI-NEXT: v_mov_b32_e32 v4, s03603; CI-NEXT: flat_store_dword v[2:3], v03604; CI-NEXT: flat_store_dword v[4:5], v13605; CI-NEXT: s_endpgm3606;3607; VI-LABEL: flat_atomic_inc_ret_i64_offset_system:3608; VI: ; %bb.0:3609; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03610; VI-NEXT: s_add_i32 s12, s12, s173611; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83612; VI-NEXT: v_mov_b32_e32 v0, 423613; VI-NEXT: s_mov_b32 flat_scratch_lo, s133614; VI-NEXT: s_waitcnt lgkmcnt(0)3615; VI-NEXT: s_add_u32 s2, s2, 323616; VI-NEXT: s_addc_u32 s3, s3, 03617; VI-NEXT: v_mov_b32_e32 v2, s23618; VI-NEXT: v_mov_b32_e32 v1, 03619; VI-NEXT: v_mov_b32_e32 v3, s33620; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc3621; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3622; VI-NEXT: buffer_wbinvl1_vol3623; VI-NEXT: v_mov_b32_e32 v3, s13624; VI-NEXT: v_mov_b32_e32 v2, s03625; VI-NEXT: s_add_u32 s0, s0, 43626; VI-NEXT: s_addc_u32 s1, s1, 03627; VI-NEXT: v_mov_b32_e32 v5, s13628; VI-NEXT: v_mov_b32_e32 v4, s03629; VI-NEXT: flat_store_dword v[2:3], v03630; VI-NEXT: flat_store_dword v[4:5], v13631; VI-NEXT: s_endpgm3632;3633; GFX9-LABEL: flat_atomic_inc_ret_i64_offset_system:3634; GFX9: ; %bb.0:3635; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03636; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s173637; GFX9-NEXT: v_mov_b32_e32 v0, 423638; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 03639; GFX9-NEXT: v_mov_b32_e32 v1, 03640; GFX9-NEXT: s_waitcnt lgkmcnt(0)3641; GFX9-NEXT: v_mov_b32_e32 v2, s23642; GFX9-NEXT: v_mov_b32_e32 v3, s33643; GFX9-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] offset:32 glc3644; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3645; GFX9-NEXT: buffer_wbinvl1_vol3646; GFX9-NEXT: v_mov_b32_e32 v3, s13647; GFX9-NEXT: v_mov_b32_e32 v2, s03648; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]3649; GFX9-NEXT: s_endpgm3650;3651; GFX10-LABEL: flat_atomic_inc_ret_i64_offset_system:3652; GFX10: ; %bb.0:3653; GFX10-NEXT: s_add_u32 s12, s12, s173654; GFX10-NEXT: s_addc_u32 s13, s13, 03655; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s123656; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s133657; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x03658; GFX10-NEXT: v_mov_b32_e32 v0, 423659; GFX10-NEXT: v_mov_b32_e32 v1, 03660; GFX10-NEXT: s_waitcnt lgkmcnt(0)3661; GFX10-NEXT: s_add_u32 s2, s2, 323662; GFX10-NEXT: s_addc_u32 s3, s3, 03663; GFX10-NEXT: v_mov_b32_e32 v2, s23664; GFX10-NEXT: v_mov_b32_e32 v3, s33665; GFX10-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc3666; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3667; GFX10-NEXT: buffer_gl1_inv3668; GFX10-NEXT: buffer_gl0_inv3669; GFX10-NEXT: v_mov_b32_e32 v3, s13670; GFX10-NEXT: v_mov_b32_e32 v2, s03671; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]3672; GFX10-NEXT: s_endpgm3673;3674; GFX11-LABEL: flat_atomic_inc_ret_i64_offset_system:3675; GFX11: ; %bb.0:3676; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x03677; GFX11-NEXT: v_mov_b32_e32 v0, 423678; GFX11-NEXT: s_waitcnt lgkmcnt(0)3679; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s23680; GFX11-NEXT: v_mov_b32_e32 v3, s33681; GFX11-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3], v[0:1] offset:32 glc3682; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3683; GFX11-NEXT: buffer_gl1_inv3684; GFX11-NEXT: buffer_gl0_inv3685; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s03686; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]3687; GFX11-NEXT: s_endpgm3688;3689; GFX12-LABEL: flat_atomic_inc_ret_i64_offset_system:3690; GFX12: ; %bb.0:3691; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x03692; GFX12-NEXT: v_mov_b32_e32 v0, 423693; GFX12-NEXT: s_wait_kmcnt 0x03694; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s23695; GFX12-NEXT: v_mov_b32_e32 v3, s33696; GFX12-NEXT: global_wb scope:SCOPE_SYS3697; GFX12-NEXT: s_wait_storecnt 0x03698; GFX12-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3], v[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_SYS3699; GFX12-NEXT: s_wait_loadcnt_dscnt 0x03700; GFX12-NEXT: global_inv scope:SCOPE_SYS3701; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s03702; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]3703; GFX12-NEXT: s_endpgm3704 %gep = getelementptr inbounds i64, ptr %ptr, i32 43705 %result = atomicrmw uinc_wrap ptr %gep, i64 42 seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !13706 store i64 %result, ptr %out, align 43707 ret void3708}3709 3710define amdgpu_kernel void @flat_atomic_inc_noret_i64(ptr %ptr) #1 {3711; CI-LABEL: flat_atomic_inc_noret_i64:3712; CI: ; %bb.0:3713; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03714; CI-NEXT: s_add_i32 s12, s12, s173715; CI-NEXT: v_mov_b32_e32 v0, 423716; CI-NEXT: s_mov_b32 flat_scratch_lo, s133717; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83718; CI-NEXT: s_waitcnt lgkmcnt(0)3719; CI-NEXT: v_mov_b32_e32 v3, s13720; CI-NEXT: v_mov_b32_e32 v1, 03721; CI-NEXT: v_mov_b32_e32 v2, s03722; CI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]3723; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3724; CI-NEXT: buffer_wbinvl1_vol3725; CI-NEXT: s_endpgm3726;3727; VI-LABEL: flat_atomic_inc_noret_i64:3728; VI: ; %bb.0:3729; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03730; VI-NEXT: s_add_i32 s12, s12, s173731; VI-NEXT: v_mov_b32_e32 v0, 423732; VI-NEXT: s_mov_b32 flat_scratch_lo, s133733; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83734; VI-NEXT: s_waitcnt lgkmcnt(0)3735; VI-NEXT: v_mov_b32_e32 v3, s13736; VI-NEXT: v_mov_b32_e32 v1, 03737; VI-NEXT: v_mov_b32_e32 v2, s03738; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]3739; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3740; VI-NEXT: buffer_wbinvl1_vol3741; VI-NEXT: s_endpgm3742;3743; GFX9-LABEL: flat_atomic_inc_noret_i64:3744; GFX9: ; %bb.0:3745; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03746; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s173747; GFX9-NEXT: v_mov_b32_e32 v0, 423748; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 03749; GFX9-NEXT: v_mov_b32_e32 v1, 03750; GFX9-NEXT: s_waitcnt lgkmcnt(0)3751; GFX9-NEXT: v_mov_b32_e32 v3, s13752; GFX9-NEXT: v_mov_b32_e32 v2, s03753; GFX9-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]3754; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3755; GFX9-NEXT: buffer_wbinvl1_vol3756; GFX9-NEXT: s_endpgm3757;3758; GFX10-LABEL: flat_atomic_inc_noret_i64:3759; GFX10: ; %bb.0:3760; GFX10-NEXT: s_add_u32 s12, s12, s173761; GFX10-NEXT: s_addc_u32 s13, s13, 03762; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s123763; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s133764; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03765; GFX10-NEXT: v_mov_b32_e32 v0, 423766; GFX10-NEXT: v_mov_b32_e32 v1, 03767; GFX10-NEXT: s_waitcnt lgkmcnt(0)3768; GFX10-NEXT: v_mov_b32_e32 v3, s13769; GFX10-NEXT: v_mov_b32_e32 v2, s03770; GFX10-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]3771; GFX10-NEXT: s_waitcnt lgkmcnt(0)3772; GFX10-NEXT: s_waitcnt_vscnt null, 0x03773; GFX10-NEXT: buffer_gl1_inv3774; GFX10-NEXT: buffer_gl0_inv3775; GFX10-NEXT: s_endpgm3776;3777; GFX11-LABEL: flat_atomic_inc_noret_i64:3778; GFX11: ; %bb.0:3779; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x03780; GFX11-NEXT: v_mov_b32_e32 v0, 423781; GFX11-NEXT: v_mov_b32_e32 v1, 03782; GFX11-NEXT: s_waitcnt lgkmcnt(0)3783; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s03784; GFX11-NEXT: flat_atomic_inc_u64 v[2:3], v[0:1]3785; GFX11-NEXT: s_waitcnt lgkmcnt(0)3786; GFX11-NEXT: s_waitcnt_vscnt null, 0x03787; GFX11-NEXT: buffer_gl1_inv3788; GFX11-NEXT: buffer_gl0_inv3789; GFX11-NEXT: s_endpgm3790;3791; GFX12-LABEL: flat_atomic_inc_noret_i64:3792; GFX12: ; %bb.0:3793; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x03794; GFX12-NEXT: v_mov_b32_e32 v0, 423795; GFX12-NEXT: v_mov_b32_e32 v1, 03796; GFX12-NEXT: s_wait_kmcnt 0x03797; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s03798; GFX12-NEXT: flat_atomic_inc_u64 v[2:3], v[0:1] scope:SCOPE_DEV3799; GFX12-NEXT: s_wait_storecnt_dscnt 0x03800; GFX12-NEXT: global_inv scope:SCOPE_DEV3801; GFX12-NEXT: s_endpgm3802 %result = atomicrmw uinc_wrap ptr %ptr, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !13803 ret void3804}3805 3806define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset(ptr %ptr) #1 {3807; CI-LABEL: flat_atomic_inc_noret_i64_offset:3808; CI: ; %bb.0:3809; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03810; CI-NEXT: s_add_i32 s12, s12, s173811; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83812; CI-NEXT: v_mov_b32_e32 v0, 423813; CI-NEXT: s_mov_b32 flat_scratch_lo, s133814; CI-NEXT: s_waitcnt lgkmcnt(0)3815; CI-NEXT: s_add_u32 s0, s0, 323816; CI-NEXT: s_addc_u32 s1, s1, 03817; CI-NEXT: v_mov_b32_e32 v3, s13818; CI-NEXT: v_mov_b32_e32 v1, 03819; CI-NEXT: v_mov_b32_e32 v2, s03820; CI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]3821; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3822; CI-NEXT: buffer_wbinvl1_vol3823; CI-NEXT: s_endpgm3824;3825; VI-LABEL: flat_atomic_inc_noret_i64_offset:3826; VI: ; %bb.0:3827; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03828; VI-NEXT: s_add_i32 s12, s12, s173829; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83830; VI-NEXT: v_mov_b32_e32 v0, 423831; VI-NEXT: s_mov_b32 flat_scratch_lo, s133832; VI-NEXT: s_waitcnt lgkmcnt(0)3833; VI-NEXT: s_add_u32 s0, s0, 323834; VI-NEXT: s_addc_u32 s1, s1, 03835; VI-NEXT: v_mov_b32_e32 v3, s13836; VI-NEXT: v_mov_b32_e32 v1, 03837; VI-NEXT: v_mov_b32_e32 v2, s03838; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]3839; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3840; VI-NEXT: buffer_wbinvl1_vol3841; VI-NEXT: s_endpgm3842;3843; GFX9-LABEL: flat_atomic_inc_noret_i64_offset:3844; GFX9: ; %bb.0:3845; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03846; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s173847; GFX9-NEXT: v_mov_b32_e32 v0, 423848; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 03849; GFX9-NEXT: v_mov_b32_e32 v1, 03850; GFX9-NEXT: s_waitcnt lgkmcnt(0)3851; GFX9-NEXT: v_mov_b32_e32 v3, s13852; GFX9-NEXT: v_mov_b32_e32 v2, s03853; GFX9-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] offset:323854; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3855; GFX9-NEXT: buffer_wbinvl1_vol3856; GFX9-NEXT: s_endpgm3857;3858; GFX10-LABEL: flat_atomic_inc_noret_i64_offset:3859; GFX10: ; %bb.0:3860; GFX10-NEXT: s_add_u32 s12, s12, s173861; GFX10-NEXT: s_addc_u32 s13, s13, 03862; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s123863; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s133864; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03865; GFX10-NEXT: v_mov_b32_e32 v0, 423866; GFX10-NEXT: v_mov_b32_e32 v1, 03867; GFX10-NEXT: s_waitcnt lgkmcnt(0)3868; GFX10-NEXT: s_add_u32 s0, s0, 323869; GFX10-NEXT: s_addc_u32 s1, s1, 03870; GFX10-NEXT: v_mov_b32_e32 v3, s13871; GFX10-NEXT: v_mov_b32_e32 v2, s03872; GFX10-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]3873; GFX10-NEXT: s_waitcnt lgkmcnt(0)3874; GFX10-NEXT: s_waitcnt_vscnt null, 0x03875; GFX10-NEXT: buffer_gl1_inv3876; GFX10-NEXT: buffer_gl0_inv3877; GFX10-NEXT: s_endpgm3878;3879; GFX11-LABEL: flat_atomic_inc_noret_i64_offset:3880; GFX11: ; %bb.0:3881; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x03882; GFX11-NEXT: v_mov_b32_e32 v0, 423883; GFX11-NEXT: v_mov_b32_e32 v1, 03884; GFX11-NEXT: s_waitcnt lgkmcnt(0)3885; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s03886; GFX11-NEXT: flat_atomic_inc_u64 v[2:3], v[0:1] offset:323887; GFX11-NEXT: s_waitcnt lgkmcnt(0)3888; GFX11-NEXT: s_waitcnt_vscnt null, 0x03889; GFX11-NEXT: buffer_gl1_inv3890; GFX11-NEXT: buffer_gl0_inv3891; GFX11-NEXT: s_endpgm3892;3893; GFX12-LABEL: flat_atomic_inc_noret_i64_offset:3894; GFX12: ; %bb.0:3895; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x03896; GFX12-NEXT: v_mov_b32_e32 v0, 423897; GFX12-NEXT: v_mov_b32_e32 v1, 03898; GFX12-NEXT: s_wait_kmcnt 0x03899; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s03900; GFX12-NEXT: flat_atomic_inc_u64 v[2:3], v[0:1] offset:32 scope:SCOPE_DEV3901; GFX12-NEXT: s_wait_storecnt_dscnt 0x03902; GFX12-NEXT: global_inv scope:SCOPE_DEV3903; GFX12-NEXT: s_endpgm3904 %gep = getelementptr inbounds i64, ptr %ptr, i32 43905 %result = atomicrmw uinc_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !13906 ret void3907}3908 3909define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_system(ptr %ptr) #1 {3910; CI-LABEL: flat_atomic_inc_noret_i64_offset_system:3911; CI: ; %bb.0:3912; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03913; CI-NEXT: s_add_i32 s12, s12, s173914; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83915; CI-NEXT: v_mov_b32_e32 v0, 423916; CI-NEXT: s_mov_b32 flat_scratch_lo, s133917; CI-NEXT: s_waitcnt lgkmcnt(0)3918; CI-NEXT: s_add_u32 s0, s0, 323919; CI-NEXT: s_addc_u32 s1, s1, 03920; CI-NEXT: v_mov_b32_e32 v3, s13921; CI-NEXT: v_mov_b32_e32 v1, 03922; CI-NEXT: v_mov_b32_e32 v2, s03923; CI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]3924; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3925; CI-NEXT: buffer_wbinvl1_vol3926; CI-NEXT: s_endpgm3927;3928; VI-LABEL: flat_atomic_inc_noret_i64_offset_system:3929; VI: ; %bb.0:3930; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03931; VI-NEXT: s_add_i32 s12, s12, s173932; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 83933; VI-NEXT: v_mov_b32_e32 v0, 423934; VI-NEXT: s_mov_b32 flat_scratch_lo, s133935; VI-NEXT: s_waitcnt lgkmcnt(0)3936; VI-NEXT: s_add_u32 s0, s0, 323937; VI-NEXT: s_addc_u32 s1, s1, 03938; VI-NEXT: v_mov_b32_e32 v3, s13939; VI-NEXT: v_mov_b32_e32 v1, 03940; VI-NEXT: v_mov_b32_e32 v2, s03941; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]3942; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3943; VI-NEXT: buffer_wbinvl1_vol3944; VI-NEXT: s_endpgm3945;3946; GFX9-LABEL: flat_atomic_inc_noret_i64_offset_system:3947; GFX9: ; %bb.0:3948; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03949; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s173950; GFX9-NEXT: v_mov_b32_e32 v0, 423951; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 03952; GFX9-NEXT: v_mov_b32_e32 v1, 03953; GFX9-NEXT: s_waitcnt lgkmcnt(0)3954; GFX9-NEXT: v_mov_b32_e32 v3, s13955; GFX9-NEXT: v_mov_b32_e32 v2, s03956; GFX9-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] offset:323957; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3958; GFX9-NEXT: buffer_wbinvl1_vol3959; GFX9-NEXT: s_endpgm3960;3961; GFX10-LABEL: flat_atomic_inc_noret_i64_offset_system:3962; GFX10: ; %bb.0:3963; GFX10-NEXT: s_add_u32 s12, s12, s173964; GFX10-NEXT: s_addc_u32 s13, s13, 03965; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s123966; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s133967; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x03968; GFX10-NEXT: v_mov_b32_e32 v0, 423969; GFX10-NEXT: v_mov_b32_e32 v1, 03970; GFX10-NEXT: s_waitcnt lgkmcnt(0)3971; GFX10-NEXT: s_add_u32 s0, s0, 323972; GFX10-NEXT: s_addc_u32 s1, s1, 03973; GFX10-NEXT: v_mov_b32_e32 v3, s13974; GFX10-NEXT: v_mov_b32_e32 v2, s03975; GFX10-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]3976; GFX10-NEXT: s_waitcnt lgkmcnt(0)3977; GFX10-NEXT: s_waitcnt_vscnt null, 0x03978; GFX10-NEXT: buffer_gl1_inv3979; GFX10-NEXT: buffer_gl0_inv3980; GFX10-NEXT: s_endpgm3981;3982; GFX11-LABEL: flat_atomic_inc_noret_i64_offset_system:3983; GFX11: ; %bb.0:3984; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x03985; GFX11-NEXT: v_mov_b32_e32 v0, 423986; GFX11-NEXT: v_mov_b32_e32 v1, 03987; GFX11-NEXT: s_waitcnt lgkmcnt(0)3988; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s03989; GFX11-NEXT: flat_atomic_inc_u64 v[2:3], v[0:1] offset:323990; GFX11-NEXT: s_waitcnt lgkmcnt(0)3991; GFX11-NEXT: s_waitcnt_vscnt null, 0x03992; GFX11-NEXT: buffer_gl1_inv3993; GFX11-NEXT: buffer_gl0_inv3994; GFX11-NEXT: s_endpgm3995;3996; GFX12-LABEL: flat_atomic_inc_noret_i64_offset_system:3997; GFX12: ; %bb.0:3998; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x03999; GFX12-NEXT: v_mov_b32_e32 v0, 424000; GFX12-NEXT: v_mov_b32_e32 v1, 04001; GFX12-NEXT: s_wait_kmcnt 0x04002; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s04003; GFX12-NEXT: global_wb scope:SCOPE_SYS4004; GFX12-NEXT: s_wait_storecnt 0x04005; GFX12-NEXT: flat_atomic_inc_u64 v[2:3], v[0:1] offset:32 scope:SCOPE_SYS4006; GFX12-NEXT: s_wait_storecnt_dscnt 0x04007; GFX12-NEXT: global_inv scope:SCOPE_SYS4008; GFX12-NEXT: s_endpgm4009 %gep = getelementptr inbounds i64, ptr %ptr, i32 44010 %result = atomicrmw uinc_wrap ptr %gep, i64 42 seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !14011 ret void4012}4013 4014define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_addr64(ptr %out, ptr %ptr) #1 {4015; CI-LABEL: flat_atomic_inc_ret_i64_offset_addr64:4016; CI: ; %bb.0:4017; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x04018; CI-NEXT: v_lshlrev_b32_e32 v4, 3, v04019; CI-NEXT: s_add_i32 s12, s12, s174020; CI-NEXT: s_mov_b32 flat_scratch_lo, s134021; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 84022; CI-NEXT: s_waitcnt lgkmcnt(0)4023; CI-NEXT: v_mov_b32_e32 v0, s24024; CI-NEXT: v_mov_b32_e32 v1, s34025; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v44026; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc4027; CI-NEXT: v_mov_b32_e32 v0, 424028; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v24029; CI-NEXT: v_mov_b32_e32 v1, 04030; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc4031; CI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc4032; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4033; CI-NEXT: buffer_wbinvl1_vol4034; CI-NEXT: v_mov_b32_e32 v3, s14035; CI-NEXT: v_mov_b32_e32 v2, s04036; CI-NEXT: v_add_i32_e32 v2, vcc, v2, v44037; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc4038; CI-NEXT: v_add_i32_e32 v4, vcc, 4, v24039; CI-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc4040; CI-NEXT: flat_store_dword v[2:3], v04041; CI-NEXT: flat_store_dword v[4:5], v14042; CI-NEXT: s_endpgm4043;4044; VI-LABEL: flat_atomic_inc_ret_i64_offset_addr64:4045; VI: ; %bb.0:4046; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x04047; VI-NEXT: v_lshlrev_b32_e32 v4, 3, v04048; VI-NEXT: s_add_i32 s12, s12, s174049; VI-NEXT: s_mov_b32 flat_scratch_lo, s134050; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 84051; VI-NEXT: s_waitcnt lgkmcnt(0)4052; VI-NEXT: v_mov_b32_e32 v0, s24053; VI-NEXT: v_mov_b32_e32 v1, s34054; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v44055; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc4056; VI-NEXT: v_mov_b32_e32 v0, 424057; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v24058; VI-NEXT: v_mov_b32_e32 v1, 04059; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc4060; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc4061; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4062; VI-NEXT: buffer_wbinvl1_vol4063; VI-NEXT: v_mov_b32_e32 v3, s14064; VI-NEXT: v_mov_b32_e32 v2, s04065; VI-NEXT: v_add_u32_e32 v2, vcc, v2, v44066; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc4067; VI-NEXT: v_add_u32_e32 v4, vcc, 4, v24068; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc4069; VI-NEXT: flat_store_dword v[2:3], v04070; VI-NEXT: flat_store_dword v[4:5], v14071; VI-NEXT: s_endpgm4072;4073; GFX9-LABEL: flat_atomic_inc_ret_i64_offset_addr64:4074; GFX9: ; %bb.0:4075; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x04076; GFX9-NEXT: v_lshlrev_b32_e32 v5, 3, v04077; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s174078; GFX9-NEXT: v_mov_b32_e32 v1, 424079; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 04080; GFX9-NEXT: s_waitcnt lgkmcnt(0)4081; GFX9-NEXT: v_mov_b32_e32 v4, s34082; GFX9-NEXT: v_mov_b32_e32 v3, s24083; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, v3, v54084; GFX9-NEXT: v_mov_b32_e32 v2, 04085; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc4086; GFX9-NEXT: flat_atomic_inc_x2 v[0:1], v[3:4], v[1:2] offset:40 glc4087; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4088; GFX9-NEXT: buffer_wbinvl1_vol4089; GFX9-NEXT: v_mov_b32_e32 v3, s14090; GFX9-NEXT: v_mov_b32_e32 v2, s04091; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v54092; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc4093; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]4094; GFX9-NEXT: s_endpgm4095;4096; GFX10-LABEL: flat_atomic_inc_ret_i64_offset_addr64:4097; GFX10: ; %bb.0:4098; GFX10-NEXT: s_add_u32 s12, s12, s174099; GFX10-NEXT: s_addc_u32 s13, s13, 04100; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s124101; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s134102; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x04103; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v04104; GFX10-NEXT: s_waitcnt lgkmcnt(0)4105; GFX10-NEXT: v_mov_b32_e32 v0, s24106; GFX10-NEXT: v_mov_b32_e32 v1, s34107; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v0, v44108; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo4109; GFX10-NEXT: v_mov_b32_e32 v0, 424110; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, 404111; GFX10-NEXT: v_mov_b32_e32 v1, 04112; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo4113; GFX10-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc4114; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4115; GFX10-NEXT: buffer_gl1_inv4116; GFX10-NEXT: buffer_gl0_inv4117; GFX10-NEXT: v_mov_b32_e32 v3, s14118; GFX10-NEXT: v_mov_b32_e32 v2, s04119; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v44120; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo4121; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1]4122; GFX10-NEXT: s_endpgm4123;4124; GFX11-LABEL: flat_atomic_inc_ret_i64_offset_addr64:4125; GFX11: ; %bb.0:4126; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x04127; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v04128; GFX11-NEXT: v_mov_b32_e32 v2, 424129; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)4130; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_lshlrev_b32 v4, 3, v04131; GFX11-NEXT: s_waitcnt lgkmcnt(0)4132; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s34133; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v44134; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)4135; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo4136; GFX11-NEXT: flat_atomic_inc_u64 v[0:1], v[0:1], v[2:3] offset:40 glc4137; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4138; GFX11-NEXT: buffer_gl1_inv4139; GFX11-NEXT: buffer_gl0_inv4140; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s04141; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4142; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, v2, v44143; GFX11-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo4144; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1]4145; GFX11-NEXT: s_endpgm4146;4147; GFX12-LABEL: flat_atomic_inc_ret_i64_offset_addr64:4148; GFX12: ; %bb.0:4149; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x04150; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v04151; GFX12-NEXT: v_mov_b32_e32 v2, 424152; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)4153; GFX12-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_lshlrev_b32 v4, 3, v04154; GFX12-NEXT: s_wait_kmcnt 0x04155; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s34156; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v44157; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)4158; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo4159; GFX12-NEXT: flat_atomic_inc_u64 v[0:1], v[0:1], v[2:3] offset:40 th:TH_ATOMIC_RETURN scope:SCOPE_DEV4160; GFX12-NEXT: s_wait_loadcnt_dscnt 0x04161; GFX12-NEXT: global_inv scope:SCOPE_DEV4162; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s04163; GFX12-NEXT: v_add_co_u32 v2, vcc_lo, v2, v44164; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)4165; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)4166; GFX12-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo4167; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]4168; GFX12-NEXT: s_endpgm4169 %id = call i32 @llvm.amdgcn.workitem.id.x()4170 %gep.tid = getelementptr i64, ptr %ptr, i32 %id4171 %out.gep = getelementptr i64, ptr %out, i32 %id4172 %gep = getelementptr inbounds i64, ptr %gep.tid, i32 54173 %result = atomicrmw uinc_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !14174 store i64 %result, ptr %out.gep, align 44175 ret void4176}4177 4178define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_addr64(ptr %ptr) #1 {4179; CI-LABEL: flat_atomic_inc_noret_i64_offset_addr64:4180; CI: ; %bb.0:4181; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x04182; CI-NEXT: v_lshlrev_b32_e32 v2, 3, v04183; CI-NEXT: s_add_i32 s12, s12, s174184; CI-NEXT: s_mov_b32 flat_scratch_lo, s134185; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 84186; CI-NEXT: s_waitcnt lgkmcnt(0)4187; CI-NEXT: v_mov_b32_e32 v0, s04188; CI-NEXT: v_mov_b32_e32 v1, s14189; CI-NEXT: v_add_i32_e32 v2, vcc, v0, v24190; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc4191; CI-NEXT: v_mov_b32_e32 v0, 424192; CI-NEXT: v_add_i32_e32 v2, vcc, 40, v24193; CI-NEXT: v_mov_b32_e32 v1, 04194; CI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc4195; CI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]4196; CI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4197; CI-NEXT: buffer_wbinvl1_vol4198; CI-NEXT: s_endpgm4199;4200; VI-LABEL: flat_atomic_inc_noret_i64_offset_addr64:4201; VI: ; %bb.0:4202; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x04203; VI-NEXT: v_lshlrev_b32_e32 v2, 3, v04204; VI-NEXT: s_add_i32 s12, s12, s174205; VI-NEXT: s_mov_b32 flat_scratch_lo, s134206; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 84207; VI-NEXT: s_waitcnt lgkmcnt(0)4208; VI-NEXT: v_mov_b32_e32 v0, s04209; VI-NEXT: v_mov_b32_e32 v1, s14210; VI-NEXT: v_add_u32_e32 v2, vcc, v0, v24211; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc4212; VI-NEXT: v_mov_b32_e32 v0, 424213; VI-NEXT: v_add_u32_e32 v2, vcc, 40, v24214; VI-NEXT: v_mov_b32_e32 v1, 04215; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc4216; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]4217; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4218; VI-NEXT: buffer_wbinvl1_vol4219; VI-NEXT: s_endpgm4220;4221; GFX9-LABEL: flat_atomic_inc_noret_i64_offset_addr64:4222; GFX9: ; %bb.0:4223; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x04224; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v04225; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s174226; GFX9-NEXT: v_mov_b32_e32 v1, 424227; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 04228; GFX9-NEXT: s_waitcnt lgkmcnt(0)4229; GFX9-NEXT: v_mov_b32_e32 v4, s14230; GFX9-NEXT: v_mov_b32_e32 v3, s04231; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, v3, v04232; GFX9-NEXT: v_mov_b32_e32 v2, 04233; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc4234; GFX9-NEXT: flat_atomic_inc_x2 v[3:4], v[1:2] offset:404235; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4236; GFX9-NEXT: buffer_wbinvl1_vol4237; GFX9-NEXT: s_endpgm4238;4239; GFX10-LABEL: flat_atomic_inc_noret_i64_offset_addr64:4240; GFX10: ; %bb.0:4241; GFX10-NEXT: s_add_u32 s12, s12, s174242; GFX10-NEXT: s_addc_u32 s13, s13, 04243; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s124244; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s134245; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x04246; GFX10-NEXT: v_lshlrev_b32_e32 v2, 3, v04247; GFX10-NEXT: s_waitcnt lgkmcnt(0)4248; GFX10-NEXT: v_mov_b32_e32 v0, s04249; GFX10-NEXT: v_mov_b32_e32 v1, s14250; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v0, v24251; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo4252; GFX10-NEXT: v_mov_b32_e32 v0, 424253; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, 404254; GFX10-NEXT: v_mov_b32_e32 v1, 04255; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo4256; GFX10-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]4257; GFX10-NEXT: s_waitcnt lgkmcnt(0)4258; GFX10-NEXT: s_waitcnt_vscnt null, 0x04259; GFX10-NEXT: buffer_gl1_inv4260; GFX10-NEXT: buffer_gl0_inv4261; GFX10-NEXT: s_endpgm4262;4263; GFX11-LABEL: flat_atomic_inc_noret_i64_offset_addr64:4264; GFX11: ; %bb.0:4265; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x04266; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v04267; GFX11-NEXT: v_mov_b32_e32 v2, 424268; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)4269; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_lshlrev_b32 v4, 3, v04270; GFX11-NEXT: s_waitcnt lgkmcnt(0)4271; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14272; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v44273; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)4274; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo4275; GFX11-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3] offset:404276; GFX11-NEXT: s_waitcnt lgkmcnt(0)4277; GFX11-NEXT: s_waitcnt_vscnt null, 0x04278; GFX11-NEXT: buffer_gl1_inv4279; GFX11-NEXT: buffer_gl0_inv4280; GFX11-NEXT: s_endpgm4281;4282; GFX12-LABEL: flat_atomic_inc_noret_i64_offset_addr64:4283; GFX12: ; %bb.0:4284; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x04285; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v04286; GFX12-NEXT: v_mov_b32_e32 v2, 424287; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)4288; GFX12-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_lshlrev_b32 v4, 3, v04289; GFX12-NEXT: s_wait_kmcnt 0x04290; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14291; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v44292; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)4293; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo4294; GFX12-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3] offset:40 scope:SCOPE_DEV4295; GFX12-NEXT: s_wait_storecnt_dscnt 0x04296; GFX12-NEXT: global_inv scope:SCOPE_DEV4297; GFX12-NEXT: s_endpgm4298 %id = call i32 @llvm.amdgcn.workitem.id.x()4299 %gep.tid = getelementptr i64, ptr %ptr, i32 %id4300 %gep = getelementptr inbounds i64, ptr %gep.tid, i32 54301 %result = atomicrmw uinc_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !14302 ret void4303}4304 4305define amdgpu_kernel void @nocse_lds_atomic_inc_ret_i32(ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(3) %ptr) #1 {4306; CI-LABEL: nocse_lds_atomic_inc_ret_i32:4307; CI: ; %bb.0:4308; CI-NEXT: s_load_dword s4, s[8:9], 0x44309; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x04310; CI-NEXT: v_mov_b32_e32 v0, 424311; CI-NEXT: s_mov_b32 m0, -14312; CI-NEXT: s_add_i32 s12, s12, s174313; CI-NEXT: s_waitcnt lgkmcnt(0)4314; CI-NEXT: v_mov_b32_e32 v1, s44315; CI-NEXT: ds_inc_rtn_u32 v2, v1, v04316; CI-NEXT: s_waitcnt lgkmcnt(0)4317; CI-NEXT: ds_inc_rtn_u32 v3, v1, v04318; CI-NEXT: s_waitcnt lgkmcnt(0)4319; CI-NEXT: v_mov_b32_e32 v0, s04320; CI-NEXT: s_mov_b32 flat_scratch_lo, s134321; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 84322; CI-NEXT: v_mov_b32_e32 v1, s14323; CI-NEXT: flat_store_dword v[0:1], v24324; CI-NEXT: v_mov_b32_e32 v0, s24325; CI-NEXT: v_mov_b32_e32 v1, s34326; CI-NEXT: flat_store_dword v[0:1], v34327; CI-NEXT: s_endpgm4328;4329; VI-LABEL: nocse_lds_atomic_inc_ret_i32:4330; VI: ; %bb.0:4331; VI-NEXT: s_load_dword s4, s[8:9], 0x104332; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x04333; VI-NEXT: v_mov_b32_e32 v0, 424334; VI-NEXT: s_mov_b32 m0, -14335; VI-NEXT: s_add_i32 s12, s12, s174336; VI-NEXT: s_waitcnt lgkmcnt(0)4337; VI-NEXT: v_mov_b32_e32 v1, s44338; VI-NEXT: ds_inc_rtn_u32 v2, v1, v04339; VI-NEXT: s_waitcnt lgkmcnt(0)4340; VI-NEXT: ds_inc_rtn_u32 v3, v1, v04341; VI-NEXT: s_waitcnt lgkmcnt(0)4342; VI-NEXT: v_mov_b32_e32 v0, s04343; VI-NEXT: s_mov_b32 flat_scratch_lo, s134344; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 84345; VI-NEXT: v_mov_b32_e32 v1, s14346; VI-NEXT: flat_store_dword v[0:1], v24347; VI-NEXT: v_mov_b32_e32 v0, s24348; VI-NEXT: v_mov_b32_e32 v1, s34349; VI-NEXT: flat_store_dword v[0:1], v34350; VI-NEXT: s_endpgm4351;4352; GFX9-LABEL: nocse_lds_atomic_inc_ret_i32:4353; GFX9: ; %bb.0:4354; GFX9-NEXT: s_load_dword s4, s[8:9], 0x104355; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x04356; GFX9-NEXT: v_mov_b32_e32 v1, 424357; GFX9-NEXT: s_waitcnt lgkmcnt(0)4358; GFX9-NEXT: v_mov_b32_e32 v0, s44359; GFX9-NEXT: ds_inc_rtn_u32 v2, v0, v14360; GFX9-NEXT: s_waitcnt lgkmcnt(0)4361; GFX9-NEXT: ds_inc_rtn_u32 v0, v0, v14362; GFX9-NEXT: s_waitcnt lgkmcnt(0)4363; GFX9-NEXT: v_mov_b32_e32 v1, 04364; GFX9-NEXT: global_store_dword v1, v2, s[0:1]4365; GFX9-NEXT: global_store_dword v1, v0, s[2:3]4366; GFX9-NEXT: s_endpgm4367;4368; GFX10-LABEL: nocse_lds_atomic_inc_ret_i32:4369; GFX10: ; %bb.0:4370; GFX10-NEXT: s_load_dword s0, s[8:9], 0x104371; GFX10-NEXT: v_mov_b32_e32 v1, 424372; GFX10-NEXT: s_waitcnt lgkmcnt(0)4373; GFX10-NEXT: v_mov_b32_e32 v0, s04374; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x04375; GFX10-NEXT: s_waitcnt lgkmcnt(0)4376; GFX10-NEXT: ds_inc_rtn_u32 v2, v0, v14377; GFX10-NEXT: s_waitcnt lgkmcnt(0)4378; GFX10-NEXT: buffer_gl0_inv4379; GFX10-NEXT: ds_inc_rtn_u32 v0, v0, v14380; GFX10-NEXT: s_waitcnt lgkmcnt(0)4381; GFX10-NEXT: buffer_gl0_inv4382; GFX10-NEXT: v_mov_b32_e32 v1, 04383; GFX10-NEXT: global_store_dword v1, v2, s[0:1]4384; GFX10-NEXT: global_store_dword v1, v0, s[2:3]4385; GFX10-NEXT: s_endpgm4386;4387; GFX11-LABEL: nocse_lds_atomic_inc_ret_i32:4388; GFX11: ; %bb.0:4389; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x104390; GFX11-NEXT: s_waitcnt lgkmcnt(0)4391; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s04392; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x04393; GFX11-NEXT: s_waitcnt lgkmcnt(0)4394; GFX11-NEXT: ds_inc_rtn_u32 v2, v0, v14395; GFX11-NEXT: s_waitcnt lgkmcnt(0)4396; GFX11-NEXT: buffer_gl0_inv4397; GFX11-NEXT: ds_inc_rtn_u32 v0, v0, v14398; GFX11-NEXT: s_waitcnt lgkmcnt(0)4399; GFX11-NEXT: buffer_gl0_inv4400; GFX11-NEXT: v_mov_b32_e32 v1, 04401; GFX11-NEXT: s_clause 0x14402; GFX11-NEXT: global_store_b32 v1, v2, s[0:1]4403; GFX11-NEXT: global_store_b32 v1, v0, s[2:3]4404; GFX11-NEXT: s_endpgm4405;4406; GFX12-LABEL: nocse_lds_atomic_inc_ret_i32:4407; GFX12: ; %bb.0:4408; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x104409; GFX12-NEXT: s_wait_kmcnt 0x04410; GFX12-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s04411; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x04412; GFX12-NEXT: ds_inc_rtn_u32 v2, v0, v14413; GFX12-NEXT: s_wait_dscnt 0x04414; GFX12-NEXT: global_inv scope:SCOPE_SE4415; GFX12-NEXT: s_wait_loadcnt 0x04416; GFX12-NEXT: ds_inc_rtn_u32 v0, v0, v14417; GFX12-NEXT: s_wait_dscnt 0x04418; GFX12-NEXT: global_inv scope:SCOPE_SE4419; GFX12-NEXT: v_mov_b32_e32 v1, 04420; GFX12-NEXT: s_wait_kmcnt 0x04421; GFX12-NEXT: s_clause 0x14422; GFX12-NEXT: global_store_b32 v1, v2, s[0:1]4423; GFX12-NEXT: global_store_b32 v1, v0, s[2:3]4424; GFX12-NEXT: s_endpgm4425 %result0 = atomicrmw uinc_wrap ptr addrspace(3) %ptr, i32 42 syncscope("agent") seq_cst, align 44426 %result1 = atomicrmw uinc_wrap ptr addrspace(3) %ptr, i32 42 syncscope("agent") seq_cst, align 44427 store i32 %result0, ptr addrspace(1) %out0, align 44428 store i32 %result1, ptr addrspace(1) %out1, align 44429 ret void4430}4431 4432attributes #0 = { nounwind speculatable willreturn memory(none) }4433attributes #1 = { nounwind }4434attributes #2 = { nounwind memory(none) }4435 4436!0 = !{i32 5, i32 6}4437!1 = !{}4438 4439;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:4440; GCN: {{.*}}4441