2502 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX6 %s3; RUN: llc -mtriple=amdgcn -mcpu=tonga -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX8 %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s5; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize64 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX10,GFX10W64 %s6; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX10,GFX10W32 %s7; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11W64 %s8; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11W32 %s9; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12W64 %s10; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12W32 %s11 12declare i32 @llvm.amdgcn.workitem.id.x()13declare i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32, ptr addrspace(8), i32, i32, i32 immarg)14declare i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add(i32, ptr addrspace(8), i32, i32, i32, i32 immarg)15declare i32 @llvm.amdgcn.raw.ptr.buffer.atomic.sub(i32, ptr addrspace(8), i32, i32, i32 immarg)16 17; Show what the atomic optimization pass will do for raw buffers.18 19define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace(8) %inout) {20; GFX6-LABEL: add_i32_constant:21; GFX6: ; %bb.0: ; %entry22; GFX6-NEXT: s_mov_b64 s[2:3], exec23; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 024; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v025; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v026; GFX6-NEXT: ; implicit-def: $vgpr127; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc28; GFX6-NEXT: s_cbranch_execz .LBB0_229; GFX6-NEXT: ; %bb.1:30; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd31; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]32; GFX6-NEXT: s_mul_i32 s2, s2, 533; GFX6-NEXT: v_mov_b32_e32 v1, s234; GFX6-NEXT: s_waitcnt lgkmcnt(0)35; GFX6-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc36; GFX6-NEXT: .LBB0_2:37; GFX6-NEXT: s_or_b64 exec, exec, s[0:1]38; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x939; GFX6-NEXT: s_mov_b32 s3, 0xf00040; GFX6-NEXT: s_mov_b32 s2, -141; GFX6-NEXT: s_waitcnt vmcnt(0)42; GFX6-NEXT: v_readfirstlane_b32 s4, v143; GFX6-NEXT: v_mad_u32_u24 v0, v0, 5, s444; GFX6-NEXT: s_waitcnt lgkmcnt(0)45; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 046; GFX6-NEXT: s_endpgm47;48; GFX8-LABEL: add_i32_constant:49; GFX8: ; %bb.0: ; %entry50; GFX8-NEXT: s_mov_b64 s[2:3], exec51; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 052; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v053; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v054; GFX8-NEXT: ; implicit-def: $vgpr155; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc56; GFX8-NEXT: s_cbranch_execz .LBB0_257; GFX8-NEXT: ; %bb.1:58; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x3459; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]60; GFX8-NEXT: s_mul_i32 s2, s2, 561; GFX8-NEXT: v_mov_b32_e32 v1, s262; GFX8-NEXT: s_waitcnt lgkmcnt(0)63; GFX8-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc64; GFX8-NEXT: .LBB0_2:65; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]66; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2467; GFX8-NEXT: s_waitcnt vmcnt(0)68; GFX8-NEXT: v_readfirstlane_b32 s2, v169; GFX8-NEXT: v_mad_u32_u24 v2, v0, 5, s270; GFX8-NEXT: s_waitcnt lgkmcnt(0)71; GFX8-NEXT: v_mov_b32_e32 v0, s072; GFX8-NEXT: v_mov_b32_e32 v1, s173; GFX8-NEXT: flat_store_dword v[0:1], v274; GFX8-NEXT: s_endpgm75;76; GFX9-LABEL: add_i32_constant:77; GFX9: ; %bb.0: ; %entry78; GFX9-NEXT: s_mov_b64 s[2:3], exec79; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 080; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v081; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v082; GFX9-NEXT: ; implicit-def: $vgpr183; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc84; GFX9-NEXT: s_cbranch_execz .LBB0_285; GFX9-NEXT: ; %bb.1:86; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x3487; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]88; GFX9-NEXT: s_mul_i32 s2, s2, 589; GFX9-NEXT: v_mov_b32_e32 v1, s290; GFX9-NEXT: s_waitcnt lgkmcnt(0)91; GFX9-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc92; GFX9-NEXT: .LBB0_2:93; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]94; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2495; GFX9-NEXT: s_waitcnt vmcnt(0)96; GFX9-NEXT: v_readfirstlane_b32 s2, v197; GFX9-NEXT: v_mov_b32_e32 v2, 098; GFX9-NEXT: v_mad_u32_u24 v0, v0, 5, s299; GFX9-NEXT: s_waitcnt lgkmcnt(0)100; GFX9-NEXT: global_store_dword v2, v0, s[0:1]101; GFX9-NEXT: s_endpgm102;103; GFX10W64-LABEL: add_i32_constant:104; GFX10W64: ; %bb.0: ; %entry105; GFX10W64-NEXT: s_mov_b64 s[2:3], exec106; GFX10W64-NEXT: ; implicit-def: $vgpr1107; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0108; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0109; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0110; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc111; GFX10W64-NEXT: s_cbranch_execz .LBB0_2112; GFX10W64-NEXT: ; %bb.1:113; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34114; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]115; GFX10W64-NEXT: s_mul_i32 s2, s2, 5116; GFX10W64-NEXT: v_mov_b32_e32 v1, s2117; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)118; GFX10W64-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc119; GFX10W64-NEXT: .LBB0_2:120; GFX10W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)121; GFX10W64-NEXT: s_or_b64 exec, exec, s[0:1]122; GFX10W64-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24123; GFX10W64-NEXT: s_waitcnt vmcnt(0)124; GFX10W64-NEXT: v_readfirstlane_b32 s2, v1125; GFX10W64-NEXT: v_mov_b32_e32 v1, 0126; GFX10W64-NEXT: v_mad_u32_u24 v0, v0, 5, s2127; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)128; GFX10W64-NEXT: global_store_dword v1, v0, s[0:1]129; GFX10W64-NEXT: s_endpgm130;131; GFX10W32-LABEL: add_i32_constant:132; GFX10W32: ; %bb.0: ; %entry133; GFX10W32-NEXT: s_mov_b32 s1, exec_lo134; GFX10W32-NEXT: ; implicit-def: $vgpr1135; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0136; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0137; GFX10W32-NEXT: s_and_saveexec_b32 s0, vcc_lo138; GFX10W32-NEXT: s_cbranch_execz .LBB0_2139; GFX10W32-NEXT: ; %bb.1:140; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34141; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s1142; GFX10W32-NEXT: s_mul_i32 s1, s1, 5143; GFX10W32-NEXT: v_mov_b32_e32 v1, s1144; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)145; GFX10W32-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc146; GFX10W32-NEXT: .LBB0_2:147; GFX10W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)148; GFX10W32-NEXT: s_or_b32 exec_lo, exec_lo, s0149; GFX10W32-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24150; GFX10W32-NEXT: s_waitcnt vmcnt(0)151; GFX10W32-NEXT: v_readfirstlane_b32 s2, v1152; GFX10W32-NEXT: v_mov_b32_e32 v1, 0153; GFX10W32-NEXT: v_mad_u32_u24 v0, v0, 5, s2154; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)155; GFX10W32-NEXT: global_store_dword v1, v0, s[0:1]156; GFX10W32-NEXT: s_endpgm157;158; GFX11W64-LABEL: add_i32_constant:159; GFX11W64: ; %bb.0: ; %entry160; GFX11W64-NEXT: s_mov_b64 s[2:3], exec161; GFX11W64-NEXT: s_mov_b64 s[0:1], exec162; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0163; GFX11W64-NEXT: ; implicit-def: $vgpr1164; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)165; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0166; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0167; GFX11W64-NEXT: s_cbranch_execz .LBB0_2168; GFX11W64-NEXT: ; %bb.1:169; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34170; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]171; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)172; GFX11W64-NEXT: s_mul_i32 s2, s2, 5173; GFX11W64-NEXT: v_mov_b32_e32 v1, s2174; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)175; GFX11W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc176; GFX11W64-NEXT: .LBB0_2:177; GFX11W64-NEXT: s_or_b64 exec, exec, s[0:1]178; GFX11W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24179; GFX11W64-NEXT: s_waitcnt vmcnt(0)180; GFX11W64-NEXT: v_readfirstlane_b32 s2, v1181; GFX11W64-NEXT: v_mov_b32_e32 v1, 0182; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_2)183; GFX11W64-NEXT: v_mad_u32_u24 v0, v0, 5, s2184; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)185; GFX11W64-NEXT: global_store_b32 v1, v0, s[0:1]186; GFX11W64-NEXT: s_endpgm187;188; GFX11W32-LABEL: add_i32_constant:189; GFX11W32: ; %bb.0: ; %entry190; GFX11W32-NEXT: s_mov_b32 s1, exec_lo191; GFX11W32-NEXT: s_mov_b32 s0, exec_lo192; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0193; GFX11W32-NEXT: ; implicit-def: $vgpr1194; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)195; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0196; GFX11W32-NEXT: s_cbranch_execz .LBB0_2197; GFX11W32-NEXT: ; %bb.1:198; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34199; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s1200; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)201; GFX11W32-NEXT: s_mul_i32 s1, s1, 5202; GFX11W32-NEXT: v_mov_b32_e32 v1, s1203; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)204; GFX11W32-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc205; GFX11W32-NEXT: .LBB0_2:206; GFX11W32-NEXT: s_or_b32 exec_lo, exec_lo, s0207; GFX11W32-NEXT: s_load_b64 s[0:1], s[4:5], 0x24208; GFX11W32-NEXT: s_waitcnt vmcnt(0)209; GFX11W32-NEXT: v_readfirstlane_b32 s2, v1210; GFX11W32-NEXT: v_mov_b32_e32 v1, 0211; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_2)212; GFX11W32-NEXT: v_mad_u32_u24 v0, v0, 5, s2213; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)214; GFX11W32-NEXT: global_store_b32 v1, v0, s[0:1]215; GFX11W32-NEXT: s_endpgm216;217; GFX12W64-LABEL: add_i32_constant:218; GFX12W64: ; %bb.0: ; %entry219; GFX12W64-NEXT: s_mov_b64 s[2:3], exec220; GFX12W64-NEXT: s_mov_b64 s[0:1], exec221; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0222; GFX12W64-NEXT: ; implicit-def: $vgpr1223; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)224; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0225; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0226; GFX12W64-NEXT: s_cbranch_execz .LBB0_2227; GFX12W64-NEXT: ; %bb.1:228; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34229; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]230; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)231; GFX12W64-NEXT: s_mul_i32 s2, s2, 5232; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)233; GFX12W64-NEXT: v_mov_b32_e32 v1, s2234; GFX12W64-NEXT: s_wait_kmcnt 0x0235; GFX12W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN236; GFX12W64-NEXT: .LBB0_2:237; GFX12W64-NEXT: s_or_b64 exec, exec, s[0:1]238; GFX12W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24239; GFX12W64-NEXT: s_wait_loadcnt 0x0240; GFX12W64-NEXT: v_readfirstlane_b32 s2, v1241; GFX12W64-NEXT: v_mov_b32_e32 v1, 0242; GFX12W64-NEXT: s_wait_alu depctr_va_sdst(0)243; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_2)244; GFX12W64-NEXT: v_mad_u32_u24 v0, v0, 5, s2245; GFX12W64-NEXT: s_wait_kmcnt 0x0246; GFX12W64-NEXT: global_store_b32 v1, v0, s[0:1]247; GFX12W64-NEXT: s_endpgm248;249; GFX12W32-LABEL: add_i32_constant:250; GFX12W32: ; %bb.0: ; %entry251; GFX12W32-NEXT: s_mov_b32 s1, exec_lo252; GFX12W32-NEXT: s_mov_b32 s0, exec_lo253; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0254; GFX12W32-NEXT: ; implicit-def: $vgpr1255; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)256; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0257; GFX12W32-NEXT: s_cbranch_execz .LBB0_2258; GFX12W32-NEXT: ; %bb.1:259; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34260; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s1261; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)262; GFX12W32-NEXT: s_mul_i32 s1, s1, 5263; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)264; GFX12W32-NEXT: v_mov_b32_e32 v1, s1265; GFX12W32-NEXT: s_wait_kmcnt 0x0266; GFX12W32-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN267; GFX12W32-NEXT: .LBB0_2:268; GFX12W32-NEXT: s_or_b32 exec_lo, exec_lo, s0269; GFX12W32-NEXT: s_load_b64 s[0:1], s[4:5], 0x24270; GFX12W32-NEXT: s_wait_loadcnt 0x0271; GFX12W32-NEXT: v_readfirstlane_b32 s2, v1272; GFX12W32-NEXT: v_mov_b32_e32 v1, 0273; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_2)274; GFX12W32-NEXT: v_mad_u32_u24 v0, v0, 5, s2275; GFX12W32-NEXT: s_wait_kmcnt 0x0276; GFX12W32-NEXT: global_store_b32 v1, v0, s[0:1]277; GFX12W32-NEXT: s_endpgm278entry:279 %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32 5, ptr addrspace(8) %inout, i32 0, i32 0, i32 0)280 store i32 %old, ptr addrspace(1) %out281 ret void282}283 284define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(8) %inout, i32 %additive) {285; GFX6-LABEL: add_i32_uniform:286; GFX6: ; %bb.0: ; %entry287; GFX6-NEXT: s_mov_b64 s[2:3], exec288; GFX6-NEXT: s_load_dword s6, s[4:5], 0x11289; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0290; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0291; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0292; GFX6-NEXT: ; implicit-def: $vgpr1293; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc294; GFX6-NEXT: s_cbranch_execz .LBB1_2295; GFX6-NEXT: ; %bb.1:296; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd297; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]298; GFX6-NEXT: s_waitcnt lgkmcnt(0)299; GFX6-NEXT: s_mul_i32 s2, s6, s2300; GFX6-NEXT: v_mov_b32_e32 v1, s2301; GFX6-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc302; GFX6-NEXT: .LBB1_2:303; GFX6-NEXT: s_or_b64 exec, exec, s[0:1]304; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9305; GFX6-NEXT: s_mov_b32 s3, 0xf000306; GFX6-NEXT: s_mov_b32 s2, -1307; GFX6-NEXT: s_waitcnt vmcnt(0)308; GFX6-NEXT: v_readfirstlane_b32 s4, v1309; GFX6-NEXT: s_waitcnt lgkmcnt(0)310; GFX6-NEXT: v_mul_lo_u32 v0, s6, v0311; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v0312; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0313; GFX6-NEXT: s_endpgm314;315; GFX8-LABEL: add_i32_uniform:316; GFX8: ; %bb.0: ; %entry317; GFX8-NEXT: s_load_dword s6, s[4:5], 0x44318; GFX8-NEXT: s_mov_b64 s[2:3], exec319; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0320; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0321; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0322; GFX8-NEXT: ; implicit-def: $vgpr1323; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc324; GFX8-NEXT: s_cbranch_execz .LBB1_2325; GFX8-NEXT: ; %bb.1:326; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34327; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]328; GFX8-NEXT: s_waitcnt lgkmcnt(0)329; GFX8-NEXT: s_mul_i32 s2, s6, s2330; GFX8-NEXT: v_mov_b32_e32 v1, s2331; GFX8-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc332; GFX8-NEXT: .LBB1_2:333; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]334; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24335; GFX8-NEXT: s_waitcnt lgkmcnt(0)336; GFX8-NEXT: v_mul_lo_u32 v0, s6, v0337; GFX8-NEXT: s_waitcnt vmcnt(0)338; GFX8-NEXT: v_readfirstlane_b32 s2, v1339; GFX8-NEXT: v_add_u32_e32 v2, vcc, s2, v0340; GFX8-NEXT: v_mov_b32_e32 v0, s0341; GFX8-NEXT: v_mov_b32_e32 v1, s1342; GFX8-NEXT: flat_store_dword v[0:1], v2343; GFX8-NEXT: s_endpgm344;345; GFX9-LABEL: add_i32_uniform:346; GFX9: ; %bb.0: ; %entry347; GFX9-NEXT: s_load_dword s6, s[4:5], 0x44348; GFX9-NEXT: s_mov_b64 s[2:3], exec349; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0350; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0351; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0352; GFX9-NEXT: ; implicit-def: $vgpr1353; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc354; GFX9-NEXT: s_cbranch_execz .LBB1_2355; GFX9-NEXT: ; %bb.1:356; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34357; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]358; GFX9-NEXT: s_waitcnt lgkmcnt(0)359; GFX9-NEXT: s_mul_i32 s2, s6, s2360; GFX9-NEXT: v_mov_b32_e32 v1, s2361; GFX9-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc362; GFX9-NEXT: .LBB1_2:363; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]364; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24365; GFX9-NEXT: s_waitcnt lgkmcnt(0)366; GFX9-NEXT: v_mul_lo_u32 v0, s6, v0367; GFX9-NEXT: s_waitcnt vmcnt(0)368; GFX9-NEXT: v_readfirstlane_b32 s2, v1369; GFX9-NEXT: v_mov_b32_e32 v2, 0370; GFX9-NEXT: v_add_u32_e32 v0, s2, v0371; GFX9-NEXT: global_store_dword v2, v0, s[0:1]372; GFX9-NEXT: s_endpgm373;374; GFX10W64-LABEL: add_i32_uniform:375; GFX10W64: ; %bb.0: ; %entry376; GFX10W64-NEXT: s_load_dword s6, s[4:5], 0x44377; GFX10W64-NEXT: s_mov_b64 s[2:3], exec378; GFX10W64-NEXT: ; implicit-def: $vgpr1379; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0380; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0381; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0382; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc383; GFX10W64-NEXT: s_cbranch_execz .LBB1_2384; GFX10W64-NEXT: ; %bb.1:385; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34386; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]387; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)388; GFX10W64-NEXT: s_mul_i32 s2, s6, s2389; GFX10W64-NEXT: v_mov_b32_e32 v1, s2390; GFX10W64-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc391; GFX10W64-NEXT: .LBB1_2:392; GFX10W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)393; GFX10W64-NEXT: s_or_b64 exec, exec, s[0:1]394; GFX10W64-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24395; GFX10W64-NEXT: s_waitcnt vmcnt(0)396; GFX10W64-NEXT: v_readfirstlane_b32 s2, v1397; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)398; GFX10W64-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s6, v0, s[2:3]399; GFX10W64-NEXT: v_mov_b32_e32 v1, 0400; GFX10W64-NEXT: global_store_dword v1, v0, s[0:1]401; GFX10W64-NEXT: s_endpgm402;403; GFX10W32-LABEL: add_i32_uniform:404; GFX10W32: ; %bb.0: ; %entry405; GFX10W32-NEXT: s_load_dword s0, s[4:5], 0x44406; GFX10W32-NEXT: s_mov_b32 s2, exec_lo407; GFX10W32-NEXT: ; implicit-def: $vgpr1408; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0409; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0410; GFX10W32-NEXT: s_and_saveexec_b32 s1, vcc_lo411; GFX10W32-NEXT: s_cbranch_execz .LBB1_2412; GFX10W32-NEXT: ; %bb.1:413; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34414; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s2415; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)416; GFX10W32-NEXT: s_mul_i32 s2, s0, s2417; GFX10W32-NEXT: v_mov_b32_e32 v1, s2418; GFX10W32-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc419; GFX10W32-NEXT: .LBB1_2:420; GFX10W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)421; GFX10W32-NEXT: s_or_b32 exec_lo, exec_lo, s1422; GFX10W32-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24423; GFX10W32-NEXT: s_waitcnt vmcnt(0)424; GFX10W32-NEXT: s_mov_b32 null, 0425; GFX10W32-NEXT: v_readfirstlane_b32 s4, v1426; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)427; GFX10W32-NEXT: v_mad_u64_u32 v[0:1], s0, s0, v0, s[4:5]428; GFX10W32-NEXT: v_mov_b32_e32 v1, 0429; GFX10W32-NEXT: global_store_dword v1, v0, s[2:3]430; GFX10W32-NEXT: s_endpgm431;432; GFX11W64-LABEL: add_i32_uniform:433; GFX11W64: ; %bb.0: ; %entry434; GFX11W64-NEXT: s_load_b32 s6, s[4:5], 0x44435; GFX11W64-NEXT: s_mov_b64 s[2:3], exec436; GFX11W64-NEXT: s_mov_b64 s[0:1], exec437; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0438; GFX11W64-NEXT: ; implicit-def: $vgpr1439; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)440; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0441; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0442; GFX11W64-NEXT: s_cbranch_execz .LBB1_2443; GFX11W64-NEXT: ; %bb.1:444; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34445; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]446; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)447; GFX11W64-NEXT: s_mul_i32 s2, s6, s2448; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)449; GFX11W64-NEXT: v_mov_b32_e32 v1, s2450; GFX11W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc451; GFX11W64-NEXT: .LBB1_2:452; GFX11W64-NEXT: s_or_b64 exec, exec, s[0:1]453; GFX11W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24454; GFX11W64-NEXT: s_waitcnt vmcnt(0)455; GFX11W64-NEXT: v_readfirstlane_b32 s2, v1456; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)457; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)458; GFX11W64-NEXT: v_mad_u64_u32 v[1:2], null, s6, v0, s[2:3]459; GFX11W64-NEXT: v_mov_b32_e32 v0, 0460; GFX11W64-NEXT: global_store_b32 v0, v1, s[0:1]461; GFX11W64-NEXT: s_endpgm462;463; GFX11W32-LABEL: add_i32_uniform:464; GFX11W32: ; %bb.0: ; %entry465; GFX11W32-NEXT: s_load_b32 s0, s[4:5], 0x44466; GFX11W32-NEXT: s_mov_b32 s2, exec_lo467; GFX11W32-NEXT: s_mov_b32 s1, exec_lo468; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0469; GFX11W32-NEXT: ; implicit-def: $vgpr1470; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)471; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0472; GFX11W32-NEXT: s_cbranch_execz .LBB1_2473; GFX11W32-NEXT: ; %bb.1:474; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34475; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s2476; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)477; GFX11W32-NEXT: s_mul_i32 s2, s0, s2478; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)479; GFX11W32-NEXT: v_mov_b32_e32 v1, s2480; GFX11W32-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc481; GFX11W32-NEXT: .LBB1_2:482; GFX11W32-NEXT: s_or_b32 exec_lo, exec_lo, s1483; GFX11W32-NEXT: s_load_b64 s[2:3], s[4:5], 0x24484; GFX11W32-NEXT: s_waitcnt vmcnt(0)485; GFX11W32-NEXT: v_readfirstlane_b32 s4, v1486; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)487; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)488; GFX11W32-NEXT: v_mad_u64_u32 v[1:2], null, s0, v0, s[4:5]489; GFX11W32-NEXT: v_mov_b32_e32 v0, 0490; GFX11W32-NEXT: global_store_b32 v0, v1, s[2:3]491; GFX11W32-NEXT: s_endpgm492;493; GFX12W64-LABEL: add_i32_uniform:494; GFX12W64: ; %bb.0: ; %entry495; GFX12W64-NEXT: s_load_b32 s6, s[4:5], 0x44496; GFX12W64-NEXT: s_mov_b64 s[2:3], exec497; GFX12W64-NEXT: s_mov_b64 s[0:1], exec498; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0499; GFX12W64-NEXT: ; implicit-def: $vgpr1500; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)501; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0502; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0503; GFX12W64-NEXT: s_cbranch_execz .LBB1_2504; GFX12W64-NEXT: ; %bb.1:505; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34506; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]507; GFX12W64-NEXT: s_wait_kmcnt 0x0508; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)509; GFX12W64-NEXT: s_mul_i32 s2, s6, s2510; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)511; GFX12W64-NEXT: v_mov_b32_e32 v1, s2512; GFX12W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN513; GFX12W64-NEXT: .LBB1_2:514; GFX12W64-NEXT: s_or_b64 exec, exec, s[0:1]515; GFX12W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24516; GFX12W64-NEXT: s_wait_loadcnt 0x0517; GFX12W64-NEXT: v_readfirstlane_b32 s2, v1518; GFX12W64-NEXT: s_wait_kmcnt 0x0519; GFX12W64-NEXT: s_wait_alu depctr_va_sdst(0)520; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)521; GFX12W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]522; GFX12W64-NEXT: v_mov_b32_e32 v1, 0523; GFX12W64-NEXT: global_store_b32 v1, v0, s[0:1]524; GFX12W64-NEXT: s_endpgm525;526; GFX12W32-LABEL: add_i32_uniform:527; GFX12W32: ; %bb.0: ; %entry528; GFX12W32-NEXT: s_load_b32 s0, s[4:5], 0x44529; GFX12W32-NEXT: s_mov_b32 s2, exec_lo530; GFX12W32-NEXT: s_mov_b32 s1, exec_lo531; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0532; GFX12W32-NEXT: ; implicit-def: $vgpr1533; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)534; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0535; GFX12W32-NEXT: s_cbranch_execz .LBB1_2536; GFX12W32-NEXT: ; %bb.1:537; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34538; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s2539; GFX12W32-NEXT: s_wait_kmcnt 0x0540; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)541; GFX12W32-NEXT: s_mul_i32 s2, s0, s2542; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)543; GFX12W32-NEXT: v_mov_b32_e32 v1, s2544; GFX12W32-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN545; GFX12W32-NEXT: .LBB1_2:546; GFX12W32-NEXT: s_or_b32 exec_lo, exec_lo, s1547; GFX12W32-NEXT: s_load_b64 s[2:3], s[4:5], 0x24548; GFX12W32-NEXT: s_wait_loadcnt 0x0549; GFX12W32-NEXT: v_readfirstlane_b32 s4, v1550; GFX12W32-NEXT: s_wait_kmcnt 0x0551; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)552; GFX12W32-NEXT: v_mad_co_u64_u32 v[0:1], null, s0, v0, s[4:5]553; GFX12W32-NEXT: v_mov_b32_e32 v1, 0554; GFX12W32-NEXT: global_store_b32 v1, v0, s[2:3]555; GFX12W32-NEXT: s_endpgm556entry:557 %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32 %additive, ptr addrspace(8) %inout, i32 0, i32 0, i32 0)558 store i32 %old, ptr addrspace(1) %out559 ret void560}561 562define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addrspace(8) %inout) {563; GFX6-LABEL: add_i32_varying_vdata:564; GFX6: ; %bb.0: ; %entry565; GFX6-NEXT: s_mov_b64 s[0:1], exec566; GFX6-NEXT: s_mov_b32 s2, 0567; GFX6-NEXT: ; implicit-def: $vgpr1568; GFX6-NEXT: .LBB2_1: ; %ComputeLoop569; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1570; GFX6-NEXT: s_ff1_i32_b64 s3, s[0:1]571; GFX6-NEXT: s_mov_b32 m0, s3572; GFX6-NEXT: v_readlane_b32 s8, v0, s3573; GFX6-NEXT: v_writelane_b32 v1, s2, m0574; GFX6-NEXT: s_lshl_b64 s[6:7], 1, s3575; GFX6-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]576; GFX6-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 0577; GFX6-NEXT: s_and_b64 vcc, exec, s[6:7]578; GFX6-NEXT: s_add_i32 s2, s2, s8579; GFX6-NEXT: s_cbranch_vccnz .LBB2_1580; GFX6-NEXT: ; %bb.2: ; %ComputeEnd581; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0582; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0583; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0584; GFX6-NEXT: ; implicit-def: $vgpr0585; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc586; GFX6-NEXT: s_xor_b64 s[0:1], exec, s[0:1]587; GFX6-NEXT: s_cbranch_execz .LBB2_4588; GFX6-NEXT: ; %bb.3:589; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd590; GFX6-NEXT: v_mov_b32_e32 v0, s2591; GFX6-NEXT: s_waitcnt lgkmcnt(0)592; GFX6-NEXT: buffer_atomic_add v0, off, s[8:11], 0 glc593; GFX6-NEXT: .LBB2_4:594; GFX6-NEXT: s_or_b64 exec, exec, s[0:1]595; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9596; GFX6-NEXT: s_mov_b32 s3, 0xf000597; GFX6-NEXT: s_mov_b32 s2, -1598; GFX6-NEXT: s_waitcnt vmcnt(0)599; GFX6-NEXT: v_readfirstlane_b32 s4, v0600; GFX6-NEXT: s_waitcnt expcnt(0)601; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v1602; GFX6-NEXT: s_waitcnt lgkmcnt(0)603; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0604; GFX6-NEXT: s_endpgm605;606; GFX8-LABEL: add_i32_varying_vdata:607; GFX8: ; %bb.0: ; %entry608; GFX8-NEXT: s_mov_b64 s[0:1], exec609; GFX8-NEXT: s_mov_b32 s2, 0610; GFX8-NEXT: ; implicit-def: $vgpr1611; GFX8-NEXT: .LBB2_1: ; %ComputeLoop612; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1613; GFX8-NEXT: s_ff1_i32_b64 s3, s[0:1]614; GFX8-NEXT: s_mov_b32 m0, s3615; GFX8-NEXT: v_readlane_b32 s6, v0, s3616; GFX8-NEXT: v_writelane_b32 v1, s2, m0617; GFX8-NEXT: s_add_i32 s2, s2, s6618; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s3619; GFX8-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]620; GFX8-NEXT: s_cbranch_scc1 .LBB2_1621; GFX8-NEXT: ; %bb.2: ; %ComputeEnd622; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0623; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0624; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0625; GFX8-NEXT: ; implicit-def: $vgpr0626; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc627; GFX8-NEXT: s_xor_b64 s[0:1], exec, s[0:1]628; GFX8-NEXT: s_cbranch_execz .LBB2_4629; GFX8-NEXT: ; %bb.3:630; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34631; GFX8-NEXT: v_mov_b32_e32 v0, s2632; GFX8-NEXT: s_waitcnt lgkmcnt(0)633; GFX8-NEXT: buffer_atomic_add v0, off, s[8:11], 0 glc634; GFX8-NEXT: .LBB2_4:635; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]636; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24637; GFX8-NEXT: s_waitcnt vmcnt(0)638; GFX8-NEXT: v_readfirstlane_b32 s2, v0639; GFX8-NEXT: v_add_u32_e32 v2, vcc, s2, v1640; GFX8-NEXT: s_waitcnt lgkmcnt(0)641; GFX8-NEXT: v_mov_b32_e32 v0, s0642; GFX8-NEXT: v_mov_b32_e32 v1, s1643; GFX8-NEXT: flat_store_dword v[0:1], v2644; GFX8-NEXT: s_endpgm645;646; GFX9-LABEL: add_i32_varying_vdata:647; GFX9: ; %bb.0: ; %entry648; GFX9-NEXT: s_mov_b64 s[0:1], exec649; GFX9-NEXT: s_mov_b32 s2, 0650; GFX9-NEXT: ; implicit-def: $vgpr1651; GFX9-NEXT: .LBB2_1: ; %ComputeLoop652; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1653; GFX9-NEXT: s_ff1_i32_b64 s3, s[0:1]654; GFX9-NEXT: s_mov_b32 m0, s3655; GFX9-NEXT: v_readlane_b32 s6, v0, s3656; GFX9-NEXT: v_writelane_b32 v1, s2, m0657; GFX9-NEXT: s_add_i32 s2, s2, s6658; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s3659; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]660; GFX9-NEXT: s_cbranch_scc1 .LBB2_1661; GFX9-NEXT: ; %bb.2: ; %ComputeEnd662; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0663; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0664; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0665; GFX9-NEXT: ; implicit-def: $vgpr0666; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc667; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]668; GFX9-NEXT: s_cbranch_execz .LBB2_4669; GFX9-NEXT: ; %bb.3:670; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34671; GFX9-NEXT: v_mov_b32_e32 v0, s2672; GFX9-NEXT: s_waitcnt lgkmcnt(0)673; GFX9-NEXT: buffer_atomic_add v0, off, s[8:11], 0 glc674; GFX9-NEXT: .LBB2_4:675; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]676; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24677; GFX9-NEXT: s_waitcnt vmcnt(0)678; GFX9-NEXT: v_readfirstlane_b32 s2, v0679; GFX9-NEXT: v_mov_b32_e32 v2, 0680; GFX9-NEXT: v_add_u32_e32 v0, s2, v1681; GFX9-NEXT: s_waitcnt lgkmcnt(0)682; GFX9-NEXT: global_store_dword v2, v0, s[0:1]683; GFX9-NEXT: s_endpgm684;685; GFX10W64-LABEL: add_i32_varying_vdata:686; GFX10W64: ; %bb.0: ; %entry687; GFX10W64-NEXT: s_mov_b64 s[0:1], exec688; GFX10W64-NEXT: s_mov_b32 s2, 0689; GFX10W64-NEXT: ; implicit-def: $vgpr1690; GFX10W64-NEXT: .LBB2_1: ; %ComputeLoop691; GFX10W64-NEXT: ; =>This Inner Loop Header: Depth=1692; GFX10W64-NEXT: s_ff1_i32_b64 s3, s[0:1]693; GFX10W64-NEXT: v_readlane_b32 s8, v0, s3694; GFX10W64-NEXT: v_writelane_b32 v1, s2, s3695; GFX10W64-NEXT: s_lshl_b64 s[6:7], 1, s3696; GFX10W64-NEXT: s_add_i32 s2, s2, s8697; GFX10W64-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]698; GFX10W64-NEXT: s_cbranch_scc1 .LBB2_1699; GFX10W64-NEXT: ; %bb.2: ; %ComputeEnd700; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0701; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0702; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0703; GFX10W64-NEXT: ; implicit-def: $vgpr0704; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc705; GFX10W64-NEXT: s_xor_b64 s[0:1], exec, s[0:1]706; GFX10W64-NEXT: s_cbranch_execz .LBB2_4707; GFX10W64-NEXT: ; %bb.3:708; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34709; GFX10W64-NEXT: v_mov_b32_e32 v0, s2710; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)711; GFX10W64-NEXT: buffer_atomic_add v0, off, s[8:11], 0 glc712; GFX10W64-NEXT: .LBB2_4:713; GFX10W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)714; GFX10W64-NEXT: s_or_b64 exec, exec, s[0:1]715; GFX10W64-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24716; GFX10W64-NEXT: s_waitcnt vmcnt(0)717; GFX10W64-NEXT: v_readfirstlane_b32 s2, v0718; GFX10W64-NEXT: v_mov_b32_e32 v0, 0719; GFX10W64-NEXT: v_add_nc_u32_e32 v1, s2, v1720; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)721; GFX10W64-NEXT: global_store_dword v0, v1, s[0:1]722; GFX10W64-NEXT: s_endpgm723;724; GFX10W32-LABEL: add_i32_varying_vdata:725; GFX10W32: ; %bb.0: ; %entry726; GFX10W32-NEXT: s_mov_b32 s1, exec_lo727; GFX10W32-NEXT: s_mov_b32 s0, 0728; GFX10W32-NEXT: ; implicit-def: $vgpr1729; GFX10W32-NEXT: .LBB2_1: ; %ComputeLoop730; GFX10W32-NEXT: ; =>This Inner Loop Header: Depth=1731; GFX10W32-NEXT: s_ff1_i32_b32 s2, s1732; GFX10W32-NEXT: v_readlane_b32 s3, v0, s2733; GFX10W32-NEXT: v_writelane_b32 v1, s0, s2734; GFX10W32-NEXT: s_lshl_b32 s2, 1, s2735; GFX10W32-NEXT: s_add_i32 s0, s0, s3736; GFX10W32-NEXT: s_andn2_b32 s1, s1, s2737; GFX10W32-NEXT: s_cbranch_scc1 .LBB2_1738; GFX10W32-NEXT: ; %bb.2: ; %ComputeEnd739; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0740; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0741; GFX10W32-NEXT: ; implicit-def: $vgpr0742; GFX10W32-NEXT: s_and_saveexec_b32 s1, vcc_lo743; GFX10W32-NEXT: s_xor_b32 s1, exec_lo, s1744; GFX10W32-NEXT: s_cbranch_execz .LBB2_4745; GFX10W32-NEXT: ; %bb.3:746; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34747; GFX10W32-NEXT: v_mov_b32_e32 v0, s0748; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)749; GFX10W32-NEXT: buffer_atomic_add v0, off, s[8:11], 0 glc750; GFX10W32-NEXT: .LBB2_4:751; GFX10W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)752; GFX10W32-NEXT: s_or_b32 exec_lo, exec_lo, s1753; GFX10W32-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24754; GFX10W32-NEXT: s_waitcnt vmcnt(0)755; GFX10W32-NEXT: v_readfirstlane_b32 s2, v0756; GFX10W32-NEXT: v_mov_b32_e32 v0, 0757; GFX10W32-NEXT: v_add_nc_u32_e32 v1, s2, v1758; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)759; GFX10W32-NEXT: global_store_dword v0, v1, s[0:1]760; GFX10W32-NEXT: s_endpgm761;762; GFX11W64-LABEL: add_i32_varying_vdata:763; GFX11W64: ; %bb.0: ; %entry764; GFX11W64-NEXT: v_and_b32_e32 v1, 0x3ff, v0765; GFX11W64-NEXT: s_mov_b64 s[0:1], exec766; GFX11W64-NEXT: s_mov_b32 s2, 0767; GFX11W64-NEXT: ; implicit-def: $vgpr0768; GFX11W64-NEXT: .LBB2_1: ; %ComputeLoop769; GFX11W64-NEXT: ; =>This Inner Loop Header: Depth=1770; GFX11W64-NEXT: s_ctz_i32_b64 s3, s[0:1]771; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)772; GFX11W64-NEXT: v_readlane_b32 s8, v1, s3773; GFX11W64-NEXT: v_writelane_b32 v0, s2, s3774; GFX11W64-NEXT: s_lshl_b64 s[6:7], 1, s3775; GFX11W64-NEXT: s_add_i32 s2, s2, s8776; GFX11W64-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[6:7]777; GFX11W64-NEXT: s_cbranch_scc1 .LBB2_1778; GFX11W64-NEXT: ; %bb.2: ; %ComputeEnd779; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0780; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)781; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1782; GFX11W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1783; GFX11W64-NEXT: ; implicit-def: $vgpr1784; GFX11W64-NEXT: s_and_saveexec_b64 s[0:1], vcc785; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)786; GFX11W64-NEXT: s_xor_b64 s[0:1], exec, s[0:1]787; GFX11W64-NEXT: s_cbranch_execz .LBB2_4788; GFX11W64-NEXT: ; %bb.3:789; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34790; GFX11W64-NEXT: v_mov_b32_e32 v1, s2791; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)792; GFX11W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc793; GFX11W64-NEXT: .LBB2_4:794; GFX11W64-NEXT: s_or_b64 exec, exec, s[0:1]795; GFX11W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24796; GFX11W64-NEXT: s_waitcnt vmcnt(0)797; GFX11W64-NEXT: v_readfirstlane_b32 s2, v1798; GFX11W64-NEXT: v_mov_b32_e32 v1, 0799; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_2)800; GFX11W64-NEXT: v_add_nc_u32_e32 v0, s2, v0801; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)802; GFX11W64-NEXT: global_store_b32 v1, v0, s[0:1]803; GFX11W64-NEXT: s_endpgm804;805; GFX11W32-LABEL: add_i32_varying_vdata:806; GFX11W32: ; %bb.0: ; %entry807; GFX11W32-NEXT: v_and_b32_e32 v1, 0x3ff, v0808; GFX11W32-NEXT: s_mov_b32 s1, exec_lo809; GFX11W32-NEXT: s_mov_b32 s0, 0810; GFX11W32-NEXT: ; implicit-def: $vgpr0811; GFX11W32-NEXT: .LBB2_1: ; %ComputeLoop812; GFX11W32-NEXT: ; =>This Inner Loop Header: Depth=1813; GFX11W32-NEXT: s_ctz_i32_b32 s2, s1814; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)815; GFX11W32-NEXT: v_readlane_b32 s3, v1, s2816; GFX11W32-NEXT: v_writelane_b32 v0, s0, s2817; GFX11W32-NEXT: s_lshl_b32 s2, 1, s2818; GFX11W32-NEXT: s_add_i32 s0, s0, s3819; GFX11W32-NEXT: s_and_not1_b32 s1, s1, s2820; GFX11W32-NEXT: s_cbranch_scc1 .LBB2_1821; GFX11W32-NEXT: ; %bb.2: ; %ComputeEnd822; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0823; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)824; GFX11W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1825; GFX11W32-NEXT: ; implicit-def: $vgpr1826; GFX11W32-NEXT: s_and_saveexec_b32 s1, vcc_lo827; GFX11W32-NEXT: s_xor_b32 s1, exec_lo, s1828; GFX11W32-NEXT: s_cbranch_execz .LBB2_4829; GFX11W32-NEXT: ; %bb.3:830; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34831; GFX11W32-NEXT: v_mov_b32_e32 v1, s0832; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)833; GFX11W32-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc834; GFX11W32-NEXT: .LBB2_4:835; GFX11W32-NEXT: s_or_b32 exec_lo, exec_lo, s1836; GFX11W32-NEXT: s_load_b64 s[0:1], s[4:5], 0x24837; GFX11W32-NEXT: s_waitcnt vmcnt(0)838; GFX11W32-NEXT: v_readfirstlane_b32 s2, v1839; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)840; GFX11W32-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, s2, v0841; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)842; GFX11W32-NEXT: global_store_b32 v1, v0, s[0:1]843; GFX11W32-NEXT: s_endpgm844;845; GFX12W64-LABEL: add_i32_varying_vdata:846; GFX12W64: ; %bb.0: ; %entry847; GFX12W64-NEXT: v_and_b32_e32 v1, 0x3ff, v0848; GFX12W64-NEXT: s_mov_b64 s[0:1], exec849; GFX12W64-NEXT: s_mov_b32 s2, 0850; GFX12W64-NEXT: ; implicit-def: $vgpr0851; GFX12W64-NEXT: .LBB2_1: ; %ComputeLoop852; GFX12W64-NEXT: ; =>This Inner Loop Header: Depth=1853; GFX12W64-NEXT: s_ctz_i32_b64 s3, s[0:1]854; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)855; GFX12W64-NEXT: v_readlane_b32 s8, v1, s3856; GFX12W64-NEXT: v_writelane_b32 v0, s2, s3857; GFX12W64-NEXT: s_lshl_b64 s[6:7], 1, s3858; GFX12W64-NEXT: s_add_co_i32 s2, s2, s8859; GFX12W64-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[6:7]860; GFX12W64-NEXT: s_cbranch_scc1 .LBB2_1861; GFX12W64-NEXT: ; %bb.2: ; %ComputeEnd862; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0863; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)864; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1865; GFX12W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1866; GFX12W64-NEXT: ; implicit-def: $vgpr1867; GFX12W64-NEXT: s_and_saveexec_b64 s[0:1], vcc868; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)869; GFX12W64-NEXT: s_xor_b64 s[0:1], exec, s[0:1]870; GFX12W64-NEXT: s_cbranch_execz .LBB2_4871; GFX12W64-NEXT: ; %bb.3:872; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34873; GFX12W64-NEXT: v_mov_b32_e32 v1, s2874; GFX12W64-NEXT: s_wait_kmcnt 0x0875; GFX12W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN876; GFX12W64-NEXT: .LBB2_4:877; GFX12W64-NEXT: s_or_b64 exec, exec, s[0:1]878; GFX12W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24879; GFX12W64-NEXT: s_wait_loadcnt 0x0880; GFX12W64-NEXT: v_readfirstlane_b32 s2, v1881; GFX12W64-NEXT: v_mov_b32_e32 v1, 0882; GFX12W64-NEXT: s_wait_alu depctr_va_sdst(0)883; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_2)884; GFX12W64-NEXT: v_add_nc_u32_e32 v0, s2, v0885; GFX12W64-NEXT: s_wait_kmcnt 0x0886; GFX12W64-NEXT: global_store_b32 v1, v0, s[0:1]887; GFX12W64-NEXT: s_endpgm888;889; GFX12W32-LABEL: add_i32_varying_vdata:890; GFX12W32: ; %bb.0: ; %entry891; GFX12W32-NEXT: v_and_b32_e32 v1, 0x3ff, v0892; GFX12W32-NEXT: s_mov_b32 s1, exec_lo893; GFX12W32-NEXT: s_mov_b32 s0, 0894; GFX12W32-NEXT: ; implicit-def: $vgpr0895; GFX12W32-NEXT: .LBB2_1: ; %ComputeLoop896; GFX12W32-NEXT: ; =>This Inner Loop Header: Depth=1897; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)898; GFX12W32-NEXT: s_ctz_i32_b32 s2, s1899; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)900; GFX12W32-NEXT: v_readlane_b32 s3, v1, s2901; GFX12W32-NEXT: v_writelane_b32 v0, s0, s2902; GFX12W32-NEXT: s_lshl_b32 s2, 1, s2903; GFX12W32-NEXT: s_add_co_i32 s0, s0, s3904; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)905; GFX12W32-NEXT: s_and_not1_b32 s1, s1, s2906; GFX12W32-NEXT: s_cbranch_scc1 .LBB2_1907; GFX12W32-NEXT: ; %bb.2: ; %ComputeEnd908; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0909; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)910; GFX12W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1911; GFX12W32-NEXT: ; implicit-def: $vgpr1912; GFX12W32-NEXT: s_and_saveexec_b32 s1, vcc_lo913; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)914; GFX12W32-NEXT: s_xor_b32 s1, exec_lo, s1915; GFX12W32-NEXT: s_cbranch_execz .LBB2_4916; GFX12W32-NEXT: ; %bb.3:917; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34918; GFX12W32-NEXT: v_mov_b32_e32 v1, s0919; GFX12W32-NEXT: s_wait_kmcnt 0x0920; GFX12W32-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN921; GFX12W32-NEXT: .LBB2_4:922; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)923; GFX12W32-NEXT: s_or_b32 exec_lo, exec_lo, s1924; GFX12W32-NEXT: s_load_b64 s[0:1], s[4:5], 0x24925; GFX12W32-NEXT: s_wait_loadcnt 0x0926; GFX12W32-NEXT: v_readfirstlane_b32 s2, v1927; GFX12W32-NEXT: s_wait_alu depctr_va_sdst(0)928; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)929; GFX12W32-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, s2, v0930; GFX12W32-NEXT: s_wait_kmcnt 0x0931; GFX12W32-NEXT: global_store_b32 v1, v0, s[0:1]932; GFX12W32-NEXT: s_endpgm933entry:934 %lane = call i32 @llvm.amdgcn.workitem.id.x()935 %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32 %lane, ptr addrspace(8) %inout, i32 0, i32 0, i32 0)936 store i32 %old, ptr addrspace(1) %out937 ret void938}939 940define amdgpu_kernel void @struct_add_i32_varying_vdata(ptr addrspace(1) %out, ptr addrspace(8) %inout, i32 %vindex) {941; GFX6-LABEL: struct_add_i32_varying_vdata:942; GFX6: ; %bb.0: ; %entry943; GFX6-NEXT: s_mov_b64 s[0:1], exec944; GFX6-NEXT: s_mov_b32 s2, 0945; GFX6-NEXT: ; implicit-def: $vgpr1946; GFX6-NEXT: .LBB3_1: ; %ComputeLoop947; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1948; GFX6-NEXT: s_ff1_i32_b64 s3, s[0:1]949; GFX6-NEXT: s_mov_b32 m0, s3950; GFX6-NEXT: v_readlane_b32 s8, v0, s3951; GFX6-NEXT: v_writelane_b32 v1, s2, m0952; GFX6-NEXT: s_lshl_b64 s[6:7], 1, s3953; GFX6-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]954; GFX6-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 0955; GFX6-NEXT: s_and_b64 vcc, exec, s[6:7]956; GFX6-NEXT: s_add_i32 s2, s2, s8957; GFX6-NEXT: s_cbranch_vccnz .LBB3_1958; GFX6-NEXT: ; %bb.2: ; %ComputeEnd959; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0960; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0961; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0962; GFX6-NEXT: ; implicit-def: $vgpr0963; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc964; GFX6-NEXT: s_xor_b64 s[0:1], exec, s[0:1]965; GFX6-NEXT: s_cbranch_execz .LBB3_4966; GFX6-NEXT: ; %bb.3:967; GFX6-NEXT: s_load_dword s3, s[4:5], 0x11968; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd969; GFX6-NEXT: v_mov_b32_e32 v0, s2970; GFX6-NEXT: s_waitcnt lgkmcnt(0)971; GFX6-NEXT: v_mov_b32_e32 v2, s3972; GFX6-NEXT: buffer_atomic_add v0, v2, s[8:11], 0 idxen glc973; GFX6-NEXT: .LBB3_4:974; GFX6-NEXT: s_or_b64 exec, exec, s[0:1]975; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9976; GFX6-NEXT: s_mov_b32 s3, 0xf000977; GFX6-NEXT: s_mov_b32 s2, -1978; GFX6-NEXT: s_waitcnt vmcnt(0)979; GFX6-NEXT: v_readfirstlane_b32 s4, v0980; GFX6-NEXT: s_waitcnt expcnt(0)981; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v1982; GFX6-NEXT: s_waitcnt lgkmcnt(0)983; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0984; GFX6-NEXT: s_endpgm985;986; GFX8-LABEL: struct_add_i32_varying_vdata:987; GFX8: ; %bb.0: ; %entry988; GFX8-NEXT: s_mov_b64 s[0:1], exec989; GFX8-NEXT: s_mov_b32 s2, 0990; GFX8-NEXT: ; implicit-def: $vgpr1991; GFX8-NEXT: .LBB3_1: ; %ComputeLoop992; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1993; GFX8-NEXT: s_ff1_i32_b64 s3, s[0:1]994; GFX8-NEXT: s_mov_b32 m0, s3995; GFX8-NEXT: v_readlane_b32 s6, v0, s3996; GFX8-NEXT: v_writelane_b32 v1, s2, m0997; GFX8-NEXT: s_add_i32 s2, s2, s6998; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s3999; GFX8-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]1000; GFX8-NEXT: s_cbranch_scc1 .LBB3_11001; GFX8-NEXT: ; %bb.2: ; %ComputeEnd1002; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 01003; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v01004; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v01005; GFX8-NEXT: ; implicit-def: $vgpr01006; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc1007; GFX8-NEXT: s_xor_b64 s[0:1], exec, s[0:1]1008; GFX8-NEXT: s_cbranch_execz .LBB3_41009; GFX8-NEXT: ; %bb.3:1010; GFX8-NEXT: s_load_dword s3, s[4:5], 0x441011; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x341012; GFX8-NEXT: v_mov_b32_e32 v0, s21013; GFX8-NEXT: s_waitcnt lgkmcnt(0)1014; GFX8-NEXT: v_mov_b32_e32 v2, s31015; GFX8-NEXT: buffer_atomic_add v0, v2, s[8:11], 0 idxen glc1016; GFX8-NEXT: .LBB3_4:1017; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]1018; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241019; GFX8-NEXT: s_waitcnt vmcnt(0)1020; GFX8-NEXT: v_readfirstlane_b32 s2, v01021; GFX8-NEXT: v_add_u32_e32 v2, vcc, s2, v11022; GFX8-NEXT: s_waitcnt lgkmcnt(0)1023; GFX8-NEXT: v_mov_b32_e32 v0, s01024; GFX8-NEXT: v_mov_b32_e32 v1, s11025; GFX8-NEXT: flat_store_dword v[0:1], v21026; GFX8-NEXT: s_endpgm1027;1028; GFX9-LABEL: struct_add_i32_varying_vdata:1029; GFX9: ; %bb.0: ; %entry1030; GFX9-NEXT: s_mov_b64 s[0:1], exec1031; GFX9-NEXT: s_mov_b32 s2, 01032; GFX9-NEXT: ; implicit-def: $vgpr11033; GFX9-NEXT: .LBB3_1: ; %ComputeLoop1034; GFX9-NEXT: ; =>This Inner Loop Header: Depth=11035; GFX9-NEXT: s_ff1_i32_b64 s3, s[0:1]1036; GFX9-NEXT: s_mov_b32 m0, s31037; GFX9-NEXT: v_readlane_b32 s6, v0, s31038; GFX9-NEXT: v_writelane_b32 v1, s2, m01039; GFX9-NEXT: s_add_i32 s2, s2, s61040; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s31041; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]1042; GFX9-NEXT: s_cbranch_scc1 .LBB3_11043; GFX9-NEXT: ; %bb.2: ; %ComputeEnd1044; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 01045; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v01046; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v01047; GFX9-NEXT: ; implicit-def: $vgpr01048; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc1049; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]1050; GFX9-NEXT: s_cbranch_execz .LBB3_41051; GFX9-NEXT: ; %bb.3:1052; GFX9-NEXT: s_load_dword s3, s[4:5], 0x441053; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x341054; GFX9-NEXT: v_mov_b32_e32 v0, s21055; GFX9-NEXT: s_waitcnt lgkmcnt(0)1056; GFX9-NEXT: v_mov_b32_e32 v2, s31057; GFX9-NEXT: buffer_atomic_add v0, v2, s[8:11], 0 idxen glc1058; GFX9-NEXT: .LBB3_4:1059; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]1060; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241061; GFX9-NEXT: s_waitcnt vmcnt(0)1062; GFX9-NEXT: v_readfirstlane_b32 s2, v01063; GFX9-NEXT: v_mov_b32_e32 v2, 01064; GFX9-NEXT: v_add_u32_e32 v0, s2, v11065; GFX9-NEXT: s_waitcnt lgkmcnt(0)1066; GFX9-NEXT: global_store_dword v2, v0, s[0:1]1067; GFX9-NEXT: s_endpgm1068;1069; GFX10W64-LABEL: struct_add_i32_varying_vdata:1070; GFX10W64: ; %bb.0: ; %entry1071; GFX10W64-NEXT: s_mov_b64 s[0:1], exec1072; GFX10W64-NEXT: s_mov_b32 s2, 01073; GFX10W64-NEXT: ; implicit-def: $vgpr11074; GFX10W64-NEXT: .LBB3_1: ; %ComputeLoop1075; GFX10W64-NEXT: ; =>This Inner Loop Header: Depth=11076; GFX10W64-NEXT: s_ff1_i32_b64 s3, s[0:1]1077; GFX10W64-NEXT: v_readlane_b32 s8, v0, s31078; GFX10W64-NEXT: v_writelane_b32 v1, s2, s31079; GFX10W64-NEXT: s_lshl_b64 s[6:7], 1, s31080; GFX10W64-NEXT: s_add_i32 s2, s2, s81081; GFX10W64-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]1082; GFX10W64-NEXT: s_cbranch_scc1 .LBB3_11083; GFX10W64-NEXT: ; %bb.2: ; %ComputeEnd1084; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 01085; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v01086; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v01087; GFX10W64-NEXT: ; implicit-def: $vgpr01088; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc1089; GFX10W64-NEXT: s_xor_b64 s[0:1], exec, s[0:1]1090; GFX10W64-NEXT: s_cbranch_execz .LBB3_41091; GFX10W64-NEXT: ; %bb.3:1092; GFX10W64-NEXT: s_clause 0x11093; GFX10W64-NEXT: s_load_dword s3, s[4:5], 0x441094; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x341095; GFX10W64-NEXT: v_mov_b32_e32 v0, s21096; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)1097; GFX10W64-NEXT: v_mov_b32_e32 v2, s31098; GFX10W64-NEXT: buffer_atomic_add v0, v2, s[8:11], 0 idxen glc1099; GFX10W64-NEXT: .LBB3_4:1100; GFX10W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1101; GFX10W64-NEXT: s_or_b64 exec, exec, s[0:1]1102; GFX10W64-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241103; GFX10W64-NEXT: s_waitcnt vmcnt(0)1104; GFX10W64-NEXT: v_readfirstlane_b32 s2, v01105; GFX10W64-NEXT: v_mov_b32_e32 v0, 01106; GFX10W64-NEXT: v_add_nc_u32_e32 v1, s2, v11107; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)1108; GFX10W64-NEXT: global_store_dword v0, v1, s[0:1]1109; GFX10W64-NEXT: s_endpgm1110;1111; GFX10W32-LABEL: struct_add_i32_varying_vdata:1112; GFX10W32: ; %bb.0: ; %entry1113; GFX10W32-NEXT: s_mov_b32 s1, exec_lo1114; GFX10W32-NEXT: s_mov_b32 s0, 01115; GFX10W32-NEXT: ; implicit-def: $vgpr11116; GFX10W32-NEXT: .LBB3_1: ; %ComputeLoop1117; GFX10W32-NEXT: ; =>This Inner Loop Header: Depth=11118; GFX10W32-NEXT: s_ff1_i32_b32 s2, s11119; GFX10W32-NEXT: v_readlane_b32 s3, v0, s21120; GFX10W32-NEXT: v_writelane_b32 v1, s0, s21121; GFX10W32-NEXT: s_lshl_b32 s2, 1, s21122; GFX10W32-NEXT: s_add_i32 s0, s0, s31123; GFX10W32-NEXT: s_andn2_b32 s1, s1, s21124; GFX10W32-NEXT: s_cbranch_scc1 .LBB3_11125; GFX10W32-NEXT: ; %bb.2: ; %ComputeEnd1126; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 01127; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v01128; GFX10W32-NEXT: ; implicit-def: $vgpr01129; GFX10W32-NEXT: s_and_saveexec_b32 s1, vcc_lo1130; GFX10W32-NEXT: s_xor_b32 s1, exec_lo, s11131; GFX10W32-NEXT: s_cbranch_execz .LBB3_41132; GFX10W32-NEXT: ; %bb.3:1133; GFX10W32-NEXT: s_clause 0x11134; GFX10W32-NEXT: s_load_dword s2, s[4:5], 0x441135; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x341136; GFX10W32-NEXT: v_mov_b32_e32 v0, s01137; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)1138; GFX10W32-NEXT: v_mov_b32_e32 v2, s21139; GFX10W32-NEXT: buffer_atomic_add v0, v2, s[8:11], 0 idxen glc1140; GFX10W32-NEXT: .LBB3_4:1141; GFX10W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1142; GFX10W32-NEXT: s_or_b32 exec_lo, exec_lo, s11143; GFX10W32-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241144; GFX10W32-NEXT: s_waitcnt vmcnt(0)1145; GFX10W32-NEXT: v_readfirstlane_b32 s2, v01146; GFX10W32-NEXT: v_mov_b32_e32 v0, 01147; GFX10W32-NEXT: v_add_nc_u32_e32 v1, s2, v11148; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)1149; GFX10W32-NEXT: global_store_dword v0, v1, s[0:1]1150; GFX10W32-NEXT: s_endpgm1151;1152; GFX11W64-LABEL: struct_add_i32_varying_vdata:1153; GFX11W64: ; %bb.0: ; %entry1154; GFX11W64-NEXT: v_and_b32_e32 v1, 0x3ff, v01155; GFX11W64-NEXT: s_mov_b64 s[0:1], exec1156; GFX11W64-NEXT: s_mov_b32 s2, 01157; GFX11W64-NEXT: ; implicit-def: $vgpr01158; GFX11W64-NEXT: .LBB3_1: ; %ComputeLoop1159; GFX11W64-NEXT: ; =>This Inner Loop Header: Depth=11160; GFX11W64-NEXT: s_ctz_i32_b64 s3, s[0:1]1161; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)1162; GFX11W64-NEXT: v_readlane_b32 s8, v1, s31163; GFX11W64-NEXT: v_writelane_b32 v0, s2, s31164; GFX11W64-NEXT: s_lshl_b64 s[6:7], 1, s31165; GFX11W64-NEXT: s_add_i32 s2, s2, s81166; GFX11W64-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[6:7]1167; GFX11W64-NEXT: s_cbranch_scc1 .LBB3_11168; GFX11W64-NEXT: ; %bb.2: ; %ComputeEnd1169; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 01170; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1171; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v11172; GFX11W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v11173; GFX11W64-NEXT: ; implicit-def: $vgpr11174; GFX11W64-NEXT: s_and_saveexec_b64 s[0:1], vcc1175; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1176; GFX11W64-NEXT: s_xor_b64 s[0:1], exec, s[0:1]1177; GFX11W64-NEXT: s_cbranch_execz .LBB3_41178; GFX11W64-NEXT: ; %bb.3:1179; GFX11W64-NEXT: s_clause 0x11180; GFX11W64-NEXT: s_load_b32 s3, s[4:5], 0x441181; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x341182; GFX11W64-NEXT: v_mov_b32_e32 v1, s21183; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)1184; GFX11W64-NEXT: v_mov_b32_e32 v2, s31185; GFX11W64-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], 0 idxen glc1186; GFX11W64-NEXT: .LBB3_4:1187; GFX11W64-NEXT: s_or_b64 exec, exec, s[0:1]1188; GFX11W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x241189; GFX11W64-NEXT: s_waitcnt vmcnt(0)1190; GFX11W64-NEXT: v_readfirstlane_b32 s2, v11191; GFX11W64-NEXT: v_mov_b32_e32 v1, 01192; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_2)1193; GFX11W64-NEXT: v_add_nc_u32_e32 v0, s2, v01194; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)1195; GFX11W64-NEXT: global_store_b32 v1, v0, s[0:1]1196; GFX11W64-NEXT: s_endpgm1197;1198; GFX11W32-LABEL: struct_add_i32_varying_vdata:1199; GFX11W32: ; %bb.0: ; %entry1200; GFX11W32-NEXT: v_and_b32_e32 v1, 0x3ff, v01201; GFX11W32-NEXT: s_mov_b32 s1, exec_lo1202; GFX11W32-NEXT: s_mov_b32 s0, 01203; GFX11W32-NEXT: ; implicit-def: $vgpr01204; GFX11W32-NEXT: .LBB3_1: ; %ComputeLoop1205; GFX11W32-NEXT: ; =>This Inner Loop Header: Depth=11206; GFX11W32-NEXT: s_ctz_i32_b32 s2, s11207; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)1208; GFX11W32-NEXT: v_readlane_b32 s3, v1, s21209; GFX11W32-NEXT: v_writelane_b32 v0, s0, s21210; GFX11W32-NEXT: s_lshl_b32 s2, 1, s21211; GFX11W32-NEXT: s_add_i32 s0, s0, s31212; GFX11W32-NEXT: s_and_not1_b32 s1, s1, s21213; GFX11W32-NEXT: s_cbranch_scc1 .LBB3_11214; GFX11W32-NEXT: ; %bb.2: ; %ComputeEnd1215; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 01216; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)1217; GFX11W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v11218; GFX11W32-NEXT: ; implicit-def: $vgpr11219; GFX11W32-NEXT: s_and_saveexec_b32 s1, vcc_lo1220; GFX11W32-NEXT: s_xor_b32 s1, exec_lo, s11221; GFX11W32-NEXT: s_cbranch_execz .LBB3_41222; GFX11W32-NEXT: ; %bb.3:1223; GFX11W32-NEXT: s_clause 0x11224; GFX11W32-NEXT: s_load_b32 s2, s[4:5], 0x441225; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x341226; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)1227; GFX11W32-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v2, s21228; GFX11W32-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], 0 idxen glc1229; GFX11W32-NEXT: .LBB3_4:1230; GFX11W32-NEXT: s_or_b32 exec_lo, exec_lo, s11231; GFX11W32-NEXT: s_load_b64 s[0:1], s[4:5], 0x241232; GFX11W32-NEXT: s_waitcnt vmcnt(0)1233; GFX11W32-NEXT: v_readfirstlane_b32 s2, v11234; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)1235; GFX11W32-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, s2, v01236; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)1237; GFX11W32-NEXT: global_store_b32 v1, v0, s[0:1]1238; GFX11W32-NEXT: s_endpgm1239;1240; GFX12W64-LABEL: struct_add_i32_varying_vdata:1241; GFX12W64: ; %bb.0: ; %entry1242; GFX12W64-NEXT: v_and_b32_e32 v1, 0x3ff, v01243; GFX12W64-NEXT: s_mov_b64 s[0:1], exec1244; GFX12W64-NEXT: s_mov_b32 s2, 01245; GFX12W64-NEXT: ; implicit-def: $vgpr01246; GFX12W64-NEXT: .LBB3_1: ; %ComputeLoop1247; GFX12W64-NEXT: ; =>This Inner Loop Header: Depth=11248; GFX12W64-NEXT: s_ctz_i32_b64 s3, s[0:1]1249; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)1250; GFX12W64-NEXT: v_readlane_b32 s8, v1, s31251; GFX12W64-NEXT: v_writelane_b32 v0, s2, s31252; GFX12W64-NEXT: s_lshl_b64 s[6:7], 1, s31253; GFX12W64-NEXT: s_add_co_i32 s2, s2, s81254; GFX12W64-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[6:7]1255; GFX12W64-NEXT: s_cbranch_scc1 .LBB3_11256; GFX12W64-NEXT: ; %bb.2: ; %ComputeEnd1257; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 01258; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1259; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v11260; GFX12W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v11261; GFX12W64-NEXT: ; implicit-def: $vgpr11262; GFX12W64-NEXT: s_and_saveexec_b64 s[0:1], vcc1263; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1264; GFX12W64-NEXT: s_xor_b64 s[0:1], exec, s[0:1]1265; GFX12W64-NEXT: s_cbranch_execz .LBB3_41266; GFX12W64-NEXT: ; %bb.3:1267; GFX12W64-NEXT: s_clause 0x11268; GFX12W64-NEXT: s_load_b32 s3, s[4:5], 0x441269; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x341270; GFX12W64-NEXT: v_mov_b32_e32 v1, s21271; GFX12W64-NEXT: s_wait_kmcnt 0x01272; GFX12W64-NEXT: v_mov_b32_e32 v2, s31273; GFX12W64-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN1274; GFX12W64-NEXT: .LBB3_4:1275; GFX12W64-NEXT: s_or_b64 exec, exec, s[0:1]1276; GFX12W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x241277; GFX12W64-NEXT: s_wait_loadcnt 0x01278; GFX12W64-NEXT: v_readfirstlane_b32 s2, v11279; GFX12W64-NEXT: v_mov_b32_e32 v1, 01280; GFX12W64-NEXT: s_wait_alu depctr_va_sdst(0)1281; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_2)1282; GFX12W64-NEXT: v_add_nc_u32_e32 v0, s2, v01283; GFX12W64-NEXT: s_wait_kmcnt 0x01284; GFX12W64-NEXT: global_store_b32 v1, v0, s[0:1]1285; GFX12W64-NEXT: s_endpgm1286;1287; GFX12W32-LABEL: struct_add_i32_varying_vdata:1288; GFX12W32: ; %bb.0: ; %entry1289; GFX12W32-NEXT: v_and_b32_e32 v1, 0x3ff, v01290; GFX12W32-NEXT: s_mov_b32 s1, exec_lo1291; GFX12W32-NEXT: s_mov_b32 s0, 01292; GFX12W32-NEXT: ; implicit-def: $vgpr01293; GFX12W32-NEXT: .LBB3_1: ; %ComputeLoop1294; GFX12W32-NEXT: ; =>This Inner Loop Header: Depth=11295; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)1296; GFX12W32-NEXT: s_ctz_i32_b32 s2, s11297; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)1298; GFX12W32-NEXT: v_readlane_b32 s3, v1, s21299; GFX12W32-NEXT: v_writelane_b32 v0, s0, s21300; GFX12W32-NEXT: s_lshl_b32 s2, 1, s21301; GFX12W32-NEXT: s_add_co_i32 s0, s0, s31302; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)1303; GFX12W32-NEXT: s_and_not1_b32 s1, s1, s21304; GFX12W32-NEXT: s_cbranch_scc1 .LBB3_11305; GFX12W32-NEXT: ; %bb.2: ; %ComputeEnd1306; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 01307; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)1308; GFX12W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v11309; GFX12W32-NEXT: ; implicit-def: $vgpr11310; GFX12W32-NEXT: s_and_saveexec_b32 s1, vcc_lo1311; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)1312; GFX12W32-NEXT: s_xor_b32 s1, exec_lo, s11313; GFX12W32-NEXT: s_cbranch_execz .LBB3_41314; GFX12W32-NEXT: ; %bb.3:1315; GFX12W32-NEXT: s_clause 0x11316; GFX12W32-NEXT: s_load_b32 s2, s[4:5], 0x441317; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x341318; GFX12W32-NEXT: s_wait_kmcnt 0x01319; GFX12W32-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v2, s21320; GFX12W32-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN1321; GFX12W32-NEXT: .LBB3_4:1322; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)1323; GFX12W32-NEXT: s_or_b32 exec_lo, exec_lo, s11324; GFX12W32-NEXT: s_load_b64 s[0:1], s[4:5], 0x241325; GFX12W32-NEXT: s_wait_loadcnt 0x01326; GFX12W32-NEXT: v_readfirstlane_b32 s2, v11327; GFX12W32-NEXT: s_wait_alu depctr_va_sdst(0)1328; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)1329; GFX12W32-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, s2, v01330; GFX12W32-NEXT: s_wait_kmcnt 0x01331; GFX12W32-NEXT: global_store_b32 v1, v0, s[0:1]1332; GFX12W32-NEXT: s_endpgm1333entry:1334 %lane = call i32 @llvm.amdgcn.workitem.id.x()1335 %old = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add(i32 %lane, ptr addrspace(8) %inout, i32 %vindex, i32 0, i32 0, i32 0)1336 store i32 %old, ptr addrspace(1) %out1337 ret void1338}1339 1340define amdgpu_kernel void @add_i32_varying_offset(ptr addrspace(1) %out, ptr addrspace(8) %inout) {1341; GFX6-LABEL: add_i32_varying_offset:1342; GFX6: ; %bb.0: ; %entry1343; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xd1344; GFX6-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x91345; GFX6-NEXT: v_mov_b32_e32 v1, 11346; GFX6-NEXT: s_waitcnt lgkmcnt(0)1347; GFX6-NEXT: buffer_atomic_add v1, v0, s[0:3], 0 offen glc1348; GFX6-NEXT: s_mov_b32 s7, 0xf0001349; GFX6-NEXT: s_mov_b32 s6, -11350; GFX6-NEXT: s_waitcnt vmcnt(0)1351; GFX6-NEXT: buffer_store_dword v1, off, s[4:7], 01352; GFX6-NEXT: s_endpgm1353;1354; GFX8-LABEL: add_i32_varying_offset:1355; GFX8: ; %bb.0: ; %entry1356; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x341357; GFX8-NEXT: v_mov_b32_e32 v2, 11358; GFX8-NEXT: s_waitcnt lgkmcnt(0)1359; GFX8-NEXT: buffer_atomic_add v2, v0, s[0:3], 0 offen glc1360; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241361; GFX8-NEXT: s_waitcnt lgkmcnt(0)1362; GFX8-NEXT: v_mov_b32_e32 v0, s01363; GFX8-NEXT: v_mov_b32_e32 v1, s11364; GFX8-NEXT: s_waitcnt vmcnt(0)1365; GFX8-NEXT: flat_store_dword v[0:1], v21366; GFX8-NEXT: s_endpgm1367;1368; GFX9-LABEL: add_i32_varying_offset:1369; GFX9: ; %bb.0: ; %entry1370; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x341371; GFX9-NEXT: v_mov_b32_e32 v1, 11372; GFX9-NEXT: s_waitcnt lgkmcnt(0)1373; GFX9-NEXT: buffer_atomic_add v1, v0, s[0:3], 0 offen glc1374; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241375; GFX9-NEXT: v_mov_b32_e32 v0, 01376; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1377; GFX9-NEXT: global_store_dword v0, v1, s[0:1]1378; GFX9-NEXT: s_endpgm1379;1380; GFX10-LABEL: add_i32_varying_offset:1381; GFX10: ; %bb.0: ; %entry1382; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x341383; GFX10-NEXT: v_mov_b32_e32 v1, 11384; GFX10-NEXT: s_waitcnt lgkmcnt(0)1385; GFX10-NEXT: buffer_atomic_add v1, v0, s[0:3], 0 offen glc1386; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1387; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241388; GFX10-NEXT: v_mov_b32_e32 v0, 01389; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1390; GFX10-NEXT: global_store_dword v0, v1, s[0:1]1391; GFX10-NEXT: s_endpgm1392;1393; GFX11W64-LABEL: add_i32_varying_offset:1394; GFX11W64: ; %bb.0: ; %entry1395; GFX11W64-NEXT: s_load_b128 s[0:3], s[4:5], 0x341396; GFX11W64-NEXT: v_and_b32_e32 v0, 0x3ff, v01397; GFX11W64-NEXT: v_mov_b32_e32 v1, 11398; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)1399; GFX11W64-NEXT: buffer_atomic_add_u32 v1, v0, s[0:3], 0 offen glc1400; GFX11W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x241401; GFX11W64-NEXT: v_mov_b32_e32 v0, 01402; GFX11W64-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1403; GFX11W64-NEXT: global_store_b32 v0, v1, s[0:1]1404; GFX11W64-NEXT: s_endpgm1405;1406; GFX11W32-LABEL: add_i32_varying_offset:1407; GFX11W32: ; %bb.0: ; %entry1408; GFX11W32-NEXT: s_load_b128 s[0:3], s[4:5], 0x341409; GFX11W32-NEXT: v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v01410; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)1411; GFX11W32-NEXT: buffer_atomic_add_u32 v1, v0, s[0:3], 0 offen glc1412; GFX11W32-NEXT: s_load_b64 s[0:1], s[4:5], 0x241413; GFX11W32-NEXT: v_mov_b32_e32 v0, 01414; GFX11W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1415; GFX11W32-NEXT: global_store_b32 v0, v1, s[0:1]1416; GFX11W32-NEXT: s_endpgm1417;1418; GFX12W64-LABEL: add_i32_varying_offset:1419; GFX12W64: ; %bb.0: ; %entry1420; GFX12W64-NEXT: s_load_b128 s[0:3], s[4:5], 0x341421; GFX12W64-NEXT: v_and_b32_e32 v0, 0x3ff, v01422; GFX12W64-NEXT: v_mov_b32_e32 v1, 11423; GFX12W64-NEXT: s_wait_kmcnt 0x01424; GFX12W64-NEXT: buffer_atomic_add_u32 v1, v0, s[0:3], null offen th:TH_ATOMIC_RETURN1425; GFX12W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x241426; GFX12W64-NEXT: v_mov_b32_e32 v0, 01427; GFX12W64-NEXT: s_wait_loadcnt 0x01428; GFX12W64-NEXT: s_wait_kmcnt 0x01429; GFX12W64-NEXT: global_store_b32 v0, v1, s[0:1]1430; GFX12W64-NEXT: s_endpgm1431;1432; GFX12W32-LABEL: add_i32_varying_offset:1433; GFX12W32: ; %bb.0: ; %entry1434; GFX12W32-NEXT: s_load_b128 s[0:3], s[4:5], 0x341435; GFX12W32-NEXT: v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v01436; GFX12W32-NEXT: s_wait_kmcnt 0x01437; GFX12W32-NEXT: buffer_atomic_add_u32 v1, v0, s[0:3], null offen th:TH_ATOMIC_RETURN1438; GFX12W32-NEXT: s_load_b64 s[0:1], s[4:5], 0x241439; GFX12W32-NEXT: v_mov_b32_e32 v0, 01440; GFX12W32-NEXT: s_wait_loadcnt 0x01441; GFX12W32-NEXT: s_wait_kmcnt 0x01442; GFX12W32-NEXT: global_store_b32 v0, v1, s[0:1]1443; GFX12W32-NEXT: s_endpgm1444entry:1445 %lane = call i32 @llvm.amdgcn.workitem.id.x()1446 %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32 1, ptr addrspace(8) %inout, i32 %lane, i32 0, i32 0)1447 store i32 %old, ptr addrspace(1) %out1448 ret void1449}1450 1451define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace(8) %inout) {1452; GFX6-LABEL: sub_i32_constant:1453; GFX6: ; %bb.0: ; %entry1454; GFX6-NEXT: s_mov_b64 s[2:3], exec1455; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 01456; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v01457; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v01458; GFX6-NEXT: ; implicit-def: $vgpr11459; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc1460; GFX6-NEXT: s_cbranch_execz .LBB5_21461; GFX6-NEXT: ; %bb.1:1462; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd1463; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]1464; GFX6-NEXT: s_mul_i32 s2, s2, 51465; GFX6-NEXT: v_mov_b32_e32 v1, s21466; GFX6-NEXT: s_waitcnt lgkmcnt(0)1467; GFX6-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc1468; GFX6-NEXT: .LBB5_2:1469; GFX6-NEXT: s_or_b64 exec, exec, s[0:1]1470; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x91471; GFX6-NEXT: s_mov_b32 s3, 0xf0001472; GFX6-NEXT: s_mov_b32 s2, -11473; GFX6-NEXT: s_waitcnt vmcnt(0)1474; GFX6-NEXT: v_readfirstlane_b32 s4, v11475; GFX6-NEXT: v_mul_u32_u24_e32 v0, 5, v01476; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s4, v01477; GFX6-NEXT: s_waitcnt lgkmcnt(0)1478; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 01479; GFX6-NEXT: s_endpgm1480;1481; GFX8-LABEL: sub_i32_constant:1482; GFX8: ; %bb.0: ; %entry1483; GFX8-NEXT: s_mov_b64 s[2:3], exec1484; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 01485; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v01486; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v01487; GFX8-NEXT: ; implicit-def: $vgpr11488; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc1489; GFX8-NEXT: s_cbranch_execz .LBB5_21490; GFX8-NEXT: ; %bb.1:1491; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x341492; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]1493; GFX8-NEXT: s_mul_i32 s2, s2, 51494; GFX8-NEXT: v_mov_b32_e32 v1, s21495; GFX8-NEXT: s_waitcnt lgkmcnt(0)1496; GFX8-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc1497; GFX8-NEXT: .LBB5_2:1498; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]1499; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241500; GFX8-NEXT: s_waitcnt vmcnt(0)1501; GFX8-NEXT: v_readfirstlane_b32 s2, v11502; GFX8-NEXT: v_mul_u32_u24_e32 v0, 5, v01503; GFX8-NEXT: v_sub_u32_e32 v2, vcc, s2, v01504; GFX8-NEXT: s_waitcnt lgkmcnt(0)1505; GFX8-NEXT: v_mov_b32_e32 v0, s01506; GFX8-NEXT: v_mov_b32_e32 v1, s11507; GFX8-NEXT: flat_store_dword v[0:1], v21508; GFX8-NEXT: s_endpgm1509;1510; GFX9-LABEL: sub_i32_constant:1511; GFX9: ; %bb.0: ; %entry1512; GFX9-NEXT: s_mov_b64 s[2:3], exec1513; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 01514; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v01515; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v01516; GFX9-NEXT: ; implicit-def: $vgpr11517; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc1518; GFX9-NEXT: s_cbranch_execz .LBB5_21519; GFX9-NEXT: ; %bb.1:1520; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x341521; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]1522; GFX9-NEXT: s_mul_i32 s2, s2, 51523; GFX9-NEXT: v_mov_b32_e32 v1, s21524; GFX9-NEXT: s_waitcnt lgkmcnt(0)1525; GFX9-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc1526; GFX9-NEXT: .LBB5_2:1527; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]1528; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241529; GFX9-NEXT: s_waitcnt vmcnt(0)1530; GFX9-NEXT: v_readfirstlane_b32 s2, v11531; GFX9-NEXT: v_mul_u32_u24_e32 v0, 5, v01532; GFX9-NEXT: v_mov_b32_e32 v2, 01533; GFX9-NEXT: v_sub_u32_e32 v0, s2, v01534; GFX9-NEXT: s_waitcnt lgkmcnt(0)1535; GFX9-NEXT: global_store_dword v2, v0, s[0:1]1536; GFX9-NEXT: s_endpgm1537;1538; GFX10W64-LABEL: sub_i32_constant:1539; GFX10W64: ; %bb.0: ; %entry1540; GFX10W64-NEXT: s_mov_b64 s[2:3], exec1541; GFX10W64-NEXT: ; implicit-def: $vgpr11542; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 01543; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v01544; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v01545; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc1546; GFX10W64-NEXT: s_cbranch_execz .LBB5_21547; GFX10W64-NEXT: ; %bb.1:1548; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x341549; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]1550; GFX10W64-NEXT: s_mul_i32 s2, s2, 51551; GFX10W64-NEXT: v_mov_b32_e32 v1, s21552; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)1553; GFX10W64-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc1554; GFX10W64-NEXT: .LBB5_2:1555; GFX10W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1556; GFX10W64-NEXT: s_or_b64 exec, exec, s[0:1]1557; GFX10W64-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241558; GFX10W64-NEXT: s_waitcnt vmcnt(0)1559; GFX10W64-NEXT: v_readfirstlane_b32 s2, v11560; GFX10W64-NEXT: v_mul_u32_u24_e32 v0, 5, v01561; GFX10W64-NEXT: v_mov_b32_e32 v1, 01562; GFX10W64-NEXT: v_sub_nc_u32_e32 v0, s2, v01563; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)1564; GFX10W64-NEXT: global_store_dword v1, v0, s[0:1]1565; GFX10W64-NEXT: s_endpgm1566;1567; GFX10W32-LABEL: sub_i32_constant:1568; GFX10W32: ; %bb.0: ; %entry1569; GFX10W32-NEXT: s_mov_b32 s1, exec_lo1570; GFX10W32-NEXT: ; implicit-def: $vgpr11571; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 01572; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v01573; GFX10W32-NEXT: s_and_saveexec_b32 s0, vcc_lo1574; GFX10W32-NEXT: s_cbranch_execz .LBB5_21575; GFX10W32-NEXT: ; %bb.1:1576; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x341577; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s11578; GFX10W32-NEXT: s_mul_i32 s1, s1, 51579; GFX10W32-NEXT: v_mov_b32_e32 v1, s11580; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)1581; GFX10W32-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc1582; GFX10W32-NEXT: .LBB5_2:1583; GFX10W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1584; GFX10W32-NEXT: s_or_b32 exec_lo, exec_lo, s01585; GFX10W32-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241586; GFX10W32-NEXT: s_waitcnt vmcnt(0)1587; GFX10W32-NEXT: v_readfirstlane_b32 s2, v11588; GFX10W32-NEXT: v_mul_u32_u24_e32 v0, 5, v01589; GFX10W32-NEXT: v_mov_b32_e32 v1, 01590; GFX10W32-NEXT: v_sub_nc_u32_e32 v0, s2, v01591; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)1592; GFX10W32-NEXT: global_store_dword v1, v0, s[0:1]1593; GFX10W32-NEXT: s_endpgm1594;1595; GFX11W64-LABEL: sub_i32_constant:1596; GFX11W64: ; %bb.0: ; %entry1597; GFX11W64-NEXT: s_mov_b64 s[2:3], exec1598; GFX11W64-NEXT: s_mov_b64 s[0:1], exec1599; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 01600; GFX11W64-NEXT: ; implicit-def: $vgpr11601; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1602; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v01603; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v01604; GFX11W64-NEXT: s_cbranch_execz .LBB5_21605; GFX11W64-NEXT: ; %bb.1:1606; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x341607; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]1608; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)1609; GFX11W64-NEXT: s_mul_i32 s2, s2, 51610; GFX11W64-NEXT: v_mov_b32_e32 v1, s21611; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)1612; GFX11W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc1613; GFX11W64-NEXT: .LBB5_2:1614; GFX11W64-NEXT: s_or_b64 exec, exec, s[0:1]1615; GFX11W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x241616; GFX11W64-NEXT: s_waitcnt vmcnt(0)1617; GFX11W64-NEXT: v_readfirstlane_b32 s2, v11618; GFX11W64-NEXT: v_mul_u32_u24_e32 v0, 5, v01619; GFX11W64-NEXT: v_mov_b32_e32 v1, 01620; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_2)1621; GFX11W64-NEXT: v_sub_nc_u32_e32 v0, s2, v01622; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)1623; GFX11W64-NEXT: global_store_b32 v1, v0, s[0:1]1624; GFX11W64-NEXT: s_endpgm1625;1626; GFX11W32-LABEL: sub_i32_constant:1627; GFX11W32: ; %bb.0: ; %entry1628; GFX11W32-NEXT: s_mov_b32 s1, exec_lo1629; GFX11W32-NEXT: s_mov_b32 s0, exec_lo1630; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 01631; GFX11W32-NEXT: ; implicit-def: $vgpr11632; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)1633; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v01634; GFX11W32-NEXT: s_cbranch_execz .LBB5_21635; GFX11W32-NEXT: ; %bb.1:1636; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x341637; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s11638; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)1639; GFX11W32-NEXT: s_mul_i32 s1, s1, 51640; GFX11W32-NEXT: v_mov_b32_e32 v1, s11641; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)1642; GFX11W32-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc1643; GFX11W32-NEXT: .LBB5_2:1644; GFX11W32-NEXT: s_or_b32 exec_lo, exec_lo, s01645; GFX11W32-NEXT: s_load_b64 s[0:1], s[4:5], 0x241646; GFX11W32-NEXT: s_waitcnt vmcnt(0)1647; GFX11W32-NEXT: v_readfirstlane_b32 s2, v11648; GFX11W32-NEXT: v_mul_u32_u24_e32 v0, 5, v01649; GFX11W32-NEXT: v_mov_b32_e32 v1, 01650; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_2)1651; GFX11W32-NEXT: v_sub_nc_u32_e32 v0, s2, v01652; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)1653; GFX11W32-NEXT: global_store_b32 v1, v0, s[0:1]1654; GFX11W32-NEXT: s_endpgm1655;1656; GFX12W64-LABEL: sub_i32_constant:1657; GFX12W64: ; %bb.0: ; %entry1658; GFX12W64-NEXT: s_mov_b64 s[2:3], exec1659; GFX12W64-NEXT: s_mov_b64 s[0:1], exec1660; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 01661; GFX12W64-NEXT: ; implicit-def: $vgpr11662; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1663; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v01664; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v01665; GFX12W64-NEXT: s_cbranch_execz .LBB5_21666; GFX12W64-NEXT: ; %bb.1:1667; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x341668; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]1669; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)1670; GFX12W64-NEXT: s_mul_i32 s2, s2, 51671; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)1672; GFX12W64-NEXT: v_mov_b32_e32 v1, s21673; GFX12W64-NEXT: s_wait_kmcnt 0x01674; GFX12W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN1675; GFX12W64-NEXT: .LBB5_2:1676; GFX12W64-NEXT: s_or_b64 exec, exec, s[0:1]1677; GFX12W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x241678; GFX12W64-NEXT: s_wait_loadcnt 0x01679; GFX12W64-NEXT: v_readfirstlane_b32 s2, v11680; GFX12W64-NEXT: v_mul_u32_u24_e32 v0, 5, v01681; GFX12W64-NEXT: v_mov_b32_e32 v1, 01682; GFX12W64-NEXT: s_wait_alu depctr_va_sdst(0)1683; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_2)1684; GFX12W64-NEXT: v_sub_nc_u32_e32 v0, s2, v01685; GFX12W64-NEXT: s_wait_kmcnt 0x01686; GFX12W64-NEXT: global_store_b32 v1, v0, s[0:1]1687; GFX12W64-NEXT: s_endpgm1688;1689; GFX12W32-LABEL: sub_i32_constant:1690; GFX12W32: ; %bb.0: ; %entry1691; GFX12W32-NEXT: s_mov_b32 s1, exec_lo1692; GFX12W32-NEXT: s_mov_b32 s0, exec_lo1693; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 01694; GFX12W32-NEXT: ; implicit-def: $vgpr11695; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)1696; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v01697; GFX12W32-NEXT: s_cbranch_execz .LBB5_21698; GFX12W32-NEXT: ; %bb.1:1699; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x341700; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s11701; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)1702; GFX12W32-NEXT: s_mul_i32 s1, s1, 51703; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)1704; GFX12W32-NEXT: v_mov_b32_e32 v1, s11705; GFX12W32-NEXT: s_wait_kmcnt 0x01706; GFX12W32-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN1707; GFX12W32-NEXT: .LBB5_2:1708; GFX12W32-NEXT: s_or_b32 exec_lo, exec_lo, s01709; GFX12W32-NEXT: s_load_b64 s[0:1], s[4:5], 0x241710; GFX12W32-NEXT: s_wait_loadcnt 0x01711; GFX12W32-NEXT: v_readfirstlane_b32 s2, v11712; GFX12W32-NEXT: v_mul_u32_u24_e32 v0, 5, v01713; GFX12W32-NEXT: v_mov_b32_e32 v1, 01714; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_2)1715; GFX12W32-NEXT: v_sub_nc_u32_e32 v0, s2, v01716; GFX12W32-NEXT: s_wait_kmcnt 0x01717; GFX12W32-NEXT: global_store_b32 v1, v0, s[0:1]1718; GFX12W32-NEXT: s_endpgm1719entry:1720 %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.sub(i32 5, ptr addrspace(8) %inout, i32 0, i32 0, i32 0)1721 store i32 %old, ptr addrspace(1) %out1722 ret void1723}1724 1725define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(8) %inout, i32 %subitive) {1726; GFX6-LABEL: sub_i32_uniform:1727; GFX6: ; %bb.0: ; %entry1728; GFX6-NEXT: s_mov_b64 s[2:3], exec1729; GFX6-NEXT: s_load_dword s6, s[4:5], 0x111730; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 01731; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v01732; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v01733; GFX6-NEXT: ; implicit-def: $vgpr11734; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc1735; GFX6-NEXT: s_cbranch_execz .LBB6_21736; GFX6-NEXT: ; %bb.1:1737; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd1738; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]1739; GFX6-NEXT: s_waitcnt lgkmcnt(0)1740; GFX6-NEXT: s_mul_i32 s2, s6, s21741; GFX6-NEXT: v_mov_b32_e32 v1, s21742; GFX6-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc1743; GFX6-NEXT: .LBB6_2:1744; GFX6-NEXT: s_or_b64 exec, exec, s[0:1]1745; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x91746; GFX6-NEXT: s_mov_b32 s3, 0xf0001747; GFX6-NEXT: s_mov_b32 s2, -11748; GFX6-NEXT: s_waitcnt vmcnt(0)1749; GFX6-NEXT: v_readfirstlane_b32 s4, v11750; GFX6-NEXT: s_waitcnt lgkmcnt(0)1751; GFX6-NEXT: v_mul_lo_u32 v0, s6, v01752; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s4, v01753; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 01754; GFX6-NEXT: s_endpgm1755;1756; GFX8-LABEL: sub_i32_uniform:1757; GFX8: ; %bb.0: ; %entry1758; GFX8-NEXT: s_load_dword s6, s[4:5], 0x441759; GFX8-NEXT: s_mov_b64 s[2:3], exec1760; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 01761; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v01762; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v01763; GFX8-NEXT: ; implicit-def: $vgpr11764; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc1765; GFX8-NEXT: s_cbranch_execz .LBB6_21766; GFX8-NEXT: ; %bb.1:1767; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x341768; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]1769; GFX8-NEXT: s_waitcnt lgkmcnt(0)1770; GFX8-NEXT: s_mul_i32 s2, s6, s21771; GFX8-NEXT: v_mov_b32_e32 v1, s21772; GFX8-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc1773; GFX8-NEXT: .LBB6_2:1774; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]1775; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241776; GFX8-NEXT: s_waitcnt lgkmcnt(0)1777; GFX8-NEXT: v_mul_lo_u32 v0, s6, v01778; GFX8-NEXT: s_waitcnt vmcnt(0)1779; GFX8-NEXT: v_readfirstlane_b32 s2, v11780; GFX8-NEXT: v_sub_u32_e32 v2, vcc, s2, v01781; GFX8-NEXT: v_mov_b32_e32 v0, s01782; GFX8-NEXT: v_mov_b32_e32 v1, s11783; GFX8-NEXT: flat_store_dword v[0:1], v21784; GFX8-NEXT: s_endpgm1785;1786; GFX9-LABEL: sub_i32_uniform:1787; GFX9: ; %bb.0: ; %entry1788; GFX9-NEXT: s_load_dword s6, s[4:5], 0x441789; GFX9-NEXT: s_mov_b64 s[2:3], exec1790; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 01791; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v01792; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v01793; GFX9-NEXT: ; implicit-def: $vgpr11794; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc1795; GFX9-NEXT: s_cbranch_execz .LBB6_21796; GFX9-NEXT: ; %bb.1:1797; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x341798; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]1799; GFX9-NEXT: s_waitcnt lgkmcnt(0)1800; GFX9-NEXT: s_mul_i32 s2, s6, s21801; GFX9-NEXT: v_mov_b32_e32 v1, s21802; GFX9-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc1803; GFX9-NEXT: .LBB6_2:1804; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]1805; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241806; GFX9-NEXT: s_waitcnt lgkmcnt(0)1807; GFX9-NEXT: v_mul_lo_u32 v0, s6, v01808; GFX9-NEXT: s_waitcnt vmcnt(0)1809; GFX9-NEXT: v_readfirstlane_b32 s2, v11810; GFX9-NEXT: v_mov_b32_e32 v2, 01811; GFX9-NEXT: v_sub_u32_e32 v0, s2, v01812; GFX9-NEXT: global_store_dword v2, v0, s[0:1]1813; GFX9-NEXT: s_endpgm1814;1815; GFX10W64-LABEL: sub_i32_uniform:1816; GFX10W64: ; %bb.0: ; %entry1817; GFX10W64-NEXT: s_load_dword s6, s[4:5], 0x441818; GFX10W64-NEXT: s_mov_b64 s[2:3], exec1819; GFX10W64-NEXT: ; implicit-def: $vgpr11820; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 01821; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v01822; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v01823; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc1824; GFX10W64-NEXT: s_cbranch_execz .LBB6_21825; GFX10W64-NEXT: ; %bb.1:1826; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x341827; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]1828; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)1829; GFX10W64-NEXT: s_mul_i32 s2, s6, s21830; GFX10W64-NEXT: v_mov_b32_e32 v1, s21831; GFX10W64-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc1832; GFX10W64-NEXT: .LBB6_2:1833; GFX10W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1834; GFX10W64-NEXT: s_or_b64 exec, exec, s[0:1]1835; GFX10W64-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241836; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)1837; GFX10W64-NEXT: v_mul_lo_u32 v0, s6, v01838; GFX10W64-NEXT: s_waitcnt vmcnt(0)1839; GFX10W64-NEXT: v_readfirstlane_b32 s2, v11840; GFX10W64-NEXT: v_mov_b32_e32 v1, 01841; GFX10W64-NEXT: v_sub_nc_u32_e32 v0, s2, v01842; GFX10W64-NEXT: global_store_dword v1, v0, s[0:1]1843; GFX10W64-NEXT: s_endpgm1844;1845; GFX10W32-LABEL: sub_i32_uniform:1846; GFX10W32: ; %bb.0: ; %entry1847; GFX10W32-NEXT: s_load_dword s0, s[4:5], 0x441848; GFX10W32-NEXT: s_mov_b32 s2, exec_lo1849; GFX10W32-NEXT: ; implicit-def: $vgpr11850; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 01851; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v01852; GFX10W32-NEXT: s_and_saveexec_b32 s1, vcc_lo1853; GFX10W32-NEXT: s_cbranch_execz .LBB6_21854; GFX10W32-NEXT: ; %bb.1:1855; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x341856; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s21857; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)1858; GFX10W32-NEXT: s_mul_i32 s2, s0, s21859; GFX10W32-NEXT: v_mov_b32_e32 v1, s21860; GFX10W32-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc1861; GFX10W32-NEXT: .LBB6_2:1862; GFX10W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1863; GFX10W32-NEXT: s_or_b32 exec_lo, exec_lo, s11864; GFX10W32-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x241865; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)1866; GFX10W32-NEXT: v_mul_lo_u32 v0, s0, v01867; GFX10W32-NEXT: s_waitcnt vmcnt(0)1868; GFX10W32-NEXT: v_readfirstlane_b32 s0, v11869; GFX10W32-NEXT: v_mov_b32_e32 v1, 01870; GFX10W32-NEXT: v_sub_nc_u32_e32 v0, s0, v01871; GFX10W32-NEXT: global_store_dword v1, v0, s[2:3]1872; GFX10W32-NEXT: s_endpgm1873;1874; GFX11W64-LABEL: sub_i32_uniform:1875; GFX11W64: ; %bb.0: ; %entry1876; GFX11W64-NEXT: s_load_b32 s6, s[4:5], 0x441877; GFX11W64-NEXT: s_mov_b64 s[2:3], exec1878; GFX11W64-NEXT: s_mov_b64 s[0:1], exec1879; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 01880; GFX11W64-NEXT: ; implicit-def: $vgpr11881; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1882; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v01883; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v01884; GFX11W64-NEXT: s_cbranch_execz .LBB6_21885; GFX11W64-NEXT: ; %bb.1:1886; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x341887; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]1888; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)1889; GFX11W64-NEXT: s_mul_i32 s2, s6, s21890; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1891; GFX11W64-NEXT: v_mov_b32_e32 v1, s21892; GFX11W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc1893; GFX11W64-NEXT: .LBB6_2:1894; GFX11W64-NEXT: s_or_b64 exec, exec, s[0:1]1895; GFX11W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x241896; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)1897; GFX11W64-NEXT: v_mul_lo_u32 v0, s6, v01898; GFX11W64-NEXT: s_waitcnt vmcnt(0)1899; GFX11W64-NEXT: v_readfirstlane_b32 s2, v11900; GFX11W64-NEXT: v_mov_b32_e32 v1, 01901; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_2)1902; GFX11W64-NEXT: v_sub_nc_u32_e32 v0, s2, v01903; GFX11W64-NEXT: global_store_b32 v1, v0, s[0:1]1904; GFX11W64-NEXT: s_endpgm1905;1906; GFX11W32-LABEL: sub_i32_uniform:1907; GFX11W32: ; %bb.0: ; %entry1908; GFX11W32-NEXT: s_load_b32 s0, s[4:5], 0x441909; GFX11W32-NEXT: s_mov_b32 s2, exec_lo1910; GFX11W32-NEXT: s_mov_b32 s1, exec_lo1911; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 01912; GFX11W32-NEXT: ; implicit-def: $vgpr11913; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)1914; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v01915; GFX11W32-NEXT: s_cbranch_execz .LBB6_21916; GFX11W32-NEXT: ; %bb.1:1917; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x341918; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s21919; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)1920; GFX11W32-NEXT: s_mul_i32 s2, s0, s21921; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1922; GFX11W32-NEXT: v_mov_b32_e32 v1, s21923; GFX11W32-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc1924; GFX11W32-NEXT: .LBB6_2:1925; GFX11W32-NEXT: s_or_b32 exec_lo, exec_lo, s11926; GFX11W32-NEXT: s_load_b64 s[2:3], s[4:5], 0x241927; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)1928; GFX11W32-NEXT: v_mul_lo_u32 v0, s0, v01929; GFX11W32-NEXT: s_waitcnt vmcnt(0)1930; GFX11W32-NEXT: v_readfirstlane_b32 s0, v11931; GFX11W32-NEXT: v_mov_b32_e32 v1, 01932; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_2)1933; GFX11W32-NEXT: v_sub_nc_u32_e32 v0, s0, v01934; GFX11W32-NEXT: global_store_b32 v1, v0, s[2:3]1935; GFX11W32-NEXT: s_endpgm1936;1937; GFX12W64-LABEL: sub_i32_uniform:1938; GFX12W64: ; %bb.0: ; %entry1939; GFX12W64-NEXT: s_load_b32 s6, s[4:5], 0x441940; GFX12W64-NEXT: s_mov_b64 s[2:3], exec1941; GFX12W64-NEXT: s_mov_b64 s[0:1], exec1942; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 01943; GFX12W64-NEXT: ; implicit-def: $vgpr11944; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1945; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v01946; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v01947; GFX12W64-NEXT: s_cbranch_execz .LBB6_21948; GFX12W64-NEXT: ; %bb.1:1949; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x341950; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]1951; GFX12W64-NEXT: s_wait_kmcnt 0x01952; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)1953; GFX12W64-NEXT: s_mul_i32 s2, s6, s21954; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)1955; GFX12W64-NEXT: v_mov_b32_e32 v1, s21956; GFX12W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN1957; GFX12W64-NEXT: .LBB6_2:1958; GFX12W64-NEXT: s_or_b64 exec, exec, s[0:1]1959; GFX12W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x241960; GFX12W64-NEXT: s_wait_kmcnt 0x01961; GFX12W64-NEXT: v_mul_lo_u32 v0, s6, v01962; GFX12W64-NEXT: s_wait_loadcnt 0x01963; GFX12W64-NEXT: v_readfirstlane_b32 s2, v11964; GFX12W64-NEXT: v_mov_b32_e32 v1, 01965; GFX12W64-NEXT: s_wait_alu depctr_va_sdst(0)1966; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_2)1967; GFX12W64-NEXT: v_sub_nc_u32_e32 v0, s2, v01968; GFX12W64-NEXT: global_store_b32 v1, v0, s[0:1]1969; GFX12W64-NEXT: s_endpgm1970;1971; GFX12W32-LABEL: sub_i32_uniform:1972; GFX12W32: ; %bb.0: ; %entry1973; GFX12W32-NEXT: s_load_b32 s0, s[4:5], 0x441974; GFX12W32-NEXT: s_mov_b32 s2, exec_lo1975; GFX12W32-NEXT: s_mov_b32 s1, exec_lo1976; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 01977; GFX12W32-NEXT: ; implicit-def: $vgpr11978; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)1979; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v01980; GFX12W32-NEXT: s_cbranch_execz .LBB6_21981; GFX12W32-NEXT: ; %bb.1:1982; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x341983; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s21984; GFX12W32-NEXT: s_wait_kmcnt 0x01985; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)1986; GFX12W32-NEXT: s_mul_i32 s2, s0, s21987; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)1988; GFX12W32-NEXT: v_mov_b32_e32 v1, s21989; GFX12W32-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN1990; GFX12W32-NEXT: .LBB6_2:1991; GFX12W32-NEXT: s_or_b32 exec_lo, exec_lo, s11992; GFX12W32-NEXT: s_load_b64 s[2:3], s[4:5], 0x241993; GFX12W32-NEXT: s_wait_kmcnt 0x01994; GFX12W32-NEXT: v_mul_lo_u32 v0, s0, v01995; GFX12W32-NEXT: s_wait_loadcnt 0x01996; GFX12W32-NEXT: v_readfirstlane_b32 s0, v11997; GFX12W32-NEXT: v_mov_b32_e32 v1, 01998; GFX12W32-NEXT: s_wait_alu depctr_va_sdst(0)1999; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_2)2000; GFX12W32-NEXT: v_sub_nc_u32_e32 v0, s0, v02001; GFX12W32-NEXT: global_store_b32 v1, v0, s[2:3]2002; GFX12W32-NEXT: s_endpgm2003entry:2004 %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.sub(i32 %subitive, ptr addrspace(8) %inout, i32 0, i32 0, i32 0)2005 store i32 %old, ptr addrspace(1) %out2006 ret void2007}2008 2009define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addrspace(8) %inout) {2010; GFX6-LABEL: sub_i32_varying_vdata:2011; GFX6: ; %bb.0: ; %entry2012; GFX6-NEXT: s_mov_b64 s[0:1], exec2013; GFX6-NEXT: s_mov_b32 s2, 02014; GFX6-NEXT: ; implicit-def: $vgpr12015; GFX6-NEXT: .LBB7_1: ; %ComputeLoop2016; GFX6-NEXT: ; =>This Inner Loop Header: Depth=12017; GFX6-NEXT: s_ff1_i32_b64 s3, s[0:1]2018; GFX6-NEXT: s_mov_b32 m0, s32019; GFX6-NEXT: v_readlane_b32 s8, v0, s32020; GFX6-NEXT: v_writelane_b32 v1, s2, m02021; GFX6-NEXT: s_lshl_b64 s[6:7], 1, s32022; GFX6-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]2023; GFX6-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 02024; GFX6-NEXT: s_and_b64 vcc, exec, s[6:7]2025; GFX6-NEXT: s_add_i32 s2, s2, s82026; GFX6-NEXT: s_cbranch_vccnz .LBB7_12027; GFX6-NEXT: ; %bb.2: ; %ComputeEnd2028; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 02029; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v02030; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v02031; GFX6-NEXT: ; implicit-def: $vgpr02032; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc2033; GFX6-NEXT: s_xor_b64 s[0:1], exec, s[0:1]2034; GFX6-NEXT: s_cbranch_execz .LBB7_42035; GFX6-NEXT: ; %bb.3:2036; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd2037; GFX6-NEXT: v_mov_b32_e32 v0, s22038; GFX6-NEXT: s_waitcnt lgkmcnt(0)2039; GFX6-NEXT: buffer_atomic_sub v0, off, s[8:11], 0 glc2040; GFX6-NEXT: .LBB7_4:2041; GFX6-NEXT: s_or_b64 exec, exec, s[0:1]2042; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x92043; GFX6-NEXT: s_mov_b32 s3, 0xf0002044; GFX6-NEXT: s_mov_b32 s2, -12045; GFX6-NEXT: s_waitcnt vmcnt(0)2046; GFX6-NEXT: v_readfirstlane_b32 s4, v02047; GFX6-NEXT: s_waitcnt expcnt(0)2048; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s4, v12049; GFX6-NEXT: s_waitcnt lgkmcnt(0)2050; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 02051; GFX6-NEXT: s_endpgm2052;2053; GFX8-LABEL: sub_i32_varying_vdata:2054; GFX8: ; %bb.0: ; %entry2055; GFX8-NEXT: s_mov_b64 s[0:1], exec2056; GFX8-NEXT: s_mov_b32 s2, 02057; GFX8-NEXT: ; implicit-def: $vgpr12058; GFX8-NEXT: .LBB7_1: ; %ComputeLoop2059; GFX8-NEXT: ; =>This Inner Loop Header: Depth=12060; GFX8-NEXT: s_ff1_i32_b64 s3, s[0:1]2061; GFX8-NEXT: s_mov_b32 m0, s32062; GFX8-NEXT: v_readlane_b32 s6, v0, s32063; GFX8-NEXT: v_writelane_b32 v1, s2, m02064; GFX8-NEXT: s_add_i32 s2, s2, s62065; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s32066; GFX8-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]2067; GFX8-NEXT: s_cbranch_scc1 .LBB7_12068; GFX8-NEXT: ; %bb.2: ; %ComputeEnd2069; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 02070; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v02071; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v02072; GFX8-NEXT: ; implicit-def: $vgpr02073; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc2074; GFX8-NEXT: s_xor_b64 s[0:1], exec, s[0:1]2075; GFX8-NEXT: s_cbranch_execz .LBB7_42076; GFX8-NEXT: ; %bb.3:2077; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x342078; GFX8-NEXT: v_mov_b32_e32 v0, s22079; GFX8-NEXT: s_waitcnt lgkmcnt(0)2080; GFX8-NEXT: buffer_atomic_sub v0, off, s[8:11], 0 glc2081; GFX8-NEXT: .LBB7_4:2082; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]2083; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x242084; GFX8-NEXT: s_waitcnt vmcnt(0)2085; GFX8-NEXT: v_readfirstlane_b32 s2, v02086; GFX8-NEXT: v_sub_u32_e32 v2, vcc, s2, v12087; GFX8-NEXT: s_waitcnt lgkmcnt(0)2088; GFX8-NEXT: v_mov_b32_e32 v0, s02089; GFX8-NEXT: v_mov_b32_e32 v1, s12090; GFX8-NEXT: flat_store_dword v[0:1], v22091; GFX8-NEXT: s_endpgm2092;2093; GFX9-LABEL: sub_i32_varying_vdata:2094; GFX9: ; %bb.0: ; %entry2095; GFX9-NEXT: s_mov_b64 s[0:1], exec2096; GFX9-NEXT: s_mov_b32 s2, 02097; GFX9-NEXT: ; implicit-def: $vgpr12098; GFX9-NEXT: .LBB7_1: ; %ComputeLoop2099; GFX9-NEXT: ; =>This Inner Loop Header: Depth=12100; GFX9-NEXT: s_ff1_i32_b64 s3, s[0:1]2101; GFX9-NEXT: s_mov_b32 m0, s32102; GFX9-NEXT: v_readlane_b32 s6, v0, s32103; GFX9-NEXT: v_writelane_b32 v1, s2, m02104; GFX9-NEXT: s_add_i32 s2, s2, s62105; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s32106; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]2107; GFX9-NEXT: s_cbranch_scc1 .LBB7_12108; GFX9-NEXT: ; %bb.2: ; %ComputeEnd2109; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 02110; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v02111; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v02112; GFX9-NEXT: ; implicit-def: $vgpr02113; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc2114; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]2115; GFX9-NEXT: s_cbranch_execz .LBB7_42116; GFX9-NEXT: ; %bb.3:2117; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x342118; GFX9-NEXT: v_mov_b32_e32 v0, s22119; GFX9-NEXT: s_waitcnt lgkmcnt(0)2120; GFX9-NEXT: buffer_atomic_sub v0, off, s[8:11], 0 glc2121; GFX9-NEXT: .LBB7_4:2122; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]2123; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x242124; GFX9-NEXT: s_waitcnt vmcnt(0)2125; GFX9-NEXT: v_readfirstlane_b32 s2, v02126; GFX9-NEXT: v_mov_b32_e32 v2, 02127; GFX9-NEXT: v_sub_u32_e32 v0, s2, v12128; GFX9-NEXT: s_waitcnt lgkmcnt(0)2129; GFX9-NEXT: global_store_dword v2, v0, s[0:1]2130; GFX9-NEXT: s_endpgm2131;2132; GFX10W64-LABEL: sub_i32_varying_vdata:2133; GFX10W64: ; %bb.0: ; %entry2134; GFX10W64-NEXT: s_mov_b64 s[0:1], exec2135; GFX10W64-NEXT: s_mov_b32 s2, 02136; GFX10W64-NEXT: ; implicit-def: $vgpr12137; GFX10W64-NEXT: .LBB7_1: ; %ComputeLoop2138; GFX10W64-NEXT: ; =>This Inner Loop Header: Depth=12139; GFX10W64-NEXT: s_ff1_i32_b64 s3, s[0:1]2140; GFX10W64-NEXT: v_readlane_b32 s8, v0, s32141; GFX10W64-NEXT: v_writelane_b32 v1, s2, s32142; GFX10W64-NEXT: s_lshl_b64 s[6:7], 1, s32143; GFX10W64-NEXT: s_add_i32 s2, s2, s82144; GFX10W64-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]2145; GFX10W64-NEXT: s_cbranch_scc1 .LBB7_12146; GFX10W64-NEXT: ; %bb.2: ; %ComputeEnd2147; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 02148; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v02149; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v02150; GFX10W64-NEXT: ; implicit-def: $vgpr02151; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc2152; GFX10W64-NEXT: s_xor_b64 s[0:1], exec, s[0:1]2153; GFX10W64-NEXT: s_cbranch_execz .LBB7_42154; GFX10W64-NEXT: ; %bb.3:2155; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x342156; GFX10W64-NEXT: v_mov_b32_e32 v0, s22157; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)2158; GFX10W64-NEXT: buffer_atomic_sub v0, off, s[8:11], 0 glc2159; GFX10W64-NEXT: .LBB7_4:2160; GFX10W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)2161; GFX10W64-NEXT: s_or_b64 exec, exec, s[0:1]2162; GFX10W64-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x242163; GFX10W64-NEXT: s_waitcnt vmcnt(0)2164; GFX10W64-NEXT: v_readfirstlane_b32 s2, v02165; GFX10W64-NEXT: v_mov_b32_e32 v0, 02166; GFX10W64-NEXT: v_sub_nc_u32_e32 v1, s2, v12167; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)2168; GFX10W64-NEXT: global_store_dword v0, v1, s[0:1]2169; GFX10W64-NEXT: s_endpgm2170;2171; GFX10W32-LABEL: sub_i32_varying_vdata:2172; GFX10W32: ; %bb.0: ; %entry2173; GFX10W32-NEXT: s_mov_b32 s1, exec_lo2174; GFX10W32-NEXT: s_mov_b32 s0, 02175; GFX10W32-NEXT: ; implicit-def: $vgpr12176; GFX10W32-NEXT: .LBB7_1: ; %ComputeLoop2177; GFX10W32-NEXT: ; =>This Inner Loop Header: Depth=12178; GFX10W32-NEXT: s_ff1_i32_b32 s2, s12179; GFX10W32-NEXT: v_readlane_b32 s3, v0, s22180; GFX10W32-NEXT: v_writelane_b32 v1, s0, s22181; GFX10W32-NEXT: s_lshl_b32 s2, 1, s22182; GFX10W32-NEXT: s_add_i32 s0, s0, s32183; GFX10W32-NEXT: s_andn2_b32 s1, s1, s22184; GFX10W32-NEXT: s_cbranch_scc1 .LBB7_12185; GFX10W32-NEXT: ; %bb.2: ; %ComputeEnd2186; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 02187; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v02188; GFX10W32-NEXT: ; implicit-def: $vgpr02189; GFX10W32-NEXT: s_and_saveexec_b32 s1, vcc_lo2190; GFX10W32-NEXT: s_xor_b32 s1, exec_lo, s12191; GFX10W32-NEXT: s_cbranch_execz .LBB7_42192; GFX10W32-NEXT: ; %bb.3:2193; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x342194; GFX10W32-NEXT: v_mov_b32_e32 v0, s02195; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)2196; GFX10W32-NEXT: buffer_atomic_sub v0, off, s[8:11], 0 glc2197; GFX10W32-NEXT: .LBB7_4:2198; GFX10W32-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)2199; GFX10W32-NEXT: s_or_b32 exec_lo, exec_lo, s12200; GFX10W32-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x242201; GFX10W32-NEXT: s_waitcnt vmcnt(0)2202; GFX10W32-NEXT: v_readfirstlane_b32 s2, v02203; GFX10W32-NEXT: v_mov_b32_e32 v0, 02204; GFX10W32-NEXT: v_sub_nc_u32_e32 v1, s2, v12205; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)2206; GFX10W32-NEXT: global_store_dword v0, v1, s[0:1]2207; GFX10W32-NEXT: s_endpgm2208;2209; GFX11W64-LABEL: sub_i32_varying_vdata:2210; GFX11W64: ; %bb.0: ; %entry2211; GFX11W64-NEXT: v_and_b32_e32 v1, 0x3ff, v02212; GFX11W64-NEXT: s_mov_b64 s[0:1], exec2213; GFX11W64-NEXT: s_mov_b32 s2, 02214; GFX11W64-NEXT: ; implicit-def: $vgpr02215; GFX11W64-NEXT: .LBB7_1: ; %ComputeLoop2216; GFX11W64-NEXT: ; =>This Inner Loop Header: Depth=12217; GFX11W64-NEXT: s_ctz_i32_b64 s3, s[0:1]2218; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)2219; GFX11W64-NEXT: v_readlane_b32 s8, v1, s32220; GFX11W64-NEXT: v_writelane_b32 v0, s2, s32221; GFX11W64-NEXT: s_lshl_b64 s[6:7], 1, s32222; GFX11W64-NEXT: s_add_i32 s2, s2, s82223; GFX11W64-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[6:7]2224; GFX11W64-NEXT: s_cbranch_scc1 .LBB7_12225; GFX11W64-NEXT: ; %bb.2: ; %ComputeEnd2226; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 02227; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2228; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v12229; GFX11W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v12230; GFX11W64-NEXT: ; implicit-def: $vgpr12231; GFX11W64-NEXT: s_and_saveexec_b64 s[0:1], vcc2232; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2233; GFX11W64-NEXT: s_xor_b64 s[0:1], exec, s[0:1]2234; GFX11W64-NEXT: s_cbranch_execz .LBB7_42235; GFX11W64-NEXT: ; %bb.3:2236; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x342237; GFX11W64-NEXT: v_mov_b32_e32 v1, s22238; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)2239; GFX11W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc2240; GFX11W64-NEXT: .LBB7_4:2241; GFX11W64-NEXT: s_or_b64 exec, exec, s[0:1]2242; GFX11W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x242243; GFX11W64-NEXT: s_waitcnt vmcnt(0)2244; GFX11W64-NEXT: v_readfirstlane_b32 s2, v12245; GFX11W64-NEXT: v_mov_b32_e32 v1, 02246; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_2)2247; GFX11W64-NEXT: v_sub_nc_u32_e32 v0, s2, v02248; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)2249; GFX11W64-NEXT: global_store_b32 v1, v0, s[0:1]2250; GFX11W64-NEXT: s_endpgm2251;2252; GFX11W32-LABEL: sub_i32_varying_vdata:2253; GFX11W32: ; %bb.0: ; %entry2254; GFX11W32-NEXT: v_and_b32_e32 v1, 0x3ff, v02255; GFX11W32-NEXT: s_mov_b32 s1, exec_lo2256; GFX11W32-NEXT: s_mov_b32 s0, 02257; GFX11W32-NEXT: ; implicit-def: $vgpr02258; GFX11W32-NEXT: .LBB7_1: ; %ComputeLoop2259; GFX11W32-NEXT: ; =>This Inner Loop Header: Depth=12260; GFX11W32-NEXT: s_ctz_i32_b32 s2, s12261; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)2262; GFX11W32-NEXT: v_readlane_b32 s3, v1, s22263; GFX11W32-NEXT: v_writelane_b32 v0, s0, s22264; GFX11W32-NEXT: s_lshl_b32 s2, 1, s22265; GFX11W32-NEXT: s_add_i32 s0, s0, s32266; GFX11W32-NEXT: s_and_not1_b32 s1, s1, s22267; GFX11W32-NEXT: s_cbranch_scc1 .LBB7_12268; GFX11W32-NEXT: ; %bb.2: ; %ComputeEnd2269; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 02270; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)2271; GFX11W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v12272; GFX11W32-NEXT: ; implicit-def: $vgpr12273; GFX11W32-NEXT: s_and_saveexec_b32 s1, vcc_lo2274; GFX11W32-NEXT: s_xor_b32 s1, exec_lo, s12275; GFX11W32-NEXT: s_cbranch_execz .LBB7_42276; GFX11W32-NEXT: ; %bb.3:2277; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x342278; GFX11W32-NEXT: v_mov_b32_e32 v1, s02279; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)2280; GFX11W32-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc2281; GFX11W32-NEXT: .LBB7_4:2282; GFX11W32-NEXT: s_or_b32 exec_lo, exec_lo, s12283; GFX11W32-NEXT: s_load_b64 s[0:1], s[4:5], 0x242284; GFX11W32-NEXT: s_waitcnt vmcnt(0)2285; GFX11W32-NEXT: v_readfirstlane_b32 s2, v12286; GFX11W32-NEXT: v_mov_b32_e32 v1, 02287; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_2)2288; GFX11W32-NEXT: v_sub_nc_u32_e32 v0, s2, v02289; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)2290; GFX11W32-NEXT: global_store_b32 v1, v0, s[0:1]2291; GFX11W32-NEXT: s_endpgm2292;2293; GFX12W64-LABEL: sub_i32_varying_vdata:2294; GFX12W64: ; %bb.0: ; %entry2295; GFX12W64-NEXT: v_and_b32_e32 v1, 0x3ff, v02296; GFX12W64-NEXT: s_mov_b64 s[0:1], exec2297; GFX12W64-NEXT: s_mov_b32 s2, 02298; GFX12W64-NEXT: ; implicit-def: $vgpr02299; GFX12W64-NEXT: .LBB7_1: ; %ComputeLoop2300; GFX12W64-NEXT: ; =>This Inner Loop Header: Depth=12301; GFX12W64-NEXT: s_ctz_i32_b64 s3, s[0:1]2302; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)2303; GFX12W64-NEXT: v_readlane_b32 s8, v1, s32304; GFX12W64-NEXT: v_writelane_b32 v0, s2, s32305; GFX12W64-NEXT: s_lshl_b64 s[6:7], 1, s32306; GFX12W64-NEXT: s_add_co_i32 s2, s2, s82307; GFX12W64-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[6:7]2308; GFX12W64-NEXT: s_cbranch_scc1 .LBB7_12309; GFX12W64-NEXT: ; %bb.2: ; %ComputeEnd2310; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 02311; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2312; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v12313; GFX12W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v12314; GFX12W64-NEXT: ; implicit-def: $vgpr12315; GFX12W64-NEXT: s_and_saveexec_b64 s[0:1], vcc2316; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2317; GFX12W64-NEXT: s_xor_b64 s[0:1], exec, s[0:1]2318; GFX12W64-NEXT: s_cbranch_execz .LBB7_42319; GFX12W64-NEXT: ; %bb.3:2320; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x342321; GFX12W64-NEXT: v_mov_b32_e32 v1, s22322; GFX12W64-NEXT: s_wait_kmcnt 0x02323; GFX12W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN2324; GFX12W64-NEXT: .LBB7_4:2325; GFX12W64-NEXT: s_or_b64 exec, exec, s[0:1]2326; GFX12W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x242327; GFX12W64-NEXT: s_wait_loadcnt 0x02328; GFX12W64-NEXT: v_readfirstlane_b32 s2, v12329; GFX12W64-NEXT: v_mov_b32_e32 v1, 02330; GFX12W64-NEXT: s_wait_alu depctr_va_sdst(0)2331; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_2)2332; GFX12W64-NEXT: v_sub_nc_u32_e32 v0, s2, v02333; GFX12W64-NEXT: s_wait_kmcnt 0x02334; GFX12W64-NEXT: global_store_b32 v1, v0, s[0:1]2335; GFX12W64-NEXT: s_endpgm2336;2337; GFX12W32-LABEL: sub_i32_varying_vdata:2338; GFX12W32: ; %bb.0: ; %entry2339; GFX12W32-NEXT: v_and_b32_e32 v1, 0x3ff, v02340; GFX12W32-NEXT: s_mov_b32 s1, exec_lo2341; GFX12W32-NEXT: s_mov_b32 s0, 02342; GFX12W32-NEXT: ; implicit-def: $vgpr02343; GFX12W32-NEXT: .LBB7_1: ; %ComputeLoop2344; GFX12W32-NEXT: ; =>This Inner Loop Header: Depth=12345; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)2346; GFX12W32-NEXT: s_ctz_i32_b32 s2, s12347; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)2348; GFX12W32-NEXT: v_readlane_b32 s3, v1, s22349; GFX12W32-NEXT: v_writelane_b32 v0, s0, s22350; GFX12W32-NEXT: s_lshl_b32 s2, 1, s22351; GFX12W32-NEXT: s_add_co_i32 s0, s0, s32352; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)2353; GFX12W32-NEXT: s_and_not1_b32 s1, s1, s22354; GFX12W32-NEXT: s_cbranch_scc1 .LBB7_12355; GFX12W32-NEXT: ; %bb.2: ; %ComputeEnd2356; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 02357; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)2358; GFX12W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v12359; GFX12W32-NEXT: ; implicit-def: $vgpr12360; GFX12W32-NEXT: s_and_saveexec_b32 s1, vcc_lo2361; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)2362; GFX12W32-NEXT: s_xor_b32 s1, exec_lo, s12363; GFX12W32-NEXT: s_cbranch_execz .LBB7_42364; GFX12W32-NEXT: ; %bb.3:2365; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x342366; GFX12W32-NEXT: v_mov_b32_e32 v1, s02367; GFX12W32-NEXT: s_wait_kmcnt 0x02368; GFX12W32-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN2369; GFX12W32-NEXT: .LBB7_4:2370; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)2371; GFX12W32-NEXT: s_or_b32 exec_lo, exec_lo, s12372; GFX12W32-NEXT: s_load_b64 s[0:1], s[4:5], 0x242373; GFX12W32-NEXT: s_wait_loadcnt 0x02374; GFX12W32-NEXT: v_readfirstlane_b32 s2, v12375; GFX12W32-NEXT: v_mov_b32_e32 v1, 02376; GFX12W32-NEXT: s_wait_alu depctr_va_sdst(0)2377; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_2)2378; GFX12W32-NEXT: v_sub_nc_u32_e32 v0, s2, v02379; GFX12W32-NEXT: s_wait_kmcnt 0x02380; GFX12W32-NEXT: global_store_b32 v1, v0, s[0:1]2381; GFX12W32-NEXT: s_endpgm2382entry:2383 %lane = call i32 @llvm.amdgcn.workitem.id.x()2384 %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.sub(i32 %lane, ptr addrspace(8) %inout, i32 0, i32 0, i32 0)2385 store i32 %old, ptr addrspace(1) %out2386 ret void2387}2388 2389define amdgpu_kernel void @sub_i32_varying_offset(ptr addrspace(1) %out, ptr addrspace(8) %inout) {2390; GFX6-LABEL: sub_i32_varying_offset:2391; GFX6: ; %bb.0: ; %entry2392; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xd2393; GFX6-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x92394; GFX6-NEXT: v_mov_b32_e32 v1, 12395; GFX6-NEXT: s_waitcnt lgkmcnt(0)2396; GFX6-NEXT: buffer_atomic_sub v1, v0, s[0:3], 0 offen glc2397; GFX6-NEXT: s_mov_b32 s7, 0xf0002398; GFX6-NEXT: s_mov_b32 s6, -12399; GFX6-NEXT: s_waitcnt vmcnt(0)2400; GFX6-NEXT: buffer_store_dword v1, off, s[4:7], 02401; GFX6-NEXT: s_endpgm2402;2403; GFX8-LABEL: sub_i32_varying_offset:2404; GFX8: ; %bb.0: ; %entry2405; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x342406; GFX8-NEXT: v_mov_b32_e32 v2, 12407; GFX8-NEXT: s_waitcnt lgkmcnt(0)2408; GFX8-NEXT: buffer_atomic_sub v2, v0, s[0:3], 0 offen glc2409; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x242410; GFX8-NEXT: s_waitcnt lgkmcnt(0)2411; GFX8-NEXT: v_mov_b32_e32 v0, s02412; GFX8-NEXT: v_mov_b32_e32 v1, s12413; GFX8-NEXT: s_waitcnt vmcnt(0)2414; GFX8-NEXT: flat_store_dword v[0:1], v22415; GFX8-NEXT: s_endpgm2416;2417; GFX9-LABEL: sub_i32_varying_offset:2418; GFX9: ; %bb.0: ; %entry2419; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x342420; GFX9-NEXT: v_mov_b32_e32 v1, 12421; GFX9-NEXT: s_waitcnt lgkmcnt(0)2422; GFX9-NEXT: buffer_atomic_sub v1, v0, s[0:3], 0 offen glc2423; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x242424; GFX9-NEXT: v_mov_b32_e32 v0, 02425; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2426; GFX9-NEXT: global_store_dword v0, v1, s[0:1]2427; GFX9-NEXT: s_endpgm2428;2429; GFX10-LABEL: sub_i32_varying_offset:2430; GFX10: ; %bb.0: ; %entry2431; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x342432; GFX10-NEXT: v_mov_b32_e32 v1, 12433; GFX10-NEXT: s_waitcnt lgkmcnt(0)2434; GFX10-NEXT: buffer_atomic_sub v1, v0, s[0:3], 0 offen glc2435; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)2436; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x242437; GFX10-NEXT: v_mov_b32_e32 v0, 02438; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2439; GFX10-NEXT: global_store_dword v0, v1, s[0:1]2440; GFX10-NEXT: s_endpgm2441;2442; GFX11W64-LABEL: sub_i32_varying_offset:2443; GFX11W64: ; %bb.0: ; %entry2444; GFX11W64-NEXT: s_load_b128 s[0:3], s[4:5], 0x342445; GFX11W64-NEXT: v_and_b32_e32 v0, 0x3ff, v02446; GFX11W64-NEXT: v_mov_b32_e32 v1, 12447; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)2448; GFX11W64-NEXT: buffer_atomic_sub_u32 v1, v0, s[0:3], 0 offen glc2449; GFX11W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x242450; GFX11W64-NEXT: v_mov_b32_e32 v0, 02451; GFX11W64-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2452; GFX11W64-NEXT: global_store_b32 v0, v1, s[0:1]2453; GFX11W64-NEXT: s_endpgm2454;2455; GFX11W32-LABEL: sub_i32_varying_offset:2456; GFX11W32: ; %bb.0: ; %entry2457; GFX11W32-NEXT: s_load_b128 s[0:3], s[4:5], 0x342458; GFX11W32-NEXT: v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v02459; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)2460; GFX11W32-NEXT: buffer_atomic_sub_u32 v1, v0, s[0:3], 0 offen glc2461; GFX11W32-NEXT: s_load_b64 s[0:1], s[4:5], 0x242462; GFX11W32-NEXT: v_mov_b32_e32 v0, 02463; GFX11W32-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2464; GFX11W32-NEXT: global_store_b32 v0, v1, s[0:1]2465; GFX11W32-NEXT: s_endpgm2466;2467; GFX12W64-LABEL: sub_i32_varying_offset:2468; GFX12W64: ; %bb.0: ; %entry2469; GFX12W64-NEXT: s_load_b128 s[0:3], s[4:5], 0x342470; GFX12W64-NEXT: v_and_b32_e32 v0, 0x3ff, v02471; GFX12W64-NEXT: v_mov_b32_e32 v1, 12472; GFX12W64-NEXT: s_wait_kmcnt 0x02473; GFX12W64-NEXT: buffer_atomic_sub_u32 v1, v0, s[0:3], null offen th:TH_ATOMIC_RETURN2474; GFX12W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x242475; GFX12W64-NEXT: v_mov_b32_e32 v0, 02476; GFX12W64-NEXT: s_wait_loadcnt 0x02477; GFX12W64-NEXT: s_wait_kmcnt 0x02478; GFX12W64-NEXT: global_store_b32 v0, v1, s[0:1]2479; GFX12W64-NEXT: s_endpgm2480;2481; GFX12W32-LABEL: sub_i32_varying_offset:2482; GFX12W32: ; %bb.0: ; %entry2483; GFX12W32-NEXT: s_load_b128 s[0:3], s[4:5], 0x342484; GFX12W32-NEXT: v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v02485; GFX12W32-NEXT: s_wait_kmcnt 0x02486; GFX12W32-NEXT: buffer_atomic_sub_u32 v1, v0, s[0:3], null offen th:TH_ATOMIC_RETURN2487; GFX12W32-NEXT: s_load_b64 s[0:1], s[4:5], 0x242488; GFX12W32-NEXT: v_mov_b32_e32 v0, 02489; GFX12W32-NEXT: s_wait_loadcnt 0x02490; GFX12W32-NEXT: s_wait_kmcnt 0x02491; GFX12W32-NEXT: global_store_b32 v0, v1, s[0:1]2492; GFX12W32-NEXT: s_endpgm2493entry:2494 %lane = call i32 @llvm.amdgcn.workitem.id.x()2495 %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.sub(i32 1, ptr addrspace(8) %inout, i32 %lane, i32 0, i32 0)2496 store i32 %old, ptr addrspace(1) %out2497 ret void2498}2499;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:2500; GFX11: {{.*}}2501; GFX12: {{.*}}2502