brintos / llvm-project-archived public Read only

1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX6 %s3; RUN: llc -mtriple=amdgcn -mcpu=tonga -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX8 %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s5; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize64 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX10,GFX10W64 %s6; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX10,GFX10W32 %s7; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11W64 %s8; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11W32 %s9; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12W64 %s10; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12W32 %s11 12declare i32 @llvm.amdgcn.workitem.id.x()13declare i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32, ptr addrspace(8), i32, i32, i32)14declare i32 @llvm.amdgcn.raw.ptr.buffer.atomic.sub(i32, ptr addrspace(8), i32, i32, i32)15 16; Show what the atomic optimization pass will do for raw buffers.17 18define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace(8) %inout) {19; GFX6-LABEL: add_i32_constant:20; GFX6:       ; %bb.0: ; %entry21; GFX6-NEXT:    s_mov_b64 s[2:3], exec22; GFX6-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 023; GFX6-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v024; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v025; GFX6-NEXT:    ; implicit-def: $vgpr126; GFX6-NEXT:    s_and_saveexec_b64 s[0:1], vcc27; GFX6-NEXT:    s_cbranch_execz .LBB0_228; GFX6-NEXT:  ; %bb.1:29; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xd30; GFX6-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]31; GFX6-NEXT:    s_mul_i32 s2, s2, 532; GFX6-NEXT:    v_mov_b32_e32 v1, s233; GFX6-NEXT:    s_waitcnt lgkmcnt(0)34; GFX6-NEXT:    buffer_atomic_add v1, off, s[8:11], 0 glc35; GFX6-NEXT:  .LBB0_2:36; GFX6-NEXT:    s_or_b64 exec, exec, s[0:1]37; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x938; GFX6-NEXT:    s_mov_b32 s3, 0xf00039; GFX6-NEXT:    s_mov_b32 s2, -140; GFX6-NEXT:    s_waitcnt vmcnt(0)41; GFX6-NEXT:    v_readfirstlane_b32 s4, v142; GFX6-NEXT:    v_mad_u32_u24 v0, v0, 5, s443; GFX6-NEXT:    s_waitcnt lgkmcnt(0)44; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 045; GFX6-NEXT:    s_endpgm46;47; GFX8-LABEL: add_i32_constant:48; GFX8:       ; %bb.0: ; %entry49; GFX8-NEXT:    s_mov_b64 s[2:3], exec50; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 051; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v052; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v053; GFX8-NEXT:    ; implicit-def: $vgpr154; GFX8-NEXT:    s_and_saveexec_b64 s[0:1], vcc55; GFX8-NEXT:    s_cbranch_execz .LBB0_256; GFX8-NEXT:  ; %bb.1:57; GFX8-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x3458; GFX8-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]59; GFX8-NEXT:    s_mul_i32 s2, s2, 560; GFX8-NEXT:    v_mov_b32_e32 v1, s261; GFX8-NEXT:    s_waitcnt lgkmcnt(0)62; GFX8-NEXT:    buffer_atomic_add v1, off, s[8:11], 0 glc63; GFX8-NEXT:  .LBB0_2:64; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]65; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2466; GFX8-NEXT:    s_waitcnt vmcnt(0)67; GFX8-NEXT:    v_readfirstlane_b32 s2, v168; GFX8-NEXT:    v_mad_u32_u24 v2, v0, 5, s269; GFX8-NEXT:    s_waitcnt lgkmcnt(0)70; GFX8-NEXT:    v_mov_b32_e32 v0, s071; GFX8-NEXT:    v_mov_b32_e32 v1, s172; GFX8-NEXT:    flat_store_dword v[0:1], v273; GFX8-NEXT:    s_endpgm74;75; GFX9-LABEL: add_i32_constant:76; GFX9:       ; %bb.0: ; %entry77; GFX9-NEXT:    s_mov_b64 s[2:3], exec78; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 079; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v080; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v081; GFX9-NEXT:    ; implicit-def: $vgpr182; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc83; GFX9-NEXT:    s_cbranch_execz .LBB0_284; GFX9-NEXT:  ; %bb.1:85; GFX9-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x3486; GFX9-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]87; GFX9-NEXT:    s_mul_i32 s2, s2, 588; GFX9-NEXT:    v_mov_b32_e32 v1, s289; GFX9-NEXT:    s_waitcnt lgkmcnt(0)90; GFX9-NEXT:    buffer_atomic_add v1, off, s[8:11], 0 glc91; GFX9-NEXT:  .LBB0_2:92; GFX9-NEXT:    s_or_b64 exec, exec, s[0:1]93; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2494; GFX9-NEXT:    s_waitcnt vmcnt(0)95; GFX9-NEXT:    v_readfirstlane_b32 s2, v196; GFX9-NEXT:    v_mov_b32_e32 v2, 097; GFX9-NEXT:    v_mad_u32_u24 v0, v0, 5, s298; GFX9-NEXT:    s_waitcnt lgkmcnt(0)99; GFX9-NEXT:    global_store_dword v2, v0, s[0:1]100; GFX9-NEXT:    s_endpgm101;102; GFX10W64-LABEL: add_i32_constant:103; GFX10W64:       ; %bb.0: ; %entry104; GFX10W64-NEXT:    s_mov_b64 s[2:3], exec105; GFX10W64-NEXT:    ; implicit-def: $vgpr1106; GFX10W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0107; GFX10W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0108; GFX10W64-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0109; GFX10W64-NEXT:    s_and_saveexec_b64 s[0:1], vcc110; GFX10W64-NEXT:    s_cbranch_execz .LBB0_2111; GFX10W64-NEXT:  ; %bb.1:112; GFX10W64-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34113; GFX10W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]114; GFX10W64-NEXT:    s_mul_i32 s2, s2, 5115; GFX10W64-NEXT:    v_mov_b32_e32 v1, s2116; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)117; GFX10W64-NEXT:    buffer_atomic_add v1, off, s[8:11], 0 glc118; GFX10W64-NEXT:  .LBB0_2:119; GFX10W64-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)120; GFX10W64-NEXT:    s_or_b64 exec, exec, s[0:1]121; GFX10W64-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24122; GFX10W64-NEXT:    s_waitcnt vmcnt(0)123; GFX10W64-NEXT:    v_readfirstlane_b32 s2, v1124; GFX10W64-NEXT:    v_mov_b32_e32 v1, 0125; GFX10W64-NEXT:    v_mad_u32_u24 v0, v0, 5, s2126; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)127; GFX10W64-NEXT:    global_store_dword v1, v0, s[0:1]128; GFX10W64-NEXT:    s_endpgm129;130; GFX10W32-LABEL: add_i32_constant:131; GFX10W32:       ; %bb.0: ; %entry132; GFX10W32-NEXT:    s_mov_b32 s1, exec_lo133; GFX10W32-NEXT:    ; implicit-def: $vgpr1134; GFX10W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s1, 0135; GFX10W32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0136; GFX10W32-NEXT:    s_and_saveexec_b32 s0, vcc_lo137; GFX10W32-NEXT:    s_cbranch_execz .LBB0_2138; GFX10W32-NEXT:  ; %bb.1:139; GFX10W32-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34140; GFX10W32-NEXT:    s_bcnt1_i32_b32 s1, s1141; GFX10W32-NEXT:    s_mul_i32 s1, s1, 5142; GFX10W32-NEXT:    v_mov_b32_e32 v1, s1143; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)144; GFX10W32-NEXT:    buffer_atomic_add v1, off, s[8:11], 0 glc145; GFX10W32-NEXT:  .LBB0_2:146; GFX10W32-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)147; GFX10W32-NEXT:    s_or_b32 exec_lo, exec_lo, s0148; GFX10W32-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24149; GFX10W32-NEXT:    s_waitcnt vmcnt(0)150; GFX10W32-NEXT:    v_readfirstlane_b32 s2, v1151; GFX10W32-NEXT:    v_mov_b32_e32 v1, 0152; GFX10W32-NEXT:    v_mad_u32_u24 v0, v0, 5, s2153; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)154; GFX10W32-NEXT:    global_store_dword v1, v0, s[0:1]155; GFX10W32-NEXT:    s_endpgm156;157; GFX11W64-LABEL: add_i32_constant:158; GFX11W64:       ; %bb.0: ; %entry159; GFX11W64-NEXT:    s_mov_b64 s[2:3], exec160; GFX11W64-NEXT:    s_mov_b64 s[0:1], exec161; GFX11W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0162; GFX11W64-NEXT:    ; implicit-def: $vgpr1163; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)164; GFX11W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0165; GFX11W64-NEXT:    v_cmpx_eq_u32_e32 0, v0166; GFX11W64-NEXT:    s_cbranch_execz .LBB0_2167; GFX11W64-NEXT:  ; %bb.1:168; GFX11W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34169; GFX11W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]170; GFX11W64-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)171; GFX11W64-NEXT:    s_mul_i32 s2, s2, 5172; GFX11W64-NEXT:    v_mov_b32_e32 v1, s2173; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)174; GFX11W64-NEXT:    buffer_atomic_add_u32 v1, off, s[8:11], 0 glc175; GFX11W64-NEXT:  .LBB0_2:176; GFX11W64-NEXT:    s_or_b64 exec, exec, s[0:1]177; GFX11W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24178; GFX11W64-NEXT:    s_waitcnt vmcnt(0)179; GFX11W64-NEXT:    v_readfirstlane_b32 s2, v1180; GFX11W64-NEXT:    v_mov_b32_e32 v1, 0181; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)182; GFX11W64-NEXT:    v_mad_u32_u24 v0, v0, 5, s2183; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)184; GFX11W64-NEXT:    global_store_b32 v1, v0, s[0:1]185; GFX11W64-NEXT:    s_endpgm186;187; GFX11W32-LABEL: add_i32_constant:188; GFX11W32:       ; %bb.0: ; %entry189; GFX11W32-NEXT:    s_mov_b32 s1, exec_lo190; GFX11W32-NEXT:    s_mov_b32 s0, exec_lo191; GFX11W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s1, 0192; GFX11W32-NEXT:    ; implicit-def: $vgpr1193; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)194; GFX11W32-NEXT:    v_cmpx_eq_u32_e32 0, v0195; GFX11W32-NEXT:    s_cbranch_execz .LBB0_2196; GFX11W32-NEXT:  ; %bb.1:197; GFX11W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34198; GFX11W32-NEXT:    s_bcnt1_i32_b32 s1, s1199; GFX11W32-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)200; GFX11W32-NEXT:    s_mul_i32 s1, s1, 5201; GFX11W32-NEXT:    v_mov_b32_e32 v1, s1202; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)203; GFX11W32-NEXT:    buffer_atomic_add_u32 v1, off, s[8:11], 0 glc204; GFX11W32-NEXT:  .LBB0_2:205; GFX11W32-NEXT:    s_or_b32 exec_lo, exec_lo, s0206; GFX11W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24207; GFX11W32-NEXT:    s_waitcnt vmcnt(0)208; GFX11W32-NEXT:    v_readfirstlane_b32 s2, v1209; GFX11W32-NEXT:    v_mov_b32_e32 v1, 0210; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)211; GFX11W32-NEXT:    v_mad_u32_u24 v0, v0, 5, s2212; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)213; GFX11W32-NEXT:    global_store_b32 v1, v0, s[0:1]214; GFX11W32-NEXT:    s_endpgm215;216; GFX12W64-LABEL: add_i32_constant:217; GFX12W64:       ; %bb.0: ; %entry218; GFX12W64-NEXT:    s_mov_b64 s[2:3], exec219; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec220; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0221; GFX12W64-NEXT:    ; implicit-def: $vgpr1222; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)223; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0224; GFX12W64-NEXT:    v_cmpx_eq_u32_e32 0, v0225; GFX12W64-NEXT:    s_cbranch_execz .LBB0_2226; GFX12W64-NEXT:  ; %bb.1:227; GFX12W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34228; GFX12W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]229; GFX12W64-NEXT:    s_wait_alu depctr_sa_sdst(0)230; GFX12W64-NEXT:    s_mul_i32 s2, s2, 5231; GFX12W64-NEXT:    s_wait_alu depctr_sa_sdst(0)232; GFX12W64-NEXT:    v_mov_b32_e32 v1, s2233; GFX12W64-NEXT:    s_wait_kmcnt 0x0234; GFX12W64-NEXT:    buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN235; GFX12W64-NEXT:  .LBB0_2:236; GFX12W64-NEXT:    s_or_b64 exec, exec, s[0:1]237; GFX12W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24238; GFX12W64-NEXT:    s_wait_loadcnt 0x0239; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1240; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0241; GFX12W64-NEXT:    s_wait_alu depctr_va_sdst(0)242; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)243; GFX12W64-NEXT:    v_mad_u32_u24 v0, v0, 5, s2244; GFX12W64-NEXT:    s_wait_kmcnt 0x0245; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]246; GFX12W64-NEXT:    s_endpgm247;248; GFX12W32-LABEL: add_i32_constant:249; GFX12W32:       ; %bb.0: ; %entry250; GFX12W32-NEXT:    s_mov_b32 s1, exec_lo251; GFX12W32-NEXT:    s_mov_b32 s0, exec_lo252; GFX12W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s1, 0253; GFX12W32-NEXT:    ; implicit-def: $vgpr1254; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)255; GFX12W32-NEXT:    v_cmpx_eq_u32_e32 0, v0256; GFX12W32-NEXT:    s_cbranch_execz .LBB0_2257; GFX12W32-NEXT:  ; %bb.1:258; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34259; GFX12W32-NEXT:    s_bcnt1_i32_b32 s1, s1260; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)261; GFX12W32-NEXT:    s_mul_i32 s1, s1, 5262; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)263; GFX12W32-NEXT:    v_mov_b32_e32 v1, s1264; GFX12W32-NEXT:    s_wait_kmcnt 0x0265; GFX12W32-NEXT:    buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN266; GFX12W32-NEXT:  .LBB0_2:267; GFX12W32-NEXT:    s_or_b32 exec_lo, exec_lo, s0268; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24269; GFX12W32-NEXT:    s_wait_loadcnt 0x0270; GFX12W32-NEXT:    v_readfirstlane_b32 s2, v1271; GFX12W32-NEXT:    v_mov_b32_e32 v1, 0272; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)273; GFX12W32-NEXT:    v_mad_u32_u24 v0, v0, 5, s2274; GFX12W32-NEXT:    s_wait_kmcnt 0x0275; GFX12W32-NEXT:    global_store_b32 v1, v0, s[0:1]276; GFX12W32-NEXT:    s_endpgm277entry:278  %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32 5, ptr addrspace(8) %inout, i32 0, i32 0, i32 0)279  store i32 %old, ptr addrspace(1) %out280  ret void281}282 283define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(8) %inout, i32 %additive) {284; GFX6-LABEL: add_i32_uniform:285; GFX6:       ; %bb.0: ; %entry286; GFX6-NEXT:    s_mov_b64 s[2:3], exec287; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x11288; GFX6-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0289; GFX6-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0290; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0291; GFX6-NEXT:    ; implicit-def: $vgpr1292; GFX6-NEXT:    s_and_saveexec_b64 s[0:1], vcc293; GFX6-NEXT:    s_cbranch_execz .LBB1_2294; GFX6-NEXT:  ; %bb.1:295; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xd296; GFX6-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]297; GFX6-NEXT:    s_waitcnt lgkmcnt(0)298; GFX6-NEXT:    s_mul_i32 s2, s6, s2299; GFX6-NEXT:    v_mov_b32_e32 v1, s2300; GFX6-NEXT:    buffer_atomic_add v1, off, s[8:11], 0 glc301; GFX6-NEXT:  .LBB1_2:302; GFX6-NEXT:    s_or_b64 exec, exec, s[0:1]303; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9304; GFX6-NEXT:    s_mov_b32 s3, 0xf000305; GFX6-NEXT:    s_mov_b32 s2, -1306; GFX6-NEXT:    s_waitcnt vmcnt(0)307; GFX6-NEXT:    v_readfirstlane_b32 s4, v1308; GFX6-NEXT:    s_waitcnt lgkmcnt(0)309; GFX6-NEXT:    v_mul_lo_u32 v0, s6, v0310; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s4, v0311; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0312; GFX6-NEXT:    s_endpgm313;314; GFX8-LABEL: add_i32_uniform:315; GFX8:       ; %bb.0: ; %entry316; GFX8-NEXT:    s_load_dword s6, s[4:5], 0x44317; GFX8-NEXT:    s_mov_b64 s[2:3], exec318; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0319; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0320; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0321; GFX8-NEXT:    ; implicit-def: $vgpr1322; GFX8-NEXT:    s_and_saveexec_b64 s[0:1], vcc323; GFX8-NEXT:    s_cbranch_execz .LBB1_2324; GFX8-NEXT:  ; %bb.1:325; GFX8-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34326; GFX8-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]327; GFX8-NEXT:    s_waitcnt lgkmcnt(0)328; GFX8-NEXT:    s_mul_i32 s2, s6, s2329; GFX8-NEXT:    v_mov_b32_e32 v1, s2330; GFX8-NEXT:    buffer_atomic_add v1, off, s[8:11], 0 glc331; GFX8-NEXT:  .LBB1_2:332; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]333; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24334; GFX8-NEXT:    s_waitcnt lgkmcnt(0)335; GFX8-NEXT:    v_mul_lo_u32 v0, s6, v0336; GFX8-NEXT:    s_waitcnt vmcnt(0)337; GFX8-NEXT:    v_readfirstlane_b32 s2, v1338; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s2, v0339; GFX8-NEXT:    v_mov_b32_e32 v0, s0340; GFX8-NEXT:    v_mov_b32_e32 v1, s1341; GFX8-NEXT:    flat_store_dword v[0:1], v2342; GFX8-NEXT:    s_endpgm343;344; GFX9-LABEL: add_i32_uniform:345; GFX9:       ; %bb.0: ; %entry346; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x44347; GFX9-NEXT:    s_mov_b64 s[2:3], exec348; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0349; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0350; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0351; GFX9-NEXT:    ; implicit-def: $vgpr1352; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc353; GFX9-NEXT:    s_cbranch_execz .LBB1_2354; GFX9-NEXT:  ; %bb.1:355; GFX9-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34356; GFX9-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]357; GFX9-NEXT:    s_waitcnt lgkmcnt(0)358; GFX9-NEXT:    s_mul_i32 s2, s6, s2359; GFX9-NEXT:    v_mov_b32_e32 v1, s2360; GFX9-NEXT:    buffer_atomic_add v1, off, s[8:11], 0 glc361; GFX9-NEXT:  .LBB1_2:362; GFX9-NEXT:    s_or_b64 exec, exec, s[0:1]363; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24364; GFX9-NEXT:    s_waitcnt lgkmcnt(0)365; GFX9-NEXT:    v_mul_lo_u32 v0, s6, v0366; GFX9-NEXT:    s_waitcnt vmcnt(0)367; GFX9-NEXT:    v_readfirstlane_b32 s2, v1368; GFX9-NEXT:    v_mov_b32_e32 v2, 0369; GFX9-NEXT:    v_add_u32_e32 v0, s2, v0370; GFX9-NEXT:    global_store_dword v2, v0, s[0:1]371; GFX9-NEXT:    s_endpgm372;373; GFX10W64-LABEL: add_i32_uniform:374; GFX10W64:       ; %bb.0: ; %entry375; GFX10W64-NEXT:    s_load_dword s6, s[4:5], 0x44376; GFX10W64-NEXT:    s_mov_b64 s[2:3], exec377; GFX10W64-NEXT:    ; implicit-def: $vgpr1378; GFX10W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0379; GFX10W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0380; GFX10W64-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0381; GFX10W64-NEXT:    s_and_saveexec_b64 s[0:1], vcc382; GFX10W64-NEXT:    s_cbranch_execz .LBB1_2383; GFX10W64-NEXT:  ; %bb.1:384; GFX10W64-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34385; GFX10W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]386; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)387; GFX10W64-NEXT:    s_mul_i32 s2, s6, s2388; GFX10W64-NEXT:    v_mov_b32_e32 v1, s2389; GFX10W64-NEXT:    buffer_atomic_add v1, off, s[8:11], 0 glc390; GFX10W64-NEXT:  .LBB1_2:391; GFX10W64-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)392; GFX10W64-NEXT:    s_or_b64 exec, exec, s[0:1]393; GFX10W64-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24394; GFX10W64-NEXT:    s_waitcnt vmcnt(0)395; GFX10W64-NEXT:    v_readfirstlane_b32 s2, v1396; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)397; GFX10W64-NEXT:    v_mad_u64_u32 v[0:1], s[2:3], s6, v0, s[2:3]398; GFX10W64-NEXT:    v_mov_b32_e32 v1, 0399; GFX10W64-NEXT:    global_store_dword v1, v0, s[0:1]400; GFX10W64-NEXT:    s_endpgm401;402; GFX10W32-LABEL: add_i32_uniform:403; GFX10W32:       ; %bb.0: ; %entry404; GFX10W32-NEXT:    s_load_dword s0, s[4:5], 0x44405; GFX10W32-NEXT:    s_mov_b32 s2, exec_lo406; GFX10W32-NEXT:    ; implicit-def: $vgpr1407; GFX10W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0408; GFX10W32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0409; GFX10W32-NEXT:    s_and_saveexec_b32 s1, vcc_lo410; GFX10W32-NEXT:    s_cbranch_execz .LBB1_2411; GFX10W32-NEXT:  ; %bb.1:412; GFX10W32-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34413; GFX10W32-NEXT:    s_bcnt1_i32_b32 s2, s2414; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)415; GFX10W32-NEXT:    s_mul_i32 s2, s0, s2416; GFX10W32-NEXT:    v_mov_b32_e32 v1, s2417; GFX10W32-NEXT:    buffer_atomic_add v1, off, s[8:11], 0 glc418; GFX10W32-NEXT:  .LBB1_2:419; GFX10W32-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)420; GFX10W32-NEXT:    s_or_b32 exec_lo, exec_lo, s1421; GFX10W32-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24422; GFX10W32-NEXT:    s_waitcnt vmcnt(0)423; GFX10W32-NEXT:    s_mov_b32 null, 0424; GFX10W32-NEXT:    v_readfirstlane_b32 s4, v1425; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)426; GFX10W32-NEXT:    v_mad_u64_u32 v[0:1], s0, s0, v0, s[4:5]427; GFX10W32-NEXT:    v_mov_b32_e32 v1, 0428; GFX10W32-NEXT:    global_store_dword v1, v0, s[2:3]429; GFX10W32-NEXT:    s_endpgm430;431; GFX11W64-LABEL: add_i32_uniform:432; GFX11W64:       ; %bb.0: ; %entry433; GFX11W64-NEXT:    s_load_b32 s6, s[4:5], 0x44434; GFX11W64-NEXT:    s_mov_b64 s[2:3], exec435; GFX11W64-NEXT:    s_mov_b64 s[0:1], exec436; GFX11W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0437; GFX11W64-NEXT:    ; implicit-def: $vgpr1438; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)439; GFX11W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0440; GFX11W64-NEXT:    v_cmpx_eq_u32_e32 0, v0441; GFX11W64-NEXT:    s_cbranch_execz .LBB1_2442; GFX11W64-NEXT:  ; %bb.1:443; GFX11W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34444; GFX11W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]445; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)446; GFX11W64-NEXT:    s_mul_i32 s2, s6, s2447; GFX11W64-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)448; GFX11W64-NEXT:    v_mov_b32_e32 v1, s2449; GFX11W64-NEXT:    buffer_atomic_add_u32 v1, off, s[8:11], 0 glc450; GFX11W64-NEXT:  .LBB1_2:451; GFX11W64-NEXT:    s_or_b64 exec, exec, s[0:1]452; GFX11W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24453; GFX11W64-NEXT:    s_waitcnt vmcnt(0)454; GFX11W64-NEXT:    v_readfirstlane_b32 s2, v1455; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)456; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_1)457; GFX11W64-NEXT:    v_mad_u64_u32 v[1:2], null, s6, v0, s[2:3]458; GFX11W64-NEXT:    v_mov_b32_e32 v0, 0459; GFX11W64-NEXT:    global_store_b32 v0, v1, s[0:1]460; GFX11W64-NEXT:    s_endpgm461;462; GFX11W32-LABEL: add_i32_uniform:463; GFX11W32:       ; %bb.0: ; %entry464; GFX11W32-NEXT:    s_load_b32 s0, s[4:5], 0x44465; GFX11W32-NEXT:    s_mov_b32 s2, exec_lo466; GFX11W32-NEXT:    s_mov_b32 s1, exec_lo467; GFX11W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0468; GFX11W32-NEXT:    ; implicit-def: $vgpr1469; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)470; GFX11W32-NEXT:    v_cmpx_eq_u32_e32 0, v0471; GFX11W32-NEXT:    s_cbranch_execz .LBB1_2472; GFX11W32-NEXT:  ; %bb.1:473; GFX11W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34474; GFX11W32-NEXT:    s_bcnt1_i32_b32 s2, s2475; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)476; GFX11W32-NEXT:    s_mul_i32 s2, s0, s2477; GFX11W32-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)478; GFX11W32-NEXT:    v_mov_b32_e32 v1, s2479; GFX11W32-NEXT:    buffer_atomic_add_u32 v1, off, s[8:11], 0 glc480; GFX11W32-NEXT:  .LBB1_2:481; GFX11W32-NEXT:    s_or_b32 exec_lo, exec_lo, s1482; GFX11W32-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24483; GFX11W32-NEXT:    s_waitcnt vmcnt(0)484; GFX11W32-NEXT:    v_readfirstlane_b32 s4, v1485; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)486; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)487; GFX11W32-NEXT:    v_mad_u64_u32 v[1:2], null, s0, v0, s[4:5]488; GFX11W32-NEXT:    v_mov_b32_e32 v0, 0489; GFX11W32-NEXT:    global_store_b32 v0, v1, s[2:3]490; GFX11W32-NEXT:    s_endpgm491;492; GFX12W64-LABEL: add_i32_uniform:493; GFX12W64:       ; %bb.0: ; %entry494; GFX12W64-NEXT:    s_load_b32 s6, s[4:5], 0x44495; GFX12W64-NEXT:    s_mov_b64 s[2:3], exec496; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec497; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0498; GFX12W64-NEXT:    ; implicit-def: $vgpr1499; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)500; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0501; GFX12W64-NEXT:    v_cmpx_eq_u32_e32 0, v0502; GFX12W64-NEXT:    s_cbranch_execz .LBB1_2503; GFX12W64-NEXT:  ; %bb.1:504; GFX12W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34505; GFX12W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]506; GFX12W64-NEXT:    s_wait_kmcnt 0x0507; GFX12W64-NEXT:    s_wait_alu depctr_sa_sdst(0)508; GFX12W64-NEXT:    s_mul_i32 s2, s6, s2509; GFX12W64-NEXT:    s_wait_alu depctr_sa_sdst(0)510; GFX12W64-NEXT:    v_mov_b32_e32 v1, s2511; GFX12W64-NEXT:    buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN512; GFX12W64-NEXT:  .LBB1_2:513; GFX12W64-NEXT:    s_or_b64 exec, exec, s[0:1]514; GFX12W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24515; GFX12W64-NEXT:    s_wait_loadcnt 0x0516; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1517; GFX12W64-NEXT:    s_wait_kmcnt 0x0518; GFX12W64-NEXT:    s_wait_alu depctr_va_sdst(0)519; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1)520; GFX12W64-NEXT:    v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]521; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0522; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]523; GFX12W64-NEXT:    s_endpgm524;525; GFX12W32-LABEL: add_i32_uniform:526; GFX12W32:       ; %bb.0: ; %entry527; GFX12W32-NEXT:    s_load_b32 s0, s[4:5], 0x44528; GFX12W32-NEXT:    s_mov_b32 s2, exec_lo529; GFX12W32-NEXT:    s_mov_b32 s1, exec_lo530; GFX12W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0531; GFX12W32-NEXT:    ; implicit-def: $vgpr1532; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)533; GFX12W32-NEXT:    v_cmpx_eq_u32_e32 0, v0534; GFX12W32-NEXT:    s_cbranch_execz .LBB1_2535; GFX12W32-NEXT:  ; %bb.1:536; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34537; GFX12W32-NEXT:    s_bcnt1_i32_b32 s2, s2538; GFX12W32-NEXT:    s_wait_kmcnt 0x0539; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)540; GFX12W32-NEXT:    s_mul_i32 s2, s0, s2541; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)542; GFX12W32-NEXT:    v_mov_b32_e32 v1, s2543; GFX12W32-NEXT:    buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN544; GFX12W32-NEXT:  .LBB1_2:545; GFX12W32-NEXT:    s_or_b32 exec_lo, exec_lo, s1546; GFX12W32-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24547; GFX12W32-NEXT:    s_wait_loadcnt 0x0548; GFX12W32-NEXT:    v_readfirstlane_b32 s4, v1549; GFX12W32-NEXT:    s_wait_kmcnt 0x0550; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)551; GFX12W32-NEXT:    v_mad_co_u64_u32 v[0:1], null, s0, v0, s[4:5]552; GFX12W32-NEXT:    v_mov_b32_e32 v1, 0553; GFX12W32-NEXT:    global_store_b32 v1, v0, s[2:3]554; GFX12W32-NEXT:    s_endpgm555entry:556  %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32 %additive, ptr addrspace(8) %inout, i32 0, i32 0, i32 0)557  store i32 %old, ptr addrspace(1) %out558  ret void559}560 561define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addrspace(8) %inout) {562; GFX6-LABEL: add_i32_varying_vdata:563; GFX6:       ; %bb.0: ; %entry564; GFX6-NEXT:    s_mov_b64 s[0:1], exec565; GFX6-NEXT:    s_mov_b32 s2, 0566; GFX6-NEXT:    ; implicit-def: $vgpr1567; GFX6-NEXT:  .LBB2_1: ; %ComputeLoop568; GFX6-NEXT:    ; =>This Inner Loop Header: Depth=1569; GFX6-NEXT:    s_ff1_i32_b64 s3, s[0:1]570; GFX6-NEXT:    s_mov_b32 m0, s3571; GFX6-NEXT:    v_readlane_b32 s8, v0, s3572; GFX6-NEXT:    v_writelane_b32 v1, s2, m0573; GFX6-NEXT:    s_lshl_b64 s[6:7], 1, s3574; GFX6-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[6:7]575; GFX6-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0576; GFX6-NEXT:    s_and_b64 vcc, exec, s[6:7]577; GFX6-NEXT:    s_add_i32 s2, s2, s8578; GFX6-NEXT:    s_cbranch_vccnz .LBB2_1579; GFX6-NEXT:  ; %bb.2: ; %ComputeEnd580; GFX6-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0581; GFX6-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0582; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0583; GFX6-NEXT:    ; implicit-def: $vgpr0584; GFX6-NEXT:    s_and_saveexec_b64 s[0:1], vcc585; GFX6-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]586; GFX6-NEXT:    s_cbranch_execz .LBB2_4587; GFX6-NEXT:  ; %bb.3:588; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xd589; GFX6-NEXT:    v_mov_b32_e32 v0, s2590; GFX6-NEXT:    s_waitcnt lgkmcnt(0)591; GFX6-NEXT:    buffer_atomic_add v0, off, s[8:11], 0 glc592; GFX6-NEXT:  .LBB2_4:593; GFX6-NEXT:    s_or_b64 exec, exec, s[0:1]594; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9595; GFX6-NEXT:    s_mov_b32 s3, 0xf000596; GFX6-NEXT:    s_mov_b32 s2, -1597; GFX6-NEXT:    s_waitcnt vmcnt(0)598; GFX6-NEXT:    v_readfirstlane_b32 s4, v0599; GFX6-NEXT:    s_waitcnt expcnt(0)600; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s4, v1601; GFX6-NEXT:    s_waitcnt lgkmcnt(0)602; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0603; GFX6-NEXT:    s_endpgm604;605; GFX8-LABEL: add_i32_varying_vdata:606; GFX8:       ; %bb.0: ; %entry607; GFX8-NEXT:    s_mov_b64 s[0:1], exec608; GFX8-NEXT:    s_mov_b32 s2, 0609; GFX8-NEXT:    ; implicit-def: $vgpr1610; GFX8-NEXT:  .LBB2_1: ; %ComputeLoop611; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1612; GFX8-NEXT:    s_ff1_i32_b64 s3, s[0:1]613; GFX8-NEXT:    s_mov_b32 m0, s3614; GFX8-NEXT:    v_readlane_b32 s6, v0, s3615; GFX8-NEXT:    v_writelane_b32 v1, s2, m0616; GFX8-NEXT:    s_add_i32 s2, s2, s6617; GFX8-NEXT:    s_lshl_b64 s[6:7], 1, s3618; GFX8-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[6:7]619; GFX8-NEXT:    s_cbranch_scc1 .LBB2_1620; GFX8-NEXT:  ; %bb.2: ; %ComputeEnd621; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0622; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0623; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0624; GFX8-NEXT:    ; implicit-def: $vgpr0625; GFX8-NEXT:    s_and_saveexec_b64 s[0:1], vcc626; GFX8-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]627; GFX8-NEXT:    s_cbranch_execz .LBB2_4628; GFX8-NEXT:  ; %bb.3:629; GFX8-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34630; GFX8-NEXT:    v_mov_b32_e32 v0, s2631; GFX8-NEXT:    s_waitcnt lgkmcnt(0)632; GFX8-NEXT:    buffer_atomic_add v0, off, s[8:11], 0 glc633; GFX8-NEXT:  .LBB2_4:634; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]635; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24636; GFX8-NEXT:    s_waitcnt vmcnt(0)637; GFX8-NEXT:    v_readfirstlane_b32 s2, v0638; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s2, v1639; GFX8-NEXT:    s_waitcnt lgkmcnt(0)640; GFX8-NEXT:    v_mov_b32_e32 v0, s0641; GFX8-NEXT:    v_mov_b32_e32 v1, s1642; GFX8-NEXT:    flat_store_dword v[0:1], v2643; GFX8-NEXT:    s_endpgm644;645; GFX9-LABEL: add_i32_varying_vdata:646; GFX9:       ; %bb.0: ; %entry647; GFX9-NEXT:    s_mov_b64 s[0:1], exec648; GFX9-NEXT:    s_mov_b32 s2, 0649; GFX9-NEXT:    ; implicit-def: $vgpr1650; GFX9-NEXT:  .LBB2_1: ; %ComputeLoop651; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1652; GFX9-NEXT:    s_ff1_i32_b64 s3, s[0:1]653; GFX9-NEXT:    s_mov_b32 m0, s3654; GFX9-NEXT:    v_readlane_b32 s6, v0, s3655; GFX9-NEXT:    v_writelane_b32 v1, s2, m0656; GFX9-NEXT:    s_add_i32 s2, s2, s6657; GFX9-NEXT:    s_lshl_b64 s[6:7], 1, s3658; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[6:7]659; GFX9-NEXT:    s_cbranch_scc1 .LBB2_1660; GFX9-NEXT:  ; %bb.2: ; %ComputeEnd661; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0662; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0663; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0664; GFX9-NEXT:    ; implicit-def: $vgpr0665; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc666; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]667; GFX9-NEXT:    s_cbranch_execz .LBB2_4668; GFX9-NEXT:  ; %bb.3:669; GFX9-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34670; GFX9-NEXT:    v_mov_b32_e32 v0, s2671; GFX9-NEXT:    s_waitcnt lgkmcnt(0)672; GFX9-NEXT:    buffer_atomic_add v0, off, s[8:11], 0 glc673; GFX9-NEXT:  .LBB2_4:674; GFX9-NEXT:    s_or_b64 exec, exec, s[0:1]675; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24676; GFX9-NEXT:    s_waitcnt vmcnt(0)677; GFX9-NEXT:    v_readfirstlane_b32 s2, v0678; GFX9-NEXT:    v_mov_b32_e32 v2, 0679; GFX9-NEXT:    v_add_u32_e32 v0, s2, v1680; GFX9-NEXT:    s_waitcnt lgkmcnt(0)681; GFX9-NEXT:    global_store_dword v2, v0, s[0:1]682; GFX9-NEXT:    s_endpgm683;684; GFX10W64-LABEL: add_i32_varying_vdata:685; GFX10W64:       ; %bb.0: ; %entry686; GFX10W64-NEXT:    s_mov_b64 s[0:1], exec687; GFX10W64-NEXT:    s_mov_b32 s2, 0688; GFX10W64-NEXT:    ; implicit-def: $vgpr1689; GFX10W64-NEXT:  .LBB2_1: ; %ComputeLoop690; GFX10W64-NEXT:    ; =>This Inner Loop Header: Depth=1691; GFX10W64-NEXT:    s_ff1_i32_b64 s3, s[0:1]692; GFX10W64-NEXT:    v_readlane_b32 s8, v0, s3693; GFX10W64-NEXT:    v_writelane_b32 v1, s2, s3694; GFX10W64-NEXT:    s_lshl_b64 s[6:7], 1, s3695; GFX10W64-NEXT:    s_add_i32 s2, s2, s8696; GFX10W64-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[6:7]697; GFX10W64-NEXT:    s_cbranch_scc1 .LBB2_1698; GFX10W64-NEXT:  ; %bb.2: ; %ComputeEnd699; GFX10W64-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0700; GFX10W64-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0701; GFX10W64-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0702; GFX10W64-NEXT:    ; implicit-def: $vgpr0703; GFX10W64-NEXT:    s_and_saveexec_b64 s[0:1], vcc704; GFX10W64-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]705; GFX10W64-NEXT:    s_cbranch_execz .LBB2_4706; GFX10W64-NEXT:  ; %bb.3:707; GFX10W64-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34708; GFX10W64-NEXT:    v_mov_b32_e32 v0, s2709; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)710; GFX10W64-NEXT:    buffer_atomic_add v0, off, s[8:11], 0 glc711; GFX10W64-NEXT:  .LBB2_4:712; GFX10W64-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)713; GFX10W64-NEXT:    s_or_b64 exec, exec, s[0:1]714; GFX10W64-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24715; GFX10W64-NEXT:    s_waitcnt vmcnt(0)716; GFX10W64-NEXT:    v_readfirstlane_b32 s2, v0717; GFX10W64-NEXT:    v_mov_b32_e32 v0, 0718; GFX10W64-NEXT:    v_add_nc_u32_e32 v1, s2, v1719; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)720; GFX10W64-NEXT:    global_store_dword v0, v1, s[0:1]721; GFX10W64-NEXT:    s_endpgm722;723; GFX10W32-LABEL: add_i32_varying_vdata:724; GFX10W32:       ; %bb.0: ; %entry725; GFX10W32-NEXT:    s_mov_b32 s1, exec_lo726; GFX10W32-NEXT:    s_mov_b32 s0, 0727; GFX10W32-NEXT:    ; implicit-def: $vgpr1728; GFX10W32-NEXT:  .LBB2_1: ; %ComputeLoop729; GFX10W32-NEXT:    ; =>This Inner Loop Header: Depth=1730; GFX10W32-NEXT:    s_ff1_i32_b32 s2, s1731; GFX10W32-NEXT:    v_readlane_b32 s3, v0, s2732; GFX10W32-NEXT:    v_writelane_b32 v1, s0, s2733; GFX10W32-NEXT:    s_lshl_b32 s2, 1, s2734; GFX10W32-NEXT:    s_add_i32 s0, s0, s3735; GFX10W32-NEXT:    s_andn2_b32 s1, s1, s2736; GFX10W32-NEXT:    s_cbranch_scc1 .LBB2_1737; GFX10W32-NEXT:  ; %bb.2: ; %ComputeEnd738; GFX10W32-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0739; GFX10W32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0740; GFX10W32-NEXT:    ; implicit-def: $vgpr0741; GFX10W32-NEXT:    s_and_saveexec_b32 s1, vcc_lo742; GFX10W32-NEXT:    s_xor_b32 s1, exec_lo, s1743; GFX10W32-NEXT:    s_cbranch_execz .LBB2_4744; GFX10W32-NEXT:  ; %bb.3:745; GFX10W32-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34746; GFX10W32-NEXT:    v_mov_b32_e32 v0, s0747; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)748; GFX10W32-NEXT:    buffer_atomic_add v0, off, s[8:11], 0 glc749; GFX10W32-NEXT:  .LBB2_4:750; GFX10W32-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)751; GFX10W32-NEXT:    s_or_b32 exec_lo, exec_lo, s1752; GFX10W32-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24753; GFX10W32-NEXT:    s_waitcnt vmcnt(0)754; GFX10W32-NEXT:    v_readfirstlane_b32 s2, v0755; GFX10W32-NEXT:    v_mov_b32_e32 v0, 0756; GFX10W32-NEXT:    v_add_nc_u32_e32 v1, s2, v1757; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)758; GFX10W32-NEXT:    global_store_dword v0, v1, s[0:1]759; GFX10W32-NEXT:    s_endpgm760;761; GFX11W64-LABEL: add_i32_varying_vdata:762; GFX11W64:       ; %bb.0: ; %entry763; GFX11W64-NEXT:    v_and_b32_e32 v1, 0x3ff, v0764; GFX11W64-NEXT:    s_mov_b64 s[0:1], exec765; GFX11W64-NEXT:    s_mov_b32 s2, 0766; GFX11W64-NEXT:    ; implicit-def: $vgpr0767; GFX11W64-NEXT:  .LBB2_1: ; %ComputeLoop768; GFX11W64-NEXT:    ; =>This Inner Loop Header: Depth=1769; GFX11W64-NEXT:    s_ctz_i32_b64 s3, s[0:1]770; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)771; GFX11W64-NEXT:    v_readlane_b32 s8, v1, s3772; GFX11W64-NEXT:    v_writelane_b32 v0, s2, s3773; GFX11W64-NEXT:    s_lshl_b64 s[6:7], 1, s3774; GFX11W64-NEXT:    s_add_i32 s2, s2, s8775; GFX11W64-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[6:7]776; GFX11W64-NEXT:    s_cbranch_scc1 .LBB2_1777; GFX11W64-NEXT:  ; %bb.2: ; %ComputeEnd778; GFX11W64-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0779; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)780; GFX11W64-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1781; GFX11W64-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1782; GFX11W64-NEXT:    ; implicit-def: $vgpr1783; GFX11W64-NEXT:    s_and_saveexec_b64 s[0:1], vcc784; GFX11W64-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)785; GFX11W64-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]786; GFX11W64-NEXT:    s_cbranch_execz .LBB2_4787; GFX11W64-NEXT:  ; %bb.3:788; GFX11W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34789; GFX11W64-NEXT:    v_mov_b32_e32 v1, s2790; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)791; GFX11W64-NEXT:    buffer_atomic_add_u32 v1, off, s[8:11], 0 glc792; GFX11W64-NEXT:  .LBB2_4:793; GFX11W64-NEXT:    s_or_b64 exec, exec, s[0:1]794; GFX11W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24795; GFX11W64-NEXT:    s_waitcnt vmcnt(0)796; GFX11W64-NEXT:    v_readfirstlane_b32 s2, v1797; GFX11W64-NEXT:    v_mov_b32_e32 v1, 0798; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)799; GFX11W64-NEXT:    v_add_nc_u32_e32 v0, s2, v0800; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)801; GFX11W64-NEXT:    global_store_b32 v1, v0, s[0:1]802; GFX11W64-NEXT:    s_endpgm803;804; GFX11W32-LABEL: add_i32_varying_vdata:805; GFX11W32:       ; %bb.0: ; %entry806; GFX11W32-NEXT:    v_and_b32_e32 v1, 0x3ff, v0807; GFX11W32-NEXT:    s_mov_b32 s1, exec_lo808; GFX11W32-NEXT:    s_mov_b32 s0, 0809; GFX11W32-NEXT:    ; implicit-def: $vgpr0810; GFX11W32-NEXT:  .LBB2_1: ; %ComputeLoop811; GFX11W32-NEXT:    ; =>This Inner Loop Header: Depth=1812; GFX11W32-NEXT:    s_ctz_i32_b32 s2, s1813; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)814; GFX11W32-NEXT:    v_readlane_b32 s3, v1, s2815; GFX11W32-NEXT:    v_writelane_b32 v0, s0, s2816; GFX11W32-NEXT:    s_lshl_b32 s2, 1, s2817; GFX11W32-NEXT:    s_add_i32 s0, s0, s3818; GFX11W32-NEXT:    s_and_not1_b32 s1, s1, s2819; GFX11W32-NEXT:    s_cbranch_scc1 .LBB2_1820; GFX11W32-NEXT:  ; %bb.2: ; %ComputeEnd821; GFX11W32-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0822; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)823; GFX11W32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1824; GFX11W32-NEXT:    ; implicit-def: $vgpr1825; GFX11W32-NEXT:    s_and_saveexec_b32 s1, vcc_lo826; GFX11W32-NEXT:    s_xor_b32 s1, exec_lo, s1827; GFX11W32-NEXT:    s_cbranch_execz .LBB2_4828; GFX11W32-NEXT:  ; %bb.3:829; GFX11W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34830; GFX11W32-NEXT:    v_mov_b32_e32 v1, s0831; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)832; GFX11W32-NEXT:    buffer_atomic_add_u32 v1, off, s[8:11], 0 glc833; GFX11W32-NEXT:  .LBB2_4:834; GFX11W32-NEXT:    s_or_b32 exec_lo, exec_lo, s1835; GFX11W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24836; GFX11W32-NEXT:    s_waitcnt vmcnt(0)837; GFX11W32-NEXT:    v_readfirstlane_b32 s2, v1838; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)839; GFX11W32-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, s2, v0840; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)841; GFX11W32-NEXT:    global_store_b32 v1, v0, s[0:1]842; GFX11W32-NEXT:    s_endpgm843;844; GFX12W64-LABEL: add_i32_varying_vdata:845; GFX12W64:       ; %bb.0: ; %entry846; GFX12W64-NEXT:    v_and_b32_e32 v1, 0x3ff, v0847; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec848; GFX12W64-NEXT:    s_mov_b32 s2, 0849; GFX12W64-NEXT:    ; implicit-def: $vgpr0850; GFX12W64-NEXT:  .LBB2_1: ; %ComputeLoop851; GFX12W64-NEXT:    ; =>This Inner Loop Header: Depth=1852; GFX12W64-NEXT:    s_ctz_i32_b64 s3, s[0:1]853; GFX12W64-NEXT:    s_wait_alu depctr_sa_sdst(0)854; GFX12W64-NEXT:    v_readlane_b32 s8, v1, s3855; GFX12W64-NEXT:    v_writelane_b32 v0, s2, s3856; GFX12W64-NEXT:    s_lshl_b64 s[6:7], 1, s3857; GFX12W64-NEXT:    s_add_co_i32 s2, s2, s8858; GFX12W64-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[6:7]859; GFX12W64-NEXT:    s_cbranch_scc1 .LBB2_1860; GFX12W64-NEXT:  ; %bb.2: ; %ComputeEnd861; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0862; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)863; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1864; GFX12W64-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1865; GFX12W64-NEXT:    ; implicit-def: $vgpr1866; GFX12W64-NEXT:    s_and_saveexec_b64 s[0:1], vcc867; GFX12W64-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)868; GFX12W64-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]869; GFX12W64-NEXT:    s_cbranch_execz .LBB2_4870; GFX12W64-NEXT:  ; %bb.3:871; GFX12W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34872; GFX12W64-NEXT:    v_mov_b32_e32 v1, s2873; GFX12W64-NEXT:    s_wait_kmcnt 0x0874; GFX12W64-NEXT:    buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN875; GFX12W64-NEXT:  .LBB2_4:876; GFX12W64-NEXT:    s_or_b64 exec, exec, s[0:1]877; GFX12W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24878; GFX12W64-NEXT:    s_wait_loadcnt 0x0879; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1880; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0881; GFX12W64-NEXT:    s_wait_alu depctr_va_sdst(0)882; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)883; GFX12W64-NEXT:    v_add_nc_u32_e32 v0, s2, v0884; GFX12W64-NEXT:    s_wait_kmcnt 0x0885; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]886; GFX12W64-NEXT:    s_endpgm887;888; GFX12W32-LABEL: add_i32_varying_vdata:889; GFX12W32:       ; %bb.0: ; %entry890; GFX12W32-NEXT:    v_and_b32_e32 v1, 0x3ff, v0891; GFX12W32-NEXT:    s_mov_b32 s1, exec_lo892; GFX12W32-NEXT:    s_mov_b32 s0, 0893; GFX12W32-NEXT:    ; implicit-def: $vgpr0894; GFX12W32-NEXT:  .LBB2_1: ; %ComputeLoop895; GFX12W32-NEXT:    ; =>This Inner Loop Header: Depth=1896; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)897; GFX12W32-NEXT:    s_ctz_i32_b32 s2, s1898; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)899; GFX12W32-NEXT:    v_readlane_b32 s3, v1, s2900; GFX12W32-NEXT:    v_writelane_b32 v0, s0, s2901; GFX12W32-NEXT:    s_lshl_b32 s2, 1, s2902; GFX12W32-NEXT:    s_add_co_i32 s0, s0, s3903; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)904; GFX12W32-NEXT:    s_and_not1_b32 s1, s1, s2905; GFX12W32-NEXT:    s_cbranch_scc1 .LBB2_1906; GFX12W32-NEXT:  ; %bb.2: ; %ComputeEnd907; GFX12W32-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0908; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)909; GFX12W32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1910; GFX12W32-NEXT:    ; implicit-def: $vgpr1911; GFX12W32-NEXT:    s_and_saveexec_b32 s1, vcc_lo912; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)913; GFX12W32-NEXT:    s_xor_b32 s1, exec_lo, s1914; GFX12W32-NEXT:    s_cbranch_execz .LBB2_4915; GFX12W32-NEXT:  ; %bb.3:916; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34917; GFX12W32-NEXT:    v_mov_b32_e32 v1, s0918; GFX12W32-NEXT:    s_wait_kmcnt 0x0919; GFX12W32-NEXT:    buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN920; GFX12W32-NEXT:  .LBB2_4:921; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)922; GFX12W32-NEXT:    s_or_b32 exec_lo, exec_lo, s1923; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24924; GFX12W32-NEXT:    s_wait_loadcnt 0x0925; GFX12W32-NEXT:    v_readfirstlane_b32 s2, v1926; GFX12W32-NEXT:    s_wait_alu depctr_va_sdst(0)927; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)928; GFX12W32-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, s2, v0929; GFX12W32-NEXT:    s_wait_kmcnt 0x0930; GFX12W32-NEXT:    global_store_b32 v1, v0, s[0:1]931; GFX12W32-NEXT:    s_endpgm932entry:933  %lane = call i32 @llvm.amdgcn.workitem.id.x()934  %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32 %lane, ptr addrspace(8) %inout, i32 0, i32 0, i32 0)935  store i32 %old, ptr addrspace(1) %out936  ret void937}938 939define amdgpu_kernel void @add_i32_varying_offset(ptr addrspace(1) %out, ptr addrspace(8) %inout) {940; GFX6-LABEL: add_i32_varying_offset:941; GFX6:       ; %bb.0: ; %entry942; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xd943; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9944; GFX6-NEXT:    v_mov_b32_e32 v1, 1945; GFX6-NEXT:    s_waitcnt lgkmcnt(0)946; GFX6-NEXT:    buffer_atomic_add v1, v0, s[0:3], 0 offen glc947; GFX6-NEXT:    s_mov_b32 s7, 0xf000948; GFX6-NEXT:    s_mov_b32 s6, -1949; GFX6-NEXT:    s_waitcnt vmcnt(0)950; GFX6-NEXT:    buffer_store_dword v1, off, s[4:7], 0951; GFX6-NEXT:    s_endpgm952;953; GFX8-LABEL: add_i32_varying_offset:954; GFX8:       ; %bb.0: ; %entry955; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34956; GFX8-NEXT:    v_mov_b32_e32 v2, 1957; GFX8-NEXT:    s_waitcnt lgkmcnt(0)958; GFX8-NEXT:    buffer_atomic_add v2, v0, s[0:3], 0 offen glc959; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24960; GFX8-NEXT:    s_waitcnt lgkmcnt(0)961; GFX8-NEXT:    v_mov_b32_e32 v0, s0962; GFX8-NEXT:    v_mov_b32_e32 v1, s1963; GFX8-NEXT:    s_waitcnt vmcnt(0)964; GFX8-NEXT:    flat_store_dword v[0:1], v2965; GFX8-NEXT:    s_endpgm966;967; GFX9-LABEL: add_i32_varying_offset:968; GFX9:       ; %bb.0: ; %entry969; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34970; GFX9-NEXT:    v_mov_b32_e32 v1, 1971; GFX9-NEXT:    s_waitcnt lgkmcnt(0)972; GFX9-NEXT:    buffer_atomic_add v1, v0, s[0:3], 0 offen glc973; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24974; GFX9-NEXT:    v_mov_b32_e32 v0, 0975; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)976; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]977; GFX9-NEXT:    s_endpgm978;979; GFX10-LABEL: add_i32_varying_offset:980; GFX10:       ; %bb.0: ; %entry981; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34982; GFX10-NEXT:    v_mov_b32_e32 v1, 1983; GFX10-NEXT:    s_waitcnt lgkmcnt(0)984; GFX10-NEXT:    buffer_atomic_add v1, v0, s[0:3], 0 offen glc985; GFX10-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)986; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24987; GFX10-NEXT:    v_mov_b32_e32 v0, 0988; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)989; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]990; GFX10-NEXT:    s_endpgm991;992; GFX11W64-LABEL: add_i32_varying_offset:993; GFX11W64:       ; %bb.0: ; %entry994; GFX11W64-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34995; GFX11W64-NEXT:    v_and_b32_e32 v0, 0x3ff, v0996; GFX11W64-NEXT:    v_mov_b32_e32 v1, 1997; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)998; GFX11W64-NEXT:    buffer_atomic_add_u32 v1, v0, s[0:3], 0 offen glc999; GFX11W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241000; GFX11W64-NEXT:    v_mov_b32_e32 v0, 01001; GFX11W64-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1002; GFX11W64-NEXT:    global_store_b32 v0, v1, s[0:1]1003; GFX11W64-NEXT:    s_endpgm1004;1005; GFX11W32-LABEL: add_i32_varying_offset:1006; GFX11W32:       ; %bb.0: ; %entry1007; GFX11W32-NEXT:    s_load_b128 s[0:3], s[4:5], 0x341008; GFX11W32-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v01009; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)1010; GFX11W32-NEXT:    buffer_atomic_add_u32 v1, v0, s[0:3], 0 offen glc1011; GFX11W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241012; GFX11W32-NEXT:    v_mov_b32_e32 v0, 01013; GFX11W32-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1014; GFX11W32-NEXT:    global_store_b32 v0, v1, s[0:1]1015; GFX11W32-NEXT:    s_endpgm1016;1017; GFX12W64-LABEL: add_i32_varying_offset:1018; GFX12W64:       ; %bb.0: ; %entry1019; GFX12W64-NEXT:    s_load_b128 s[0:3], s[4:5], 0x341020; GFX12W64-NEXT:    v_and_b32_e32 v0, 0x3ff, v01021; GFX12W64-NEXT:    v_mov_b32_e32 v1, 11022; GFX12W64-NEXT:    s_wait_kmcnt 0x01023; GFX12W64-NEXT:    buffer_atomic_add_u32 v1, v0, s[0:3], null offen th:TH_ATOMIC_RETURN1024; GFX12W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241025; GFX12W64-NEXT:    v_mov_b32_e32 v0, 01026; GFX12W64-NEXT:    s_wait_loadcnt 0x01027; GFX12W64-NEXT:    s_wait_kmcnt 0x01028; GFX12W64-NEXT:    global_store_b32 v0, v1, s[0:1]1029; GFX12W64-NEXT:    s_endpgm1030;1031; GFX12W32-LABEL: add_i32_varying_offset:1032; GFX12W32:       ; %bb.0: ; %entry1033; GFX12W32-NEXT:    s_load_b128 s[0:3], s[4:5], 0x341034; GFX12W32-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v01035; GFX12W32-NEXT:    s_wait_kmcnt 0x01036; GFX12W32-NEXT:    buffer_atomic_add_u32 v1, v0, s[0:3], null offen th:TH_ATOMIC_RETURN1037; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241038; GFX12W32-NEXT:    v_mov_b32_e32 v0, 01039; GFX12W32-NEXT:    s_wait_loadcnt 0x01040; GFX12W32-NEXT:    s_wait_kmcnt 0x01041; GFX12W32-NEXT:    global_store_b32 v0, v1, s[0:1]1042; GFX12W32-NEXT:    s_endpgm1043entry:1044  %lane = call i32 @llvm.amdgcn.workitem.id.x()1045  %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32 1, ptr addrspace(8) %inout, i32 %lane, i32 0, i32 0)1046  store i32 %old, ptr addrspace(1) %out1047  ret void1048}1049 1050define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace(8) %inout) {1051; GFX6-LABEL: sub_i32_constant:1052; GFX6:       ; %bb.0: ; %entry1053; GFX6-NEXT:    s_mov_b64 s[2:3], exec1054; GFX6-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 01055; GFX6-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v01056; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01057; GFX6-NEXT:    ; implicit-def: $vgpr11058; GFX6-NEXT:    s_and_saveexec_b64 s[0:1], vcc1059; GFX6-NEXT:    s_cbranch_execz .LBB4_21060; GFX6-NEXT:  ; %bb.1:1061; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xd1062; GFX6-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1063; GFX6-NEXT:    s_mul_i32 s2, s2, 51064; GFX6-NEXT:    v_mov_b32_e32 v1, s21065; GFX6-NEXT:    s_waitcnt lgkmcnt(0)1066; GFX6-NEXT:    buffer_atomic_sub v1, off, s[8:11], 0 glc1067; GFX6-NEXT:  .LBB4_2:1068; GFX6-NEXT:    s_or_b64 exec, exec, s[0:1]1069; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x91070; GFX6-NEXT:    s_mov_b32 s3, 0xf0001071; GFX6-NEXT:    s_mov_b32 s2, -11072; GFX6-NEXT:    s_waitcnt vmcnt(0)1073; GFX6-NEXT:    v_readfirstlane_b32 s4, v11074; GFX6-NEXT:    v_mul_u32_u24_e32 v0, 5, v01075; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s4, v01076; GFX6-NEXT:    s_waitcnt lgkmcnt(0)1077; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 01078; GFX6-NEXT:    s_endpgm1079;1080; GFX8-LABEL: sub_i32_constant:1081; GFX8:       ; %bb.0: ; %entry1082; GFX8-NEXT:    s_mov_b64 s[2:3], exec1083; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01084; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v01085; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01086; GFX8-NEXT:    ; implicit-def: $vgpr11087; GFX8-NEXT:    s_and_saveexec_b64 s[0:1], vcc1088; GFX8-NEXT:    s_cbranch_execz .LBB4_21089; GFX8-NEXT:  ; %bb.1:1090; GFX8-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341091; GFX8-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1092; GFX8-NEXT:    s_mul_i32 s2, s2, 51093; GFX8-NEXT:    v_mov_b32_e32 v1, s21094; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1095; GFX8-NEXT:    buffer_atomic_sub v1, off, s[8:11], 0 glc1096; GFX8-NEXT:  .LBB4_2:1097; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]1098; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241099; GFX8-NEXT:    s_waitcnt vmcnt(0)1100; GFX8-NEXT:    v_readfirstlane_b32 s2, v11101; GFX8-NEXT:    v_mul_u32_u24_e32 v0, 5, v01102; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, s2, v01103; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1104; GFX8-NEXT:    v_mov_b32_e32 v0, s01105; GFX8-NEXT:    v_mov_b32_e32 v1, s11106; GFX8-NEXT:    flat_store_dword v[0:1], v21107; GFX8-NEXT:    s_endpgm1108;1109; GFX9-LABEL: sub_i32_constant:1110; GFX9:       ; %bb.0: ; %entry1111; GFX9-NEXT:    s_mov_b64 s[2:3], exec1112; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01113; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v01114; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01115; GFX9-NEXT:    ; implicit-def: $vgpr11116; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc1117; GFX9-NEXT:    s_cbranch_execz .LBB4_21118; GFX9-NEXT:  ; %bb.1:1119; GFX9-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341120; GFX9-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1121; GFX9-NEXT:    s_mul_i32 s2, s2, 51122; GFX9-NEXT:    v_mov_b32_e32 v1, s21123; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1124; GFX9-NEXT:    buffer_atomic_sub v1, off, s[8:11], 0 glc1125; GFX9-NEXT:  .LBB4_2:1126; GFX9-NEXT:    s_or_b64 exec, exec, s[0:1]1127; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241128; GFX9-NEXT:    s_waitcnt vmcnt(0)1129; GFX9-NEXT:    v_readfirstlane_b32 s2, v11130; GFX9-NEXT:    v_mul_u32_u24_e32 v0, 5, v01131; GFX9-NEXT:    v_mov_b32_e32 v2, 01132; GFX9-NEXT:    v_sub_u32_e32 v0, s2, v01133; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1134; GFX9-NEXT:    global_store_dword v2, v0, s[0:1]1135; GFX9-NEXT:    s_endpgm1136;1137; GFX10W64-LABEL: sub_i32_constant:1138; GFX10W64:       ; %bb.0: ; %entry1139; GFX10W64-NEXT:    s_mov_b64 s[2:3], exec1140; GFX10W64-NEXT:    ; implicit-def: $vgpr11141; GFX10W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01142; GFX10W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v01143; GFX10W64-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01144; GFX10W64-NEXT:    s_and_saveexec_b64 s[0:1], vcc1145; GFX10W64-NEXT:    s_cbranch_execz .LBB4_21146; GFX10W64-NEXT:  ; %bb.1:1147; GFX10W64-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341148; GFX10W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1149; GFX10W64-NEXT:    s_mul_i32 s2, s2, 51150; GFX10W64-NEXT:    v_mov_b32_e32 v1, s21151; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)1152; GFX10W64-NEXT:    buffer_atomic_sub v1, off, s[8:11], 0 glc1153; GFX10W64-NEXT:  .LBB4_2:1154; GFX10W64-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)1155; GFX10W64-NEXT:    s_or_b64 exec, exec, s[0:1]1156; GFX10W64-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241157; GFX10W64-NEXT:    s_waitcnt vmcnt(0)1158; GFX10W64-NEXT:    v_readfirstlane_b32 s2, v11159; GFX10W64-NEXT:    v_mul_u32_u24_e32 v0, 5, v01160; GFX10W64-NEXT:    v_mov_b32_e32 v1, 01161; GFX10W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v01162; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)1163; GFX10W64-NEXT:    global_store_dword v1, v0, s[0:1]1164; GFX10W64-NEXT:    s_endpgm1165;1166; GFX10W32-LABEL: sub_i32_constant:1167; GFX10W32:       ; %bb.0: ; %entry1168; GFX10W32-NEXT:    s_mov_b32 s1, exec_lo1169; GFX10W32-NEXT:    ; implicit-def: $vgpr11170; GFX10W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s1, 01171; GFX10W32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v01172; GFX10W32-NEXT:    s_and_saveexec_b32 s0, vcc_lo1173; GFX10W32-NEXT:    s_cbranch_execz .LBB4_21174; GFX10W32-NEXT:  ; %bb.1:1175; GFX10W32-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341176; GFX10W32-NEXT:    s_bcnt1_i32_b32 s1, s11177; GFX10W32-NEXT:    s_mul_i32 s1, s1, 51178; GFX10W32-NEXT:    v_mov_b32_e32 v1, s11179; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)1180; GFX10W32-NEXT:    buffer_atomic_sub v1, off, s[8:11], 0 glc1181; GFX10W32-NEXT:  .LBB4_2:1182; GFX10W32-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)1183; GFX10W32-NEXT:    s_or_b32 exec_lo, exec_lo, s01184; GFX10W32-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241185; GFX10W32-NEXT:    s_waitcnt vmcnt(0)1186; GFX10W32-NEXT:    v_readfirstlane_b32 s2, v11187; GFX10W32-NEXT:    v_mul_u32_u24_e32 v0, 5, v01188; GFX10W32-NEXT:    v_mov_b32_e32 v1, 01189; GFX10W32-NEXT:    v_sub_nc_u32_e32 v0, s2, v01190; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)1191; GFX10W32-NEXT:    global_store_dword v1, v0, s[0:1]1192; GFX10W32-NEXT:    s_endpgm1193;1194; GFX11W64-LABEL: sub_i32_constant:1195; GFX11W64:       ; %bb.0: ; %entry1196; GFX11W64-NEXT:    s_mov_b64 s[2:3], exec1197; GFX11W64-NEXT:    s_mov_b64 s[0:1], exec1198; GFX11W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01199; GFX11W64-NEXT:    ; implicit-def: $vgpr11200; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1201; GFX11W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v01202; GFX11W64-NEXT:    v_cmpx_eq_u32_e32 0, v01203; GFX11W64-NEXT:    s_cbranch_execz .LBB4_21204; GFX11W64-NEXT:  ; %bb.1:1205; GFX11W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x341206; GFX11W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1207; GFX11W64-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)1208; GFX11W64-NEXT:    s_mul_i32 s2, s2, 51209; GFX11W64-NEXT:    v_mov_b32_e32 v1, s21210; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)1211; GFX11W64-NEXT:    buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc1212; GFX11W64-NEXT:  .LBB4_2:1213; GFX11W64-NEXT:    s_or_b64 exec, exec, s[0:1]1214; GFX11W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241215; GFX11W64-NEXT:    s_waitcnt vmcnt(0)1216; GFX11W64-NEXT:    v_readfirstlane_b32 s2, v11217; GFX11W64-NEXT:    v_mul_u32_u24_e32 v0, 5, v01218; GFX11W64-NEXT:    v_mov_b32_e32 v1, 01219; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)1220; GFX11W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v01221; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)1222; GFX11W64-NEXT:    global_store_b32 v1, v0, s[0:1]1223; GFX11W64-NEXT:    s_endpgm1224;1225; GFX11W32-LABEL: sub_i32_constant:1226; GFX11W32:       ; %bb.0: ; %entry1227; GFX11W32-NEXT:    s_mov_b32 s1, exec_lo1228; GFX11W32-NEXT:    s_mov_b32 s0, exec_lo1229; GFX11W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s1, 01230; GFX11W32-NEXT:    ; implicit-def: $vgpr11231; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)1232; GFX11W32-NEXT:    v_cmpx_eq_u32_e32 0, v01233; GFX11W32-NEXT:    s_cbranch_execz .LBB4_21234; GFX11W32-NEXT:  ; %bb.1:1235; GFX11W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x341236; GFX11W32-NEXT:    s_bcnt1_i32_b32 s1, s11237; GFX11W32-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)1238; GFX11W32-NEXT:    s_mul_i32 s1, s1, 51239; GFX11W32-NEXT:    v_mov_b32_e32 v1, s11240; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)1241; GFX11W32-NEXT:    buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc1242; GFX11W32-NEXT:  .LBB4_2:1243; GFX11W32-NEXT:    s_or_b32 exec_lo, exec_lo, s01244; GFX11W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241245; GFX11W32-NEXT:    s_waitcnt vmcnt(0)1246; GFX11W32-NEXT:    v_readfirstlane_b32 s2, v11247; GFX11W32-NEXT:    v_mul_u32_u24_e32 v0, 5, v01248; GFX11W32-NEXT:    v_mov_b32_e32 v1, 01249; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)1250; GFX11W32-NEXT:    v_sub_nc_u32_e32 v0, s2, v01251; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)1252; GFX11W32-NEXT:    global_store_b32 v1, v0, s[0:1]1253; GFX11W32-NEXT:    s_endpgm1254;1255; GFX12W64-LABEL: sub_i32_constant:1256; GFX12W64:       ; %bb.0: ; %entry1257; GFX12W64-NEXT:    s_mov_b64 s[2:3], exec1258; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec1259; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01260; GFX12W64-NEXT:    ; implicit-def: $vgpr11261; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1262; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v01263; GFX12W64-NEXT:    v_cmpx_eq_u32_e32 0, v01264; GFX12W64-NEXT:    s_cbranch_execz .LBB4_21265; GFX12W64-NEXT:  ; %bb.1:1266; GFX12W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x341267; GFX12W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1268; GFX12W64-NEXT:    s_wait_alu depctr_sa_sdst(0)1269; GFX12W64-NEXT:    s_mul_i32 s2, s2, 51270; GFX12W64-NEXT:    s_wait_alu depctr_sa_sdst(0)1271; GFX12W64-NEXT:    v_mov_b32_e32 v1, s21272; GFX12W64-NEXT:    s_wait_kmcnt 0x01273; GFX12W64-NEXT:    buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN1274; GFX12W64-NEXT:  .LBB4_2:1275; GFX12W64-NEXT:    s_or_b64 exec, exec, s[0:1]1276; GFX12W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241277; GFX12W64-NEXT:    s_wait_loadcnt 0x01278; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v11279; GFX12W64-NEXT:    v_mul_u32_u24_e32 v0, 5, v01280; GFX12W64-NEXT:    v_mov_b32_e32 v1, 01281; GFX12W64-NEXT:    s_wait_alu depctr_va_sdst(0)1282; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)1283; GFX12W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v01284; GFX12W64-NEXT:    s_wait_kmcnt 0x01285; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]1286; GFX12W64-NEXT:    s_endpgm1287;1288; GFX12W32-LABEL: sub_i32_constant:1289; GFX12W32:       ; %bb.0: ; %entry1290; GFX12W32-NEXT:    s_mov_b32 s1, exec_lo1291; GFX12W32-NEXT:    s_mov_b32 s0, exec_lo1292; GFX12W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s1, 01293; GFX12W32-NEXT:    ; implicit-def: $vgpr11294; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)1295; GFX12W32-NEXT:    v_cmpx_eq_u32_e32 0, v01296; GFX12W32-NEXT:    s_cbranch_execz .LBB4_21297; GFX12W32-NEXT:  ; %bb.1:1298; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x341299; GFX12W32-NEXT:    s_bcnt1_i32_b32 s1, s11300; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)1301; GFX12W32-NEXT:    s_mul_i32 s1, s1, 51302; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)1303; GFX12W32-NEXT:    v_mov_b32_e32 v1, s11304; GFX12W32-NEXT:    s_wait_kmcnt 0x01305; GFX12W32-NEXT:    buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN1306; GFX12W32-NEXT:  .LBB4_2:1307; GFX12W32-NEXT:    s_or_b32 exec_lo, exec_lo, s01308; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241309; GFX12W32-NEXT:    s_wait_loadcnt 0x01310; GFX12W32-NEXT:    v_readfirstlane_b32 s2, v11311; GFX12W32-NEXT:    v_mul_u32_u24_e32 v0, 5, v01312; GFX12W32-NEXT:    v_mov_b32_e32 v1, 01313; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)1314; GFX12W32-NEXT:    v_sub_nc_u32_e32 v0, s2, v01315; GFX12W32-NEXT:    s_wait_kmcnt 0x01316; GFX12W32-NEXT:    global_store_b32 v1, v0, s[0:1]1317; GFX12W32-NEXT:    s_endpgm1318entry:1319  %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.sub(i32 5, ptr addrspace(8) %inout, i32 0, i32 0, i32 0)1320  store i32 %old, ptr addrspace(1) %out1321  ret void1322}1323 1324define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(8) %inout, i32 %subitive) {1325; GFX6-LABEL: sub_i32_uniform:1326; GFX6:       ; %bb.0: ; %entry1327; GFX6-NEXT:    s_mov_b64 s[2:3], exec1328; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x111329; GFX6-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 01330; GFX6-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v01331; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01332; GFX6-NEXT:    ; implicit-def: $vgpr11333; GFX6-NEXT:    s_and_saveexec_b64 s[0:1], vcc1334; GFX6-NEXT:    s_cbranch_execz .LBB5_21335; GFX6-NEXT:  ; %bb.1:1336; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xd1337; GFX6-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1338; GFX6-NEXT:    s_waitcnt lgkmcnt(0)1339; GFX6-NEXT:    s_mul_i32 s2, s6, s21340; GFX6-NEXT:    v_mov_b32_e32 v1, s21341; GFX6-NEXT:    buffer_atomic_sub v1, off, s[8:11], 0 glc1342; GFX6-NEXT:  .LBB5_2:1343; GFX6-NEXT:    s_or_b64 exec, exec, s[0:1]1344; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x91345; GFX6-NEXT:    s_mov_b32 s3, 0xf0001346; GFX6-NEXT:    s_mov_b32 s2, -11347; GFX6-NEXT:    s_waitcnt vmcnt(0)1348; GFX6-NEXT:    v_readfirstlane_b32 s4, v11349; GFX6-NEXT:    s_waitcnt lgkmcnt(0)1350; GFX6-NEXT:    v_mul_lo_u32 v0, s6, v01351; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s4, v01352; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 01353; GFX6-NEXT:    s_endpgm1354;1355; GFX8-LABEL: sub_i32_uniform:1356; GFX8:       ; %bb.0: ; %entry1357; GFX8-NEXT:    s_load_dword s6, s[4:5], 0x441358; GFX8-NEXT:    s_mov_b64 s[2:3], exec1359; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01360; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v01361; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01362; GFX8-NEXT:    ; implicit-def: $vgpr11363; GFX8-NEXT:    s_and_saveexec_b64 s[0:1], vcc1364; GFX8-NEXT:    s_cbranch_execz .LBB5_21365; GFX8-NEXT:  ; %bb.1:1366; GFX8-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341367; GFX8-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1368; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1369; GFX8-NEXT:    s_mul_i32 s2, s6, s21370; GFX8-NEXT:    v_mov_b32_e32 v1, s21371; GFX8-NEXT:    buffer_atomic_sub v1, off, s[8:11], 0 glc1372; GFX8-NEXT:  .LBB5_2:1373; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]1374; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241375; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1376; GFX8-NEXT:    v_mul_lo_u32 v0, s6, v01377; GFX8-NEXT:    s_waitcnt vmcnt(0)1378; GFX8-NEXT:    v_readfirstlane_b32 s2, v11379; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, s2, v01380; GFX8-NEXT:    v_mov_b32_e32 v0, s01381; GFX8-NEXT:    v_mov_b32_e32 v1, s11382; GFX8-NEXT:    flat_store_dword v[0:1], v21383; GFX8-NEXT:    s_endpgm1384;1385; GFX9-LABEL: sub_i32_uniform:1386; GFX9:       ; %bb.0: ; %entry1387; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x441388; GFX9-NEXT:    s_mov_b64 s[2:3], exec1389; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01390; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v01391; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01392; GFX9-NEXT:    ; implicit-def: $vgpr11393; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc1394; GFX9-NEXT:    s_cbranch_execz .LBB5_21395; GFX9-NEXT:  ; %bb.1:1396; GFX9-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341397; GFX9-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1398; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1399; GFX9-NEXT:    s_mul_i32 s2, s6, s21400; GFX9-NEXT:    v_mov_b32_e32 v1, s21401; GFX9-NEXT:    buffer_atomic_sub v1, off, s[8:11], 0 glc1402; GFX9-NEXT:  .LBB5_2:1403; GFX9-NEXT:    s_or_b64 exec, exec, s[0:1]1404; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241405; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1406; GFX9-NEXT:    v_mul_lo_u32 v0, s6, v01407; GFX9-NEXT:    s_waitcnt vmcnt(0)1408; GFX9-NEXT:    v_readfirstlane_b32 s2, v11409; GFX9-NEXT:    v_mov_b32_e32 v2, 01410; GFX9-NEXT:    v_sub_u32_e32 v0, s2, v01411; GFX9-NEXT:    global_store_dword v2, v0, s[0:1]1412; GFX9-NEXT:    s_endpgm1413;1414; GFX10W64-LABEL: sub_i32_uniform:1415; GFX10W64:       ; %bb.0: ; %entry1416; GFX10W64-NEXT:    s_load_dword s6, s[4:5], 0x441417; GFX10W64-NEXT:    s_mov_b64 s[2:3], exec1418; GFX10W64-NEXT:    ; implicit-def: $vgpr11419; GFX10W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01420; GFX10W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v01421; GFX10W64-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01422; GFX10W64-NEXT:    s_and_saveexec_b64 s[0:1], vcc1423; GFX10W64-NEXT:    s_cbranch_execz .LBB5_21424; GFX10W64-NEXT:  ; %bb.1:1425; GFX10W64-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341426; GFX10W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1427; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)1428; GFX10W64-NEXT:    s_mul_i32 s2, s6, s21429; GFX10W64-NEXT:    v_mov_b32_e32 v1, s21430; GFX10W64-NEXT:    buffer_atomic_sub v1, off, s[8:11], 0 glc1431; GFX10W64-NEXT:  .LBB5_2:1432; GFX10W64-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)1433; GFX10W64-NEXT:    s_or_b64 exec, exec, s[0:1]1434; GFX10W64-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241435; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)1436; GFX10W64-NEXT:    v_mul_lo_u32 v0, s6, v01437; GFX10W64-NEXT:    s_waitcnt vmcnt(0)1438; GFX10W64-NEXT:    v_readfirstlane_b32 s2, v11439; GFX10W64-NEXT:    v_mov_b32_e32 v1, 01440; GFX10W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v01441; GFX10W64-NEXT:    global_store_dword v1, v0, s[0:1]1442; GFX10W64-NEXT:    s_endpgm1443;1444; GFX10W32-LABEL: sub_i32_uniform:1445; GFX10W32:       ; %bb.0: ; %entry1446; GFX10W32-NEXT:    s_load_dword s0, s[4:5], 0x441447; GFX10W32-NEXT:    s_mov_b32 s2, exec_lo1448; GFX10W32-NEXT:    ; implicit-def: $vgpr11449; GFX10W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01450; GFX10W32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v01451; GFX10W32-NEXT:    s_and_saveexec_b32 s1, vcc_lo1452; GFX10W32-NEXT:    s_cbranch_execz .LBB5_21453; GFX10W32-NEXT:  ; %bb.1:1454; GFX10W32-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341455; GFX10W32-NEXT:    s_bcnt1_i32_b32 s2, s21456; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)1457; GFX10W32-NEXT:    s_mul_i32 s2, s0, s21458; GFX10W32-NEXT:    v_mov_b32_e32 v1, s21459; GFX10W32-NEXT:    buffer_atomic_sub v1, off, s[8:11], 0 glc1460; GFX10W32-NEXT:  .LBB5_2:1461; GFX10W32-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)1462; GFX10W32-NEXT:    s_or_b32 exec_lo, exec_lo, s11463; GFX10W32-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x241464; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)1465; GFX10W32-NEXT:    v_mul_lo_u32 v0, s0, v01466; GFX10W32-NEXT:    s_waitcnt vmcnt(0)1467; GFX10W32-NEXT:    v_readfirstlane_b32 s0, v11468; GFX10W32-NEXT:    v_mov_b32_e32 v1, 01469; GFX10W32-NEXT:    v_sub_nc_u32_e32 v0, s0, v01470; GFX10W32-NEXT:    global_store_dword v1, v0, s[2:3]1471; GFX10W32-NEXT:    s_endpgm1472;1473; GFX11W64-LABEL: sub_i32_uniform:1474; GFX11W64:       ; %bb.0: ; %entry1475; GFX11W64-NEXT:    s_load_b32 s6, s[4:5], 0x441476; GFX11W64-NEXT:    s_mov_b64 s[2:3], exec1477; GFX11W64-NEXT:    s_mov_b64 s[0:1], exec1478; GFX11W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01479; GFX11W64-NEXT:    ; implicit-def: $vgpr11480; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1481; GFX11W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v01482; GFX11W64-NEXT:    v_cmpx_eq_u32_e32 0, v01483; GFX11W64-NEXT:    s_cbranch_execz .LBB5_21484; GFX11W64-NEXT:  ; %bb.1:1485; GFX11W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x341486; GFX11W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1487; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)1488; GFX11W64-NEXT:    s_mul_i32 s2, s6, s21489; GFX11W64-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)1490; GFX11W64-NEXT:    v_mov_b32_e32 v1, s21491; GFX11W64-NEXT:    buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc1492; GFX11W64-NEXT:  .LBB5_2:1493; GFX11W64-NEXT:    s_or_b64 exec, exec, s[0:1]1494; GFX11W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241495; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)1496; GFX11W64-NEXT:    v_mul_lo_u32 v0, s6, v01497; GFX11W64-NEXT:    s_waitcnt vmcnt(0)1498; GFX11W64-NEXT:    v_readfirstlane_b32 s2, v11499; GFX11W64-NEXT:    v_mov_b32_e32 v1, 01500; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)1501; GFX11W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v01502; GFX11W64-NEXT:    global_store_b32 v1, v0, s[0:1]1503; GFX11W64-NEXT:    s_endpgm1504;1505; GFX11W32-LABEL: sub_i32_uniform:1506; GFX11W32:       ; %bb.0: ; %entry1507; GFX11W32-NEXT:    s_load_b32 s0, s[4:5], 0x441508; GFX11W32-NEXT:    s_mov_b32 s2, exec_lo1509; GFX11W32-NEXT:    s_mov_b32 s1, exec_lo1510; GFX11W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01511; GFX11W32-NEXT:    ; implicit-def: $vgpr11512; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)1513; GFX11W32-NEXT:    v_cmpx_eq_u32_e32 0, v01514; GFX11W32-NEXT:    s_cbranch_execz .LBB5_21515; GFX11W32-NEXT:  ; %bb.1:1516; GFX11W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x341517; GFX11W32-NEXT:    s_bcnt1_i32_b32 s2, s21518; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)1519; GFX11W32-NEXT:    s_mul_i32 s2, s0, s21520; GFX11W32-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)1521; GFX11W32-NEXT:    v_mov_b32_e32 v1, s21522; GFX11W32-NEXT:    buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc1523; GFX11W32-NEXT:  .LBB5_2:1524; GFX11W32-NEXT:    s_or_b32 exec_lo, exec_lo, s11525; GFX11W32-NEXT:    s_load_b64 s[2:3], s[4:5], 0x241526; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)1527; GFX11W32-NEXT:    v_mul_lo_u32 v0, s0, v01528; GFX11W32-NEXT:    s_waitcnt vmcnt(0)1529; GFX11W32-NEXT:    v_readfirstlane_b32 s0, v11530; GFX11W32-NEXT:    v_mov_b32_e32 v1, 01531; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)1532; GFX11W32-NEXT:    v_sub_nc_u32_e32 v0, s0, v01533; GFX11W32-NEXT:    global_store_b32 v1, v0, s[2:3]1534; GFX11W32-NEXT:    s_endpgm1535;1536; GFX12W64-LABEL: sub_i32_uniform:1537; GFX12W64:       ; %bb.0: ; %entry1538; GFX12W64-NEXT:    s_load_b32 s6, s[4:5], 0x441539; GFX12W64-NEXT:    s_mov_b64 s[2:3], exec1540; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec1541; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01542; GFX12W64-NEXT:    ; implicit-def: $vgpr11543; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1544; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v01545; GFX12W64-NEXT:    v_cmpx_eq_u32_e32 0, v01546; GFX12W64-NEXT:    s_cbranch_execz .LBB5_21547; GFX12W64-NEXT:  ; %bb.1:1548; GFX12W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x341549; GFX12W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1550; GFX12W64-NEXT:    s_wait_kmcnt 0x01551; GFX12W64-NEXT:    s_wait_alu depctr_sa_sdst(0)1552; GFX12W64-NEXT:    s_mul_i32 s2, s6, s21553; GFX12W64-NEXT:    s_wait_alu depctr_sa_sdst(0)1554; GFX12W64-NEXT:    v_mov_b32_e32 v1, s21555; GFX12W64-NEXT:    buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN1556; GFX12W64-NEXT:  .LBB5_2:1557; GFX12W64-NEXT:    s_or_b64 exec, exec, s[0:1]1558; GFX12W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241559; GFX12W64-NEXT:    s_wait_kmcnt 0x01560; GFX12W64-NEXT:    v_mul_lo_u32 v0, s6, v01561; GFX12W64-NEXT:    s_wait_loadcnt 0x01562; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v11563; GFX12W64-NEXT:    v_mov_b32_e32 v1, 01564; GFX12W64-NEXT:    s_wait_alu depctr_va_sdst(0)1565; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)1566; GFX12W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v01567; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]1568; GFX12W64-NEXT:    s_endpgm1569;1570; GFX12W32-LABEL: sub_i32_uniform:1571; GFX12W32:       ; %bb.0: ; %entry1572; GFX12W32-NEXT:    s_load_b32 s0, s[4:5], 0x441573; GFX12W32-NEXT:    s_mov_b32 s2, exec_lo1574; GFX12W32-NEXT:    s_mov_b32 s1, exec_lo1575; GFX12W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01576; GFX12W32-NEXT:    ; implicit-def: $vgpr11577; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)1578; GFX12W32-NEXT:    v_cmpx_eq_u32_e32 0, v01579; GFX12W32-NEXT:    s_cbranch_execz .LBB5_21580; GFX12W32-NEXT:  ; %bb.1:1581; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x341582; GFX12W32-NEXT:    s_bcnt1_i32_b32 s2, s21583; GFX12W32-NEXT:    s_wait_kmcnt 0x01584; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)1585; GFX12W32-NEXT:    s_mul_i32 s2, s0, s21586; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)1587; GFX12W32-NEXT:    v_mov_b32_e32 v1, s21588; GFX12W32-NEXT:    buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN1589; GFX12W32-NEXT:  .LBB5_2:1590; GFX12W32-NEXT:    s_or_b32 exec_lo, exec_lo, s11591; GFX12W32-NEXT:    s_load_b64 s[2:3], s[4:5], 0x241592; GFX12W32-NEXT:    s_wait_kmcnt 0x01593; GFX12W32-NEXT:    v_mul_lo_u32 v0, s0, v01594; GFX12W32-NEXT:    s_wait_loadcnt 0x01595; GFX12W32-NEXT:    v_readfirstlane_b32 s0, v11596; GFX12W32-NEXT:    v_mov_b32_e32 v1, 01597; GFX12W32-NEXT:    s_wait_alu depctr_va_sdst(0)1598; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)1599; GFX12W32-NEXT:    v_sub_nc_u32_e32 v0, s0, v01600; GFX12W32-NEXT:    global_store_b32 v1, v0, s[2:3]1601; GFX12W32-NEXT:    s_endpgm1602entry:1603  %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.sub(i32 %subitive, ptr addrspace(8) %inout, i32 0, i32 0, i32 0)1604  store i32 %old, ptr addrspace(1) %out1605  ret void1606}1607 1608define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addrspace(8) %inout) {1609; GFX6-LABEL: sub_i32_varying_vdata:1610; GFX6:       ; %bb.0: ; %entry1611; GFX6-NEXT:    s_mov_b64 s[0:1], exec1612; GFX6-NEXT:    s_mov_b32 s2, 01613; GFX6-NEXT:    ; implicit-def: $vgpr11614; GFX6-NEXT:  .LBB6_1: ; %ComputeLoop1615; GFX6-NEXT:    ; =>This Inner Loop Header: Depth=11616; GFX6-NEXT:    s_ff1_i32_b64 s3, s[0:1]1617; GFX6-NEXT:    s_mov_b32 m0, s31618; GFX6-NEXT:    v_readlane_b32 s8, v0, s31619; GFX6-NEXT:    v_writelane_b32 v1, s2, m01620; GFX6-NEXT:    s_lshl_b64 s[6:7], 1, s31621; GFX6-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[6:7]1622; GFX6-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 01623; GFX6-NEXT:    s_and_b64 vcc, exec, s[6:7]1624; GFX6-NEXT:    s_add_i32 s2, s2, s81625; GFX6-NEXT:    s_cbranch_vccnz .LBB6_11626; GFX6-NEXT:  ; %bb.2: ; %ComputeEnd1627; GFX6-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 01628; GFX6-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v01629; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01630; GFX6-NEXT:    ; implicit-def: $vgpr01631; GFX6-NEXT:    s_and_saveexec_b64 s[0:1], vcc1632; GFX6-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]1633; GFX6-NEXT:    s_cbranch_execz .LBB6_41634; GFX6-NEXT:  ; %bb.3:1635; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xd1636; GFX6-NEXT:    v_mov_b32_e32 v0, s21637; GFX6-NEXT:    s_waitcnt lgkmcnt(0)1638; GFX6-NEXT:    buffer_atomic_sub v0, off, s[8:11], 0 glc1639; GFX6-NEXT:  .LBB6_4:1640; GFX6-NEXT:    s_or_b64 exec, exec, s[0:1]1641; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x91642; GFX6-NEXT:    s_mov_b32 s3, 0xf0001643; GFX6-NEXT:    s_mov_b32 s2, -11644; GFX6-NEXT:    s_waitcnt vmcnt(0)1645; GFX6-NEXT:    v_readfirstlane_b32 s4, v01646; GFX6-NEXT:    s_waitcnt expcnt(0)1647; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s4, v11648; GFX6-NEXT:    s_waitcnt lgkmcnt(0)1649; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 01650; GFX6-NEXT:    s_endpgm1651;1652; GFX8-LABEL: sub_i32_varying_vdata:1653; GFX8:       ; %bb.0: ; %entry1654; GFX8-NEXT:    s_mov_b64 s[0:1], exec1655; GFX8-NEXT:    s_mov_b32 s2, 01656; GFX8-NEXT:    ; implicit-def: $vgpr11657; GFX8-NEXT:  .LBB6_1: ; %ComputeLoop1658; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=11659; GFX8-NEXT:    s_ff1_i32_b64 s3, s[0:1]1660; GFX8-NEXT:    s_mov_b32 m0, s31661; GFX8-NEXT:    v_readlane_b32 s6, v0, s31662; GFX8-NEXT:    v_writelane_b32 v1, s2, m01663; GFX8-NEXT:    s_add_i32 s2, s2, s61664; GFX8-NEXT:    s_lshl_b64 s[6:7], 1, s31665; GFX8-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[6:7]1666; GFX8-NEXT:    s_cbranch_scc1 .LBB6_11667; GFX8-NEXT:  ; %bb.2: ; %ComputeEnd1668; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 01669; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v01670; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01671; GFX8-NEXT:    ; implicit-def: $vgpr01672; GFX8-NEXT:    s_and_saveexec_b64 s[0:1], vcc1673; GFX8-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]1674; GFX8-NEXT:    s_cbranch_execz .LBB6_41675; GFX8-NEXT:  ; %bb.3:1676; GFX8-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341677; GFX8-NEXT:    v_mov_b32_e32 v0, s21678; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1679; GFX8-NEXT:    buffer_atomic_sub v0, off, s[8:11], 0 glc1680; GFX8-NEXT:  .LBB6_4:1681; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]1682; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241683; GFX8-NEXT:    s_waitcnt vmcnt(0)1684; GFX8-NEXT:    v_readfirstlane_b32 s2, v01685; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, s2, v11686; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1687; GFX8-NEXT:    v_mov_b32_e32 v0, s01688; GFX8-NEXT:    v_mov_b32_e32 v1, s11689; GFX8-NEXT:    flat_store_dword v[0:1], v21690; GFX8-NEXT:    s_endpgm1691;1692; GFX9-LABEL: sub_i32_varying_vdata:1693; GFX9:       ; %bb.0: ; %entry1694; GFX9-NEXT:    s_mov_b64 s[0:1], exec1695; GFX9-NEXT:    s_mov_b32 s2, 01696; GFX9-NEXT:    ; implicit-def: $vgpr11697; GFX9-NEXT:  .LBB6_1: ; %ComputeLoop1698; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=11699; GFX9-NEXT:    s_ff1_i32_b64 s3, s[0:1]1700; GFX9-NEXT:    s_mov_b32 m0, s31701; GFX9-NEXT:    v_readlane_b32 s6, v0, s31702; GFX9-NEXT:    v_writelane_b32 v1, s2, m01703; GFX9-NEXT:    s_add_i32 s2, s2, s61704; GFX9-NEXT:    s_lshl_b64 s[6:7], 1, s31705; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[6:7]1706; GFX9-NEXT:    s_cbranch_scc1 .LBB6_11707; GFX9-NEXT:  ; %bb.2: ; %ComputeEnd1708; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 01709; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v01710; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01711; GFX9-NEXT:    ; implicit-def: $vgpr01712; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc1713; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]1714; GFX9-NEXT:    s_cbranch_execz .LBB6_41715; GFX9-NEXT:  ; %bb.3:1716; GFX9-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341717; GFX9-NEXT:    v_mov_b32_e32 v0, s21718; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1719; GFX9-NEXT:    buffer_atomic_sub v0, off, s[8:11], 0 glc1720; GFX9-NEXT:  .LBB6_4:1721; GFX9-NEXT:    s_or_b64 exec, exec, s[0:1]1722; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241723; GFX9-NEXT:    s_waitcnt vmcnt(0)1724; GFX9-NEXT:    v_readfirstlane_b32 s2, v01725; GFX9-NEXT:    v_mov_b32_e32 v2, 01726; GFX9-NEXT:    v_sub_u32_e32 v0, s2, v11727; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1728; GFX9-NEXT:    global_store_dword v2, v0, s[0:1]1729; GFX9-NEXT:    s_endpgm1730;1731; GFX10W64-LABEL: sub_i32_varying_vdata:1732; GFX10W64:       ; %bb.0: ; %entry1733; GFX10W64-NEXT:    s_mov_b64 s[0:1], exec1734; GFX10W64-NEXT:    s_mov_b32 s2, 01735; GFX10W64-NEXT:    ; implicit-def: $vgpr11736; GFX10W64-NEXT:  .LBB6_1: ; %ComputeLoop1737; GFX10W64-NEXT:    ; =>This Inner Loop Header: Depth=11738; GFX10W64-NEXT:    s_ff1_i32_b64 s3, s[0:1]1739; GFX10W64-NEXT:    v_readlane_b32 s8, v0, s31740; GFX10W64-NEXT:    v_writelane_b32 v1, s2, s31741; GFX10W64-NEXT:    s_lshl_b64 s[6:7], 1, s31742; GFX10W64-NEXT:    s_add_i32 s2, s2, s81743; GFX10W64-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[6:7]1744; GFX10W64-NEXT:    s_cbranch_scc1 .LBB6_11745; GFX10W64-NEXT:  ; %bb.2: ; %ComputeEnd1746; GFX10W64-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 01747; GFX10W64-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v01748; GFX10W64-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01749; GFX10W64-NEXT:    ; implicit-def: $vgpr01750; GFX10W64-NEXT:    s_and_saveexec_b64 s[0:1], vcc1751; GFX10W64-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]1752; GFX10W64-NEXT:    s_cbranch_execz .LBB6_41753; GFX10W64-NEXT:  ; %bb.3:1754; GFX10W64-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341755; GFX10W64-NEXT:    v_mov_b32_e32 v0, s21756; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)1757; GFX10W64-NEXT:    buffer_atomic_sub v0, off, s[8:11], 0 glc1758; GFX10W64-NEXT:  .LBB6_4:1759; GFX10W64-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)1760; GFX10W64-NEXT:    s_or_b64 exec, exec, s[0:1]1761; GFX10W64-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241762; GFX10W64-NEXT:    s_waitcnt vmcnt(0)1763; GFX10W64-NEXT:    v_readfirstlane_b32 s2, v01764; GFX10W64-NEXT:    v_mov_b32_e32 v0, 01765; GFX10W64-NEXT:    v_sub_nc_u32_e32 v1, s2, v11766; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)1767; GFX10W64-NEXT:    global_store_dword v0, v1, s[0:1]1768; GFX10W64-NEXT:    s_endpgm1769;1770; GFX10W32-LABEL: sub_i32_varying_vdata:1771; GFX10W32:       ; %bb.0: ; %entry1772; GFX10W32-NEXT:    s_mov_b32 s1, exec_lo1773; GFX10W32-NEXT:    s_mov_b32 s0, 01774; GFX10W32-NEXT:    ; implicit-def: $vgpr11775; GFX10W32-NEXT:  .LBB6_1: ; %ComputeLoop1776; GFX10W32-NEXT:    ; =>This Inner Loop Header: Depth=11777; GFX10W32-NEXT:    s_ff1_i32_b32 s2, s11778; GFX10W32-NEXT:    v_readlane_b32 s3, v0, s21779; GFX10W32-NEXT:    v_writelane_b32 v1, s0, s21780; GFX10W32-NEXT:    s_lshl_b32 s2, 1, s21781; GFX10W32-NEXT:    s_add_i32 s0, s0, s31782; GFX10W32-NEXT:    s_andn2_b32 s1, s1, s21783; GFX10W32-NEXT:    s_cbranch_scc1 .LBB6_11784; GFX10W32-NEXT:  ; %bb.2: ; %ComputeEnd1785; GFX10W32-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 01786; GFX10W32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v01787; GFX10W32-NEXT:    ; implicit-def: $vgpr01788; GFX10W32-NEXT:    s_and_saveexec_b32 s1, vcc_lo1789; GFX10W32-NEXT:    s_xor_b32 s1, exec_lo, s11790; GFX10W32-NEXT:    s_cbranch_execz .LBB6_41791; GFX10W32-NEXT:  ; %bb.3:1792; GFX10W32-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341793; GFX10W32-NEXT:    v_mov_b32_e32 v0, s01794; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)1795; GFX10W32-NEXT:    buffer_atomic_sub v0, off, s[8:11], 0 glc1796; GFX10W32-NEXT:  .LBB6_4:1797; GFX10W32-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)1798; GFX10W32-NEXT:    s_or_b32 exec_lo, exec_lo, s11799; GFX10W32-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241800; GFX10W32-NEXT:    s_waitcnt vmcnt(0)1801; GFX10W32-NEXT:    v_readfirstlane_b32 s2, v01802; GFX10W32-NEXT:    v_mov_b32_e32 v0, 01803; GFX10W32-NEXT:    v_sub_nc_u32_e32 v1, s2, v11804; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)1805; GFX10W32-NEXT:    global_store_dword v0, v1, s[0:1]1806; GFX10W32-NEXT:    s_endpgm1807;1808; GFX11W64-LABEL: sub_i32_varying_vdata:1809; GFX11W64:       ; %bb.0: ; %entry1810; GFX11W64-NEXT:    v_and_b32_e32 v1, 0x3ff, v01811; GFX11W64-NEXT:    s_mov_b64 s[0:1], exec1812; GFX11W64-NEXT:    s_mov_b32 s2, 01813; GFX11W64-NEXT:    ; implicit-def: $vgpr01814; GFX11W64-NEXT:  .LBB6_1: ; %ComputeLoop1815; GFX11W64-NEXT:    ; =>This Inner Loop Header: Depth=11816; GFX11W64-NEXT:    s_ctz_i32_b64 s3, s[0:1]1817; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)1818; GFX11W64-NEXT:    v_readlane_b32 s8, v1, s31819; GFX11W64-NEXT:    v_writelane_b32 v0, s2, s31820; GFX11W64-NEXT:    s_lshl_b64 s[6:7], 1, s31821; GFX11W64-NEXT:    s_add_i32 s2, s2, s81822; GFX11W64-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[6:7]1823; GFX11W64-NEXT:    s_cbranch_scc1 .LBB6_11824; GFX11W64-NEXT:  ; %bb.2: ; %ComputeEnd1825; GFX11W64-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 01826; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1827; GFX11W64-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v11828; GFX11W64-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v11829; GFX11W64-NEXT:    ; implicit-def: $vgpr11830; GFX11W64-NEXT:    s_and_saveexec_b64 s[0:1], vcc1831; GFX11W64-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)1832; GFX11W64-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]1833; GFX11W64-NEXT:    s_cbranch_execz .LBB6_41834; GFX11W64-NEXT:  ; %bb.3:1835; GFX11W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x341836; GFX11W64-NEXT:    v_mov_b32_e32 v1, s21837; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)1838; GFX11W64-NEXT:    buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc1839; GFX11W64-NEXT:  .LBB6_4:1840; GFX11W64-NEXT:    s_or_b64 exec, exec, s[0:1]1841; GFX11W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241842; GFX11W64-NEXT:    s_waitcnt vmcnt(0)1843; GFX11W64-NEXT:    v_readfirstlane_b32 s2, v11844; GFX11W64-NEXT:    v_mov_b32_e32 v1, 01845; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)1846; GFX11W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v01847; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)1848; GFX11W64-NEXT:    global_store_b32 v1, v0, s[0:1]1849; GFX11W64-NEXT:    s_endpgm1850;1851; GFX11W32-LABEL: sub_i32_varying_vdata:1852; GFX11W32:       ; %bb.0: ; %entry1853; GFX11W32-NEXT:    v_and_b32_e32 v1, 0x3ff, v01854; GFX11W32-NEXT:    s_mov_b32 s1, exec_lo1855; GFX11W32-NEXT:    s_mov_b32 s0, 01856; GFX11W32-NEXT:    ; implicit-def: $vgpr01857; GFX11W32-NEXT:  .LBB6_1: ; %ComputeLoop1858; GFX11W32-NEXT:    ; =>This Inner Loop Header: Depth=11859; GFX11W32-NEXT:    s_ctz_i32_b32 s2, s11860; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)1861; GFX11W32-NEXT:    v_readlane_b32 s3, v1, s21862; GFX11W32-NEXT:    v_writelane_b32 v0, s0, s21863; GFX11W32-NEXT:    s_lshl_b32 s2, 1, s21864; GFX11W32-NEXT:    s_add_i32 s0, s0, s31865; GFX11W32-NEXT:    s_and_not1_b32 s1, s1, s21866; GFX11W32-NEXT:    s_cbranch_scc1 .LBB6_11867; GFX11W32-NEXT:  ; %bb.2: ; %ComputeEnd1868; GFX11W32-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 01869; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)1870; GFX11W32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v11871; GFX11W32-NEXT:    ; implicit-def: $vgpr11872; GFX11W32-NEXT:    s_and_saveexec_b32 s1, vcc_lo1873; GFX11W32-NEXT:    s_xor_b32 s1, exec_lo, s11874; GFX11W32-NEXT:    s_cbranch_execz .LBB6_41875; GFX11W32-NEXT:  ; %bb.3:1876; GFX11W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x341877; GFX11W32-NEXT:    v_mov_b32_e32 v1, s01878; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)1879; GFX11W32-NEXT:    buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc1880; GFX11W32-NEXT:  .LBB6_4:1881; GFX11W32-NEXT:    s_or_b32 exec_lo, exec_lo, s11882; GFX11W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241883; GFX11W32-NEXT:    s_waitcnt vmcnt(0)1884; GFX11W32-NEXT:    v_readfirstlane_b32 s2, v11885; GFX11W32-NEXT:    v_mov_b32_e32 v1, 01886; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)1887; GFX11W32-NEXT:    v_sub_nc_u32_e32 v0, s2, v01888; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)1889; GFX11W32-NEXT:    global_store_b32 v1, v0, s[0:1]1890; GFX11W32-NEXT:    s_endpgm1891;1892; GFX12W64-LABEL: sub_i32_varying_vdata:1893; GFX12W64:       ; %bb.0: ; %entry1894; GFX12W64-NEXT:    v_and_b32_e32 v1, 0x3ff, v01895; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec1896; GFX12W64-NEXT:    s_mov_b32 s2, 01897; GFX12W64-NEXT:    ; implicit-def: $vgpr01898; GFX12W64-NEXT:  .LBB6_1: ; %ComputeLoop1899; GFX12W64-NEXT:    ; =>This Inner Loop Header: Depth=11900; GFX12W64-NEXT:    s_ctz_i32_b64 s3, s[0:1]1901; GFX12W64-NEXT:    s_wait_alu depctr_sa_sdst(0)1902; GFX12W64-NEXT:    v_readlane_b32 s8, v1, s31903; GFX12W64-NEXT:    v_writelane_b32 v0, s2, s31904; GFX12W64-NEXT:    s_lshl_b64 s[6:7], 1, s31905; GFX12W64-NEXT:    s_add_co_i32 s2, s2, s81906; GFX12W64-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[6:7]1907; GFX12W64-NEXT:    s_cbranch_scc1 .LBB6_11908; GFX12W64-NEXT:  ; %bb.2: ; %ComputeEnd1909; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 01910; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1911; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v11912; GFX12W64-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v11913; GFX12W64-NEXT:    ; implicit-def: $vgpr11914; GFX12W64-NEXT:    s_and_saveexec_b64 s[0:1], vcc1915; GFX12W64-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)1916; GFX12W64-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]1917; GFX12W64-NEXT:    s_cbranch_execz .LBB6_41918; GFX12W64-NEXT:  ; %bb.3:1919; GFX12W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x341920; GFX12W64-NEXT:    v_mov_b32_e32 v1, s21921; GFX12W64-NEXT:    s_wait_kmcnt 0x01922; GFX12W64-NEXT:    buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN1923; GFX12W64-NEXT:  .LBB6_4:1924; GFX12W64-NEXT:    s_or_b64 exec, exec, s[0:1]1925; GFX12W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241926; GFX12W64-NEXT:    s_wait_loadcnt 0x01927; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v11928; GFX12W64-NEXT:    v_mov_b32_e32 v1, 01929; GFX12W64-NEXT:    s_wait_alu depctr_va_sdst(0)1930; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)1931; GFX12W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v01932; GFX12W64-NEXT:    s_wait_kmcnt 0x01933; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]1934; GFX12W64-NEXT:    s_endpgm1935;1936; GFX12W32-LABEL: sub_i32_varying_vdata:1937; GFX12W32:       ; %bb.0: ; %entry1938; GFX12W32-NEXT:    v_and_b32_e32 v1, 0x3ff, v01939; GFX12W32-NEXT:    s_mov_b32 s1, exec_lo1940; GFX12W32-NEXT:    s_mov_b32 s0, 01941; GFX12W32-NEXT:    ; implicit-def: $vgpr01942; GFX12W32-NEXT:  .LBB6_1: ; %ComputeLoop1943; GFX12W32-NEXT:    ; =>This Inner Loop Header: Depth=11944; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)1945; GFX12W32-NEXT:    s_ctz_i32_b32 s2, s11946; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)1947; GFX12W32-NEXT:    v_readlane_b32 s3, v1, s21948; GFX12W32-NEXT:    v_writelane_b32 v0, s0, s21949; GFX12W32-NEXT:    s_lshl_b32 s2, 1, s21950; GFX12W32-NEXT:    s_add_co_i32 s0, s0, s31951; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)1952; GFX12W32-NEXT:    s_and_not1_b32 s1, s1, s21953; GFX12W32-NEXT:    s_cbranch_scc1 .LBB6_11954; GFX12W32-NEXT:  ; %bb.2: ; %ComputeEnd1955; GFX12W32-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 01956; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)1957; GFX12W32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v11958; GFX12W32-NEXT:    ; implicit-def: $vgpr11959; GFX12W32-NEXT:    s_and_saveexec_b32 s1, vcc_lo1960; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)1961; GFX12W32-NEXT:    s_xor_b32 s1, exec_lo, s11962; GFX12W32-NEXT:    s_cbranch_execz .LBB6_41963; GFX12W32-NEXT:  ; %bb.3:1964; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x341965; GFX12W32-NEXT:    v_mov_b32_e32 v1, s01966; GFX12W32-NEXT:    s_wait_kmcnt 0x01967; GFX12W32-NEXT:    buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN1968; GFX12W32-NEXT:  .LBB6_4:1969; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)1970; GFX12W32-NEXT:    s_or_b32 exec_lo, exec_lo, s11971; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241972; GFX12W32-NEXT:    s_wait_loadcnt 0x01973; GFX12W32-NEXT:    v_readfirstlane_b32 s2, v11974; GFX12W32-NEXT:    v_mov_b32_e32 v1, 01975; GFX12W32-NEXT:    s_wait_alu depctr_va_sdst(0)1976; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)1977; GFX12W32-NEXT:    v_sub_nc_u32_e32 v0, s2, v01978; GFX12W32-NEXT:    s_wait_kmcnt 0x01979; GFX12W32-NEXT:    global_store_b32 v1, v0, s[0:1]1980; GFX12W32-NEXT:    s_endpgm1981entry:1982  %lane = call i32 @llvm.amdgcn.workitem.id.x()1983  %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.sub(i32 %lane, ptr addrspace(8) %inout, i32 0, i32 0, i32 0)1984  store i32 %old, ptr addrspace(1) %out1985  ret void1986}1987 1988define amdgpu_kernel void @sub_i32_varying_offset(ptr addrspace(1) %out, ptr addrspace(8) %inout) {1989; GFX6-LABEL: sub_i32_varying_offset:1990; GFX6:       ; %bb.0: ; %entry1991; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xd1992; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x91993; GFX6-NEXT:    v_mov_b32_e32 v1, 11994; GFX6-NEXT:    s_waitcnt lgkmcnt(0)1995; GFX6-NEXT:    buffer_atomic_sub v1, v0, s[0:3], 0 offen glc1996; GFX6-NEXT:    s_mov_b32 s7, 0xf0001997; GFX6-NEXT:    s_mov_b32 s6, -11998; GFX6-NEXT:    s_waitcnt vmcnt(0)1999; GFX6-NEXT:    buffer_store_dword v1, off, s[4:7], 02000; GFX6-NEXT:    s_endpgm2001;2002; GFX8-LABEL: sub_i32_varying_offset:2003; GFX8:       ; %bb.0: ; %entry2004; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x342005; GFX8-NEXT:    v_mov_b32_e32 v2, 12006; GFX8-NEXT:    s_waitcnt lgkmcnt(0)2007; GFX8-NEXT:    buffer_atomic_sub v2, v0, s[0:3], 0 offen glc2008; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x242009; GFX8-NEXT:    s_waitcnt lgkmcnt(0)2010; GFX8-NEXT:    v_mov_b32_e32 v0, s02011; GFX8-NEXT:    v_mov_b32_e32 v1, s12012; GFX8-NEXT:    s_waitcnt vmcnt(0)2013; GFX8-NEXT:    flat_store_dword v[0:1], v22014; GFX8-NEXT:    s_endpgm2015;2016; GFX9-LABEL: sub_i32_varying_offset:2017; GFX9:       ; %bb.0: ; %entry2018; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x342019; GFX9-NEXT:    v_mov_b32_e32 v1, 12020; GFX9-NEXT:    s_waitcnt lgkmcnt(0)2021; GFX9-NEXT:    buffer_atomic_sub v1, v0, s[0:3], 0 offen glc2022; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x242023; GFX9-NEXT:    v_mov_b32_e32 v0, 02024; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2025; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]2026; GFX9-NEXT:    s_endpgm2027;2028; GFX10-LABEL: sub_i32_varying_offset:2029; GFX10:       ; %bb.0: ; %entry2030; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x342031; GFX10-NEXT:    v_mov_b32_e32 v1, 12032; GFX10-NEXT:    s_waitcnt lgkmcnt(0)2033; GFX10-NEXT:    buffer_atomic_sub v1, v0, s[0:3], 0 offen glc2034; GFX10-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)2035; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x242036; GFX10-NEXT:    v_mov_b32_e32 v0, 02037; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2038; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]2039; GFX10-NEXT:    s_endpgm2040;2041; GFX11W64-LABEL: sub_i32_varying_offset:2042; GFX11W64:       ; %bb.0: ; %entry2043; GFX11W64-NEXT:    s_load_b128 s[0:3], s[4:5], 0x342044; GFX11W64-NEXT:    v_and_b32_e32 v0, 0x3ff, v02045; GFX11W64-NEXT:    v_mov_b32_e32 v1, 12046; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)2047; GFX11W64-NEXT:    buffer_atomic_sub_u32 v1, v0, s[0:3], 0 offen glc2048; GFX11W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x242049; GFX11W64-NEXT:    v_mov_b32_e32 v0, 02050; GFX11W64-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2051; GFX11W64-NEXT:    global_store_b32 v0, v1, s[0:1]2052; GFX11W64-NEXT:    s_endpgm2053;2054; GFX11W32-LABEL: sub_i32_varying_offset:2055; GFX11W32:       ; %bb.0: ; %entry2056; GFX11W32-NEXT:    s_load_b128 s[0:3], s[4:5], 0x342057; GFX11W32-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v02058; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)2059; GFX11W32-NEXT:    buffer_atomic_sub_u32 v1, v0, s[0:3], 0 offen glc2060; GFX11W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x242061; GFX11W32-NEXT:    v_mov_b32_e32 v0, 02062; GFX11W32-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2063; GFX11W32-NEXT:    global_store_b32 v0, v1, s[0:1]2064; GFX11W32-NEXT:    s_endpgm2065;2066; GFX12W64-LABEL: sub_i32_varying_offset:2067; GFX12W64:       ; %bb.0: ; %entry2068; GFX12W64-NEXT:    s_load_b128 s[0:3], s[4:5], 0x342069; GFX12W64-NEXT:    v_and_b32_e32 v0, 0x3ff, v02070; GFX12W64-NEXT:    v_mov_b32_e32 v1, 12071; GFX12W64-NEXT:    s_wait_kmcnt 0x02072; GFX12W64-NEXT:    buffer_atomic_sub_u32 v1, v0, s[0:3], null offen th:TH_ATOMIC_RETURN2073; GFX12W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x242074; GFX12W64-NEXT:    v_mov_b32_e32 v0, 02075; GFX12W64-NEXT:    s_wait_loadcnt 0x02076; GFX12W64-NEXT:    s_wait_kmcnt 0x02077; GFX12W64-NEXT:    global_store_b32 v0, v1, s[0:1]2078; GFX12W64-NEXT:    s_endpgm2079;2080; GFX12W32-LABEL: sub_i32_varying_offset:2081; GFX12W32:       ; %bb.0: ; %entry2082; GFX12W32-NEXT:    s_load_b128 s[0:3], s[4:5], 0x342083; GFX12W32-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v02084; GFX12W32-NEXT:    s_wait_kmcnt 0x02085; GFX12W32-NEXT:    buffer_atomic_sub_u32 v1, v0, s[0:3], null offen th:TH_ATOMIC_RETURN2086; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x242087; GFX12W32-NEXT:    v_mov_b32_e32 v0, 02088; GFX12W32-NEXT:    s_wait_loadcnt 0x02089; GFX12W32-NEXT:    s_wait_kmcnt 0x02090; GFX12W32-NEXT:    global_store_b32 v0, v1, s[0:1]2091; GFX12W32-NEXT:    s_endpgm2092entry:2093  %lane = call i32 @llvm.amdgcn.workitem.id.x()2094  %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.sub(i32 1, ptr addrspace(8) %inout, i32 %lane, i32 0, i32 0)2095  store i32 %old, ptr addrspace(1) %out2096  ret void2097}2098;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:2099; GFX11: {{.*}}2100; GFX12: {{.*}}2101