brintos / llvm-project-archived public Read only

1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX6 %s3; RUN: llc -mtriple=amdgcn -mcpu=tonga -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX8 %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s5; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize64 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX10,GFX10W64 %s6; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX10,GFX10W32 %s7; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11W64 %s8; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11W32 %s9; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12W64 %s10; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12W32 %s11 12declare i32 @llvm.amdgcn.workitem.id.x()13declare i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add(i32, ptr addrspace(8), i32, i32, i32, i32)14declare i32 @llvm.amdgcn.struct.ptr.buffer.atomic.sub(i32, ptr addrspace(8), i32, i32, i32, i32)15 16; Show what the atomic optimization pass will do for struct buffers.17 18define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace(8) %inout) {19; GFX6-LABEL: add_i32_constant:20; GFX6:       ; %bb.0: ; %entry21; GFX6-NEXT:    s_mov_b64 s[2:3], exec22; GFX6-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 023; GFX6-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v024; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v025; GFX6-NEXT:    ; implicit-def: $vgpr126; GFX6-NEXT:    s_and_saveexec_b64 s[0:1], vcc27; GFX6-NEXT:    s_cbranch_execz .LBB0_228; GFX6-NEXT:  ; %bb.1:29; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xd30; GFX6-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]31; GFX6-NEXT:    s_mul_i32 s2, s2, 532; GFX6-NEXT:    v_mov_b32_e32 v1, s233; GFX6-NEXT:    v_mov_b32_e32 v2, 034; GFX6-NEXT:    s_waitcnt lgkmcnt(0)35; GFX6-NEXT:    buffer_atomic_add v1, v2, s[8:11], 0 idxen glc36; GFX6-NEXT:  .LBB0_2:37; GFX6-NEXT:    s_or_b64 exec, exec, s[0:1]38; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x939; GFX6-NEXT:    s_mov_b32 s3, 0xf00040; GFX6-NEXT:    s_mov_b32 s2, -141; GFX6-NEXT:    s_waitcnt vmcnt(0)42; GFX6-NEXT:    v_readfirstlane_b32 s4, v143; GFX6-NEXT:    v_mad_u32_u24 v0, v0, 5, s444; GFX6-NEXT:    s_waitcnt lgkmcnt(0)45; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 046; GFX6-NEXT:    s_endpgm47;48; GFX8-LABEL: add_i32_constant:49; GFX8:       ; %bb.0: ; %entry50; GFX8-NEXT:    s_mov_b64 s[2:3], exec51; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 052; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v053; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v054; GFX8-NEXT:    ; implicit-def: $vgpr155; GFX8-NEXT:    s_and_saveexec_b64 s[0:1], vcc56; GFX8-NEXT:    s_cbranch_execz .LBB0_257; GFX8-NEXT:  ; %bb.1:58; GFX8-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x3459; GFX8-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]60; GFX8-NEXT:    s_mul_i32 s2, s2, 561; GFX8-NEXT:    v_mov_b32_e32 v1, s262; GFX8-NEXT:    v_mov_b32_e32 v2, 063; GFX8-NEXT:    s_waitcnt lgkmcnt(0)64; GFX8-NEXT:    buffer_atomic_add v1, v2, s[8:11], 0 idxen glc65; GFX8-NEXT:  .LBB0_2:66; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]67; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2468; GFX8-NEXT:    s_waitcnt vmcnt(0)69; GFX8-NEXT:    v_readfirstlane_b32 s2, v170; GFX8-NEXT:    v_mad_u32_u24 v2, v0, 5, s271; GFX8-NEXT:    s_waitcnt lgkmcnt(0)72; GFX8-NEXT:    v_mov_b32_e32 v0, s073; GFX8-NEXT:    v_mov_b32_e32 v1, s174; GFX8-NEXT:    flat_store_dword v[0:1], v275; GFX8-NEXT:    s_endpgm76;77; GFX9-LABEL: add_i32_constant:78; GFX9:       ; %bb.0: ; %entry79; GFX9-NEXT:    s_mov_b64 s[2:3], exec80; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 081; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v082; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v083; GFX9-NEXT:    ; implicit-def: $vgpr184; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc85; GFX9-NEXT:    s_cbranch_execz .LBB0_286; GFX9-NEXT:  ; %bb.1:87; GFX9-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x3488; GFX9-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]89; GFX9-NEXT:    s_mul_i32 s2, s2, 590; GFX9-NEXT:    v_mov_b32_e32 v1, s291; GFX9-NEXT:    v_mov_b32_e32 v2, 092; GFX9-NEXT:    s_waitcnt lgkmcnt(0)93; GFX9-NEXT:    buffer_atomic_add v1, v2, s[8:11], 0 idxen glc94; GFX9-NEXT:  .LBB0_2:95; GFX9-NEXT:    s_or_b64 exec, exec, s[0:1]96; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2497; GFX9-NEXT:    s_waitcnt vmcnt(0)98; GFX9-NEXT:    v_readfirstlane_b32 s2, v199; GFX9-NEXT:    v_mov_b32_e32 v2, 0100; GFX9-NEXT:    v_mad_u32_u24 v0, v0, 5, s2101; GFX9-NEXT:    s_waitcnt lgkmcnt(0)102; GFX9-NEXT:    global_store_dword v2, v0, s[0:1]103; GFX9-NEXT:    s_endpgm104;105; GFX10W64-LABEL: add_i32_constant:106; GFX10W64:       ; %bb.0: ; %entry107; GFX10W64-NEXT:    s_mov_b64 s[2:3], exec108; GFX10W64-NEXT:    ; implicit-def: $vgpr1109; GFX10W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0110; GFX10W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0111; GFX10W64-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0112; GFX10W64-NEXT:    s_and_saveexec_b64 s[0:1], vcc113; GFX10W64-NEXT:    s_cbranch_execz .LBB0_2114; GFX10W64-NEXT:  ; %bb.1:115; GFX10W64-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34116; GFX10W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]117; GFX10W64-NEXT:    v_mov_b32_e32 v2, 0118; GFX10W64-NEXT:    s_mul_i32 s2, s2, 5119; GFX10W64-NEXT:    v_mov_b32_e32 v1, s2120; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)121; GFX10W64-NEXT:    buffer_atomic_add v1, v2, s[8:11], 0 idxen glc122; GFX10W64-NEXT:  .LBB0_2:123; GFX10W64-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)124; GFX10W64-NEXT:    s_or_b64 exec, exec, s[0:1]125; GFX10W64-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24126; GFX10W64-NEXT:    s_waitcnt vmcnt(0)127; GFX10W64-NEXT:    v_readfirstlane_b32 s2, v1128; GFX10W64-NEXT:    v_mov_b32_e32 v1, 0129; GFX10W64-NEXT:    v_mad_u32_u24 v0, v0, 5, s2130; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)131; GFX10W64-NEXT:    global_store_dword v1, v0, s[0:1]132; GFX10W64-NEXT:    s_endpgm133;134; GFX10W32-LABEL: add_i32_constant:135; GFX10W32:       ; %bb.0: ; %entry136; GFX10W32-NEXT:    s_mov_b32 s1, exec_lo137; GFX10W32-NEXT:    ; implicit-def: $vgpr1138; GFX10W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s1, 0139; GFX10W32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0140; GFX10W32-NEXT:    s_and_saveexec_b32 s0, vcc_lo141; GFX10W32-NEXT:    s_cbranch_execz .LBB0_2142; GFX10W32-NEXT:  ; %bb.1:143; GFX10W32-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34144; GFX10W32-NEXT:    s_bcnt1_i32_b32 s1, s1145; GFX10W32-NEXT:    v_mov_b32_e32 v2, 0146; GFX10W32-NEXT:    s_mul_i32 s1, s1, 5147; GFX10W32-NEXT:    v_mov_b32_e32 v1, s1148; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)149; GFX10W32-NEXT:    buffer_atomic_add v1, v2, s[8:11], 0 idxen glc150; GFX10W32-NEXT:  .LBB0_2:151; GFX10W32-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)152; GFX10W32-NEXT:    s_or_b32 exec_lo, exec_lo, s0153; GFX10W32-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24154; GFX10W32-NEXT:    s_waitcnt vmcnt(0)155; GFX10W32-NEXT:    v_readfirstlane_b32 s2, v1156; GFX10W32-NEXT:    v_mov_b32_e32 v1, 0157; GFX10W32-NEXT:    v_mad_u32_u24 v0, v0, 5, s2158; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)159; GFX10W32-NEXT:    global_store_dword v1, v0, s[0:1]160; GFX10W32-NEXT:    s_endpgm161;162; GFX11W64-LABEL: add_i32_constant:163; GFX11W64:       ; %bb.0: ; %entry164; GFX11W64-NEXT:    s_mov_b64 s[2:3], exec165; GFX11W64-NEXT:    s_mov_b64 s[0:1], exec166; GFX11W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0167; GFX11W64-NEXT:    ; implicit-def: $vgpr1168; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)169; GFX11W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0170; GFX11W64-NEXT:    v_cmpx_eq_u32_e32 0, v0171; GFX11W64-NEXT:    s_cbranch_execz .LBB0_2172; GFX11W64-NEXT:  ; %bb.1:173; GFX11W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34174; GFX11W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]175; GFX11W64-NEXT:    v_mov_b32_e32 v2, 0176; GFX11W64-NEXT:    s_mul_i32 s2, s2, 5177; GFX11W64-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)178; GFX11W64-NEXT:    v_mov_b32_e32 v1, s2179; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)180; GFX11W64-NEXT:    buffer_atomic_add_u32 v1, v2, s[8:11], 0 idxen glc181; GFX11W64-NEXT:  .LBB0_2:182; GFX11W64-NEXT:    s_or_b64 exec, exec, s[0:1]183; GFX11W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24184; GFX11W64-NEXT:    s_waitcnt vmcnt(0)185; GFX11W64-NEXT:    v_readfirstlane_b32 s2, v1186; GFX11W64-NEXT:    v_mov_b32_e32 v1, 0187; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)188; GFX11W64-NEXT:    v_mad_u32_u24 v0, v0, 5, s2189; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)190; GFX11W64-NEXT:    global_store_b32 v1, v0, s[0:1]191; GFX11W64-NEXT:    s_endpgm192;193; GFX11W32-LABEL: add_i32_constant:194; GFX11W32:       ; %bb.0: ; %entry195; GFX11W32-NEXT:    s_mov_b32 s1, exec_lo196; GFX11W32-NEXT:    s_mov_b32 s0, exec_lo197; GFX11W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s1, 0198; GFX11W32-NEXT:    ; implicit-def: $vgpr1199; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)200; GFX11W32-NEXT:    v_cmpx_eq_u32_e32 0, v0201; GFX11W32-NEXT:    s_cbranch_execz .LBB0_2202; GFX11W32-NEXT:  ; %bb.1:203; GFX11W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34204; GFX11W32-NEXT:    s_bcnt1_i32_b32 s1, s1205; GFX11W32-NEXT:    v_mov_b32_e32 v2, 0206; GFX11W32-NEXT:    s_mul_i32 s1, s1, 5207; GFX11W32-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)208; GFX11W32-NEXT:    v_mov_b32_e32 v1, s1209; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)210; GFX11W32-NEXT:    buffer_atomic_add_u32 v1, v2, s[8:11], 0 idxen glc211; GFX11W32-NEXT:  .LBB0_2:212; GFX11W32-NEXT:    s_or_b32 exec_lo, exec_lo, s0213; GFX11W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24214; GFX11W32-NEXT:    s_waitcnt vmcnt(0)215; GFX11W32-NEXT:    v_readfirstlane_b32 s2, v1216; GFX11W32-NEXT:    v_mov_b32_e32 v1, 0217; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)218; GFX11W32-NEXT:    v_mad_u32_u24 v0, v0, 5, s2219; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)220; GFX11W32-NEXT:    global_store_b32 v1, v0, s[0:1]221; GFX11W32-NEXT:    s_endpgm222;223; GFX12W64-LABEL: add_i32_constant:224; GFX12W64:       ; %bb.0: ; %entry225; GFX12W64-NEXT:    s_mov_b64 s[2:3], exec226; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec227; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0228; GFX12W64-NEXT:    ; implicit-def: $vgpr1229; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)230; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0231; GFX12W64-NEXT:    v_cmpx_eq_u32_e32 0, v0232; GFX12W64-NEXT:    s_cbranch_execz .LBB0_2233; GFX12W64-NEXT:  ; %bb.1:234; GFX12W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34235; GFX12W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]236; GFX12W64-NEXT:    v_mov_b32_e32 v2, 0237; GFX12W64-NEXT:    s_wait_alu depctr_sa_sdst(0)238; GFX12W64-NEXT:    s_mul_i32 s2, s2, 5239; GFX12W64-NEXT:    s_wait_alu depctr_sa_sdst(0)240; GFX12W64-NEXT:    v_mov_b32_e32 v1, s2241; GFX12W64-NEXT:    s_wait_kmcnt 0x0242; GFX12W64-NEXT:    buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN243; GFX12W64-NEXT:  .LBB0_2:244; GFX12W64-NEXT:    s_or_b64 exec, exec, s[0:1]245; GFX12W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24246; GFX12W64-NEXT:    s_wait_loadcnt 0x0247; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1248; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0249; GFX12W64-NEXT:    s_wait_alu depctr_va_sdst(0)250; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)251; GFX12W64-NEXT:    v_mad_u32_u24 v0, v0, 5, s2252; GFX12W64-NEXT:    s_wait_kmcnt 0x0253; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]254; GFX12W64-NEXT:    s_endpgm255;256; GFX12W32-LABEL: add_i32_constant:257; GFX12W32:       ; %bb.0: ; %entry258; GFX12W32-NEXT:    s_mov_b32 s1, exec_lo259; GFX12W32-NEXT:    s_mov_b32 s0, exec_lo260; GFX12W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s1, 0261; GFX12W32-NEXT:    ; implicit-def: $vgpr1262; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)263; GFX12W32-NEXT:    v_cmpx_eq_u32_e32 0, v0264; GFX12W32-NEXT:    s_cbranch_execz .LBB0_2265; GFX12W32-NEXT:  ; %bb.1:266; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34267; GFX12W32-NEXT:    s_bcnt1_i32_b32 s1, s1268; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)269; GFX12W32-NEXT:    s_mul_i32 s1, s1, 5270; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)271; GFX12W32-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1272; GFX12W32-NEXT:    s_wait_kmcnt 0x0273; GFX12W32-NEXT:    buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN274; GFX12W32-NEXT:  .LBB0_2:275; GFX12W32-NEXT:    s_or_b32 exec_lo, exec_lo, s0276; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24277; GFX12W32-NEXT:    s_wait_loadcnt 0x0278; GFX12W32-NEXT:    v_readfirstlane_b32 s2, v1279; GFX12W32-NEXT:    v_mov_b32_e32 v1, 0280; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)281; GFX12W32-NEXT:    v_mad_u32_u24 v0, v0, 5, s2282; GFX12W32-NEXT:    s_wait_kmcnt 0x0283; GFX12W32-NEXT:    global_store_b32 v1, v0, s[0:1]284; GFX12W32-NEXT:    s_endpgm285entry:286  %old = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add(i32 5, ptr addrspace(8) %inout, i32 0, i32 0, i32 0, i32 0)287  store i32 %old, ptr addrspace(1) %out288  ret void289}290 291define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(8) %inout, i32 %additive) {292; GFX6-LABEL: add_i32_uniform:293; GFX6:       ; %bb.0: ; %entry294; GFX6-NEXT:    s_mov_b64 s[2:3], exec295; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x11296; GFX6-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0297; GFX6-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0298; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0299; GFX6-NEXT:    ; implicit-def: $vgpr1300; GFX6-NEXT:    s_and_saveexec_b64 s[0:1], vcc301; GFX6-NEXT:    s_cbranch_execz .LBB1_2302; GFX6-NEXT:  ; %bb.1:303; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xd304; GFX6-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]305; GFX6-NEXT:    s_waitcnt lgkmcnt(0)306; GFX6-NEXT:    s_mul_i32 s2, s6, s2307; GFX6-NEXT:    v_mov_b32_e32 v1, s2308; GFX6-NEXT:    v_mov_b32_e32 v2, 0309; GFX6-NEXT:    buffer_atomic_add v1, v2, s[8:11], 0 idxen glc310; GFX6-NEXT:  .LBB1_2:311; GFX6-NEXT:    s_or_b64 exec, exec, s[0:1]312; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9313; GFX6-NEXT:    s_mov_b32 s3, 0xf000314; GFX6-NEXT:    s_mov_b32 s2, -1315; GFX6-NEXT:    s_waitcnt vmcnt(0)316; GFX6-NEXT:    v_readfirstlane_b32 s4, v1317; GFX6-NEXT:    s_waitcnt lgkmcnt(0)318; GFX6-NEXT:    v_mul_lo_u32 v0, s6, v0319; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s4, v0320; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0321; GFX6-NEXT:    s_endpgm322;323; GFX8-LABEL: add_i32_uniform:324; GFX8:       ; %bb.0: ; %entry325; GFX8-NEXT:    s_load_dword s6, s[4:5], 0x44326; GFX8-NEXT:    s_mov_b64 s[2:3], exec327; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0328; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0329; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0330; GFX8-NEXT:    ; implicit-def: $vgpr1331; GFX8-NEXT:    s_and_saveexec_b64 s[0:1], vcc332; GFX8-NEXT:    s_cbranch_execz .LBB1_2333; GFX8-NEXT:  ; %bb.1:334; GFX8-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34335; GFX8-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]336; GFX8-NEXT:    s_waitcnt lgkmcnt(0)337; GFX8-NEXT:    s_mul_i32 s2, s6, s2338; GFX8-NEXT:    v_mov_b32_e32 v1, s2339; GFX8-NEXT:    v_mov_b32_e32 v2, 0340; GFX8-NEXT:    buffer_atomic_add v1, v2, s[8:11], 0 idxen glc341; GFX8-NEXT:  .LBB1_2:342; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]343; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24344; GFX8-NEXT:    s_waitcnt lgkmcnt(0)345; GFX8-NEXT:    v_mul_lo_u32 v0, s6, v0346; GFX8-NEXT:    s_waitcnt vmcnt(0)347; GFX8-NEXT:    v_readfirstlane_b32 s2, v1348; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s2, v0349; GFX8-NEXT:    v_mov_b32_e32 v0, s0350; GFX8-NEXT:    v_mov_b32_e32 v1, s1351; GFX8-NEXT:    flat_store_dword v[0:1], v2352; GFX8-NEXT:    s_endpgm353;354; GFX9-LABEL: add_i32_uniform:355; GFX9:       ; %bb.0: ; %entry356; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x44357; GFX9-NEXT:    s_mov_b64 s[2:3], exec358; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0359; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0360; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0361; GFX9-NEXT:    ; implicit-def: $vgpr1362; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc363; GFX9-NEXT:    s_cbranch_execz .LBB1_2364; GFX9-NEXT:  ; %bb.1:365; GFX9-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34366; GFX9-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]367; GFX9-NEXT:    s_waitcnt lgkmcnt(0)368; GFX9-NEXT:    s_mul_i32 s2, s6, s2369; GFX9-NEXT:    v_mov_b32_e32 v1, s2370; GFX9-NEXT:    v_mov_b32_e32 v2, 0371; GFX9-NEXT:    buffer_atomic_add v1, v2, s[8:11], 0 idxen glc372; GFX9-NEXT:  .LBB1_2:373; GFX9-NEXT:    s_or_b64 exec, exec, s[0:1]374; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24375; GFX9-NEXT:    s_waitcnt lgkmcnt(0)376; GFX9-NEXT:    v_mul_lo_u32 v0, s6, v0377; GFX9-NEXT:    s_waitcnt vmcnt(0)378; GFX9-NEXT:    v_readfirstlane_b32 s2, v1379; GFX9-NEXT:    v_mov_b32_e32 v2, 0380; GFX9-NEXT:    v_add_u32_e32 v0, s2, v0381; GFX9-NEXT:    global_store_dword v2, v0, s[0:1]382; GFX9-NEXT:    s_endpgm383;384; GFX10W64-LABEL: add_i32_uniform:385; GFX10W64:       ; %bb.0: ; %entry386; GFX10W64-NEXT:    s_load_dword s6, s[4:5], 0x44387; GFX10W64-NEXT:    s_mov_b64 s[2:3], exec388; GFX10W64-NEXT:    ; implicit-def: $vgpr1389; GFX10W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0390; GFX10W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0391; GFX10W64-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0392; GFX10W64-NEXT:    s_and_saveexec_b64 s[0:1], vcc393; GFX10W64-NEXT:    s_cbranch_execz .LBB1_2394; GFX10W64-NEXT:  ; %bb.1:395; GFX10W64-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34396; GFX10W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]397; GFX10W64-NEXT:    v_mov_b32_e32 v2, 0398; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)399; GFX10W64-NEXT:    s_mul_i32 s2, s6, s2400; GFX10W64-NEXT:    v_mov_b32_e32 v1, s2401; GFX10W64-NEXT:    buffer_atomic_add v1, v2, s[8:11], 0 idxen glc402; GFX10W64-NEXT:  .LBB1_2:403; GFX10W64-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)404; GFX10W64-NEXT:    s_or_b64 exec, exec, s[0:1]405; GFX10W64-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24406; GFX10W64-NEXT:    s_waitcnt vmcnt(0)407; GFX10W64-NEXT:    v_readfirstlane_b32 s2, v1408; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)409; GFX10W64-NEXT:    v_mad_u64_u32 v[0:1], s[2:3], s6, v0, s[2:3]410; GFX10W64-NEXT:    v_mov_b32_e32 v1, 0411; GFX10W64-NEXT:    global_store_dword v1, v0, s[0:1]412; GFX10W64-NEXT:    s_endpgm413;414; GFX10W32-LABEL: add_i32_uniform:415; GFX10W32:       ; %bb.0: ; %entry416; GFX10W32-NEXT:    s_load_dword s0, s[4:5], 0x44417; GFX10W32-NEXT:    s_mov_b32 s2, exec_lo418; GFX10W32-NEXT:    ; implicit-def: $vgpr1419; GFX10W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0420; GFX10W32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0421; GFX10W32-NEXT:    s_and_saveexec_b32 s1, vcc_lo422; GFX10W32-NEXT:    s_cbranch_execz .LBB1_2423; GFX10W32-NEXT:  ; %bb.1:424; GFX10W32-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34425; GFX10W32-NEXT:    s_bcnt1_i32_b32 s2, s2426; GFX10W32-NEXT:    v_mov_b32_e32 v2, 0427; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)428; GFX10W32-NEXT:    s_mul_i32 s2, s0, s2429; GFX10W32-NEXT:    v_mov_b32_e32 v1, s2430; GFX10W32-NEXT:    buffer_atomic_add v1, v2, s[8:11], 0 idxen glc431; GFX10W32-NEXT:  .LBB1_2:432; GFX10W32-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)433; GFX10W32-NEXT:    s_or_b32 exec_lo, exec_lo, s1434; GFX10W32-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24435; GFX10W32-NEXT:    s_waitcnt vmcnt(0)436; GFX10W32-NEXT:    s_mov_b32 null, 0437; GFX10W32-NEXT:    v_readfirstlane_b32 s4, v1438; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)439; GFX10W32-NEXT:    v_mad_u64_u32 v[0:1], s0, s0, v0, s[4:5]440; GFX10W32-NEXT:    v_mov_b32_e32 v1, 0441; GFX10W32-NEXT:    global_store_dword v1, v0, s[2:3]442; GFX10W32-NEXT:    s_endpgm443;444; GFX11W64-LABEL: add_i32_uniform:445; GFX11W64:       ; %bb.0: ; %entry446; GFX11W64-NEXT:    s_load_b32 s6, s[4:5], 0x44447; GFX11W64-NEXT:    s_mov_b64 s[2:3], exec448; GFX11W64-NEXT:    s_mov_b64 s[0:1], exec449; GFX11W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0450; GFX11W64-NEXT:    ; implicit-def: $vgpr1451; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)452; GFX11W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0453; GFX11W64-NEXT:    v_cmpx_eq_u32_e32 0, v0454; GFX11W64-NEXT:    s_cbranch_execz .LBB1_2455; GFX11W64-NEXT:  ; %bb.1:456; GFX11W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34457; GFX11W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]458; GFX11W64-NEXT:    v_mov_b32_e32 v2, 0459; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)460; GFX11W64-NEXT:    s_mul_i32 s2, s6, s2461; GFX11W64-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)462; GFX11W64-NEXT:    v_mov_b32_e32 v1, s2463; GFX11W64-NEXT:    buffer_atomic_add_u32 v1, v2, s[8:11], 0 idxen glc464; GFX11W64-NEXT:  .LBB1_2:465; GFX11W64-NEXT:    s_or_b64 exec, exec, s[0:1]466; GFX11W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24467; GFX11W64-NEXT:    s_waitcnt vmcnt(0)468; GFX11W64-NEXT:    v_readfirstlane_b32 s2, v1469; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)470; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_1)471; GFX11W64-NEXT:    v_mad_u64_u32 v[1:2], null, s6, v0, s[2:3]472; GFX11W64-NEXT:    v_mov_b32_e32 v0, 0473; GFX11W64-NEXT:    global_store_b32 v0, v1, s[0:1]474; GFX11W64-NEXT:    s_endpgm475;476; GFX11W32-LABEL: add_i32_uniform:477; GFX11W32:       ; %bb.0: ; %entry478; GFX11W32-NEXT:    s_load_b32 s0, s[4:5], 0x44479; GFX11W32-NEXT:    s_mov_b32 s2, exec_lo480; GFX11W32-NEXT:    s_mov_b32 s1, exec_lo481; GFX11W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0482; GFX11W32-NEXT:    ; implicit-def: $vgpr1483; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)484; GFX11W32-NEXT:    v_cmpx_eq_u32_e32 0, v0485; GFX11W32-NEXT:    s_cbranch_execz .LBB1_2486; GFX11W32-NEXT:  ; %bb.1:487; GFX11W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34488; GFX11W32-NEXT:    s_bcnt1_i32_b32 s2, s2489; GFX11W32-NEXT:    v_mov_b32_e32 v2, 0490; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)491; GFX11W32-NEXT:    s_mul_i32 s2, s0, s2492; GFX11W32-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)493; GFX11W32-NEXT:    v_mov_b32_e32 v1, s2494; GFX11W32-NEXT:    buffer_atomic_add_u32 v1, v2, s[8:11], 0 idxen glc495; GFX11W32-NEXT:  .LBB1_2:496; GFX11W32-NEXT:    s_or_b32 exec_lo, exec_lo, s1497; GFX11W32-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24498; GFX11W32-NEXT:    s_waitcnt vmcnt(0)499; GFX11W32-NEXT:    v_readfirstlane_b32 s4, v1500; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)501; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)502; GFX11W32-NEXT:    v_mad_u64_u32 v[1:2], null, s0, v0, s[4:5]503; GFX11W32-NEXT:    v_mov_b32_e32 v0, 0504; GFX11W32-NEXT:    global_store_b32 v0, v1, s[2:3]505; GFX11W32-NEXT:    s_endpgm506;507; GFX12W64-LABEL: add_i32_uniform:508; GFX12W64:       ; %bb.0: ; %entry509; GFX12W64-NEXT:    s_load_b32 s6, s[4:5], 0x44510; GFX12W64-NEXT:    s_mov_b64 s[2:3], exec511; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec512; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0513; GFX12W64-NEXT:    ; implicit-def: $vgpr1514; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)515; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0516; GFX12W64-NEXT:    v_cmpx_eq_u32_e32 0, v0517; GFX12W64-NEXT:    s_cbranch_execz .LBB1_2518; GFX12W64-NEXT:  ; %bb.1:519; GFX12W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34520; GFX12W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]521; GFX12W64-NEXT:    v_mov_b32_e32 v2, 0522; GFX12W64-NEXT:    s_wait_kmcnt 0x0523; GFX12W64-NEXT:    s_wait_alu depctr_sa_sdst(0)524; GFX12W64-NEXT:    s_mul_i32 s2, s6, s2525; GFX12W64-NEXT:    s_wait_alu depctr_sa_sdst(0)526; GFX12W64-NEXT:    v_mov_b32_e32 v1, s2527; GFX12W64-NEXT:    buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN528; GFX12W64-NEXT:  .LBB1_2:529; GFX12W64-NEXT:    s_or_b64 exec, exec, s[0:1]530; GFX12W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24531; GFX12W64-NEXT:    s_wait_loadcnt 0x0532; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1533; GFX12W64-NEXT:    s_wait_kmcnt 0x0534; GFX12W64-NEXT:    s_wait_alu depctr_va_sdst(0)535; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1)536; GFX12W64-NEXT:    v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]537; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0538; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]539; GFX12W64-NEXT:    s_endpgm540;541; GFX12W32-LABEL: add_i32_uniform:542; GFX12W32:       ; %bb.0: ; %entry543; GFX12W32-NEXT:    s_load_b32 s0, s[4:5], 0x44544; GFX12W32-NEXT:    s_mov_b32 s2, exec_lo545; GFX12W32-NEXT:    s_mov_b32 s1, exec_lo546; GFX12W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0547; GFX12W32-NEXT:    ; implicit-def: $vgpr1548; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)549; GFX12W32-NEXT:    v_cmpx_eq_u32_e32 0, v0550; GFX12W32-NEXT:    s_cbranch_execz .LBB1_2551; GFX12W32-NEXT:  ; %bb.1:552; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34553; GFX12W32-NEXT:    s_bcnt1_i32_b32 s2, s2554; GFX12W32-NEXT:    s_wait_kmcnt 0x0555; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)556; GFX12W32-NEXT:    s_mul_i32 s2, s0, s2557; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)558; GFX12W32-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2559; GFX12W32-NEXT:    buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN560; GFX12W32-NEXT:  .LBB1_2:561; GFX12W32-NEXT:    s_or_b32 exec_lo, exec_lo, s1562; GFX12W32-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24563; GFX12W32-NEXT:    s_wait_loadcnt 0x0564; GFX12W32-NEXT:    v_readfirstlane_b32 s4, v1565; GFX12W32-NEXT:    s_wait_kmcnt 0x0566; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)567; GFX12W32-NEXT:    v_mad_co_u64_u32 v[0:1], null, s0, v0, s[4:5]568; GFX12W32-NEXT:    v_mov_b32_e32 v1, 0569; GFX12W32-NEXT:    global_store_b32 v1, v0, s[2:3]570; GFX12W32-NEXT:    s_endpgm571entry:572  %old = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add(i32 %additive, ptr addrspace(8) %inout, i32 0, i32 0, i32 0, i32 0)573  store i32 %old, ptr addrspace(1) %out574  ret void575}576 577define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addrspace(8) %inout) {578; GFX6-LABEL: add_i32_varying_vdata:579; GFX6:       ; %bb.0: ; %entry580; GFX6-NEXT:    s_mov_b64 s[0:1], exec581; GFX6-NEXT:    s_mov_b32 s2, 0582; GFX6-NEXT:    ; implicit-def: $vgpr1583; GFX6-NEXT:  .LBB2_1: ; %ComputeLoop584; GFX6-NEXT:    ; =>This Inner Loop Header: Depth=1585; GFX6-NEXT:    s_ff1_i32_b64 s3, s[0:1]586; GFX6-NEXT:    s_mov_b32 m0, s3587; GFX6-NEXT:    v_readlane_b32 s8, v0, s3588; GFX6-NEXT:    v_writelane_b32 v1, s2, m0589; GFX6-NEXT:    s_lshl_b64 s[6:7], 1, s3590; GFX6-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[6:7]591; GFX6-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0592; GFX6-NEXT:    s_and_b64 vcc, exec, s[6:7]593; GFX6-NEXT:    s_add_i32 s2, s2, s8594; GFX6-NEXT:    s_cbranch_vccnz .LBB2_1595; GFX6-NEXT:  ; %bb.2: ; %ComputeEnd596; GFX6-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0597; GFX6-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0598; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0599; GFX6-NEXT:    ; implicit-def: $vgpr0600; GFX6-NEXT:    s_and_saveexec_b64 s[0:1], vcc601; GFX6-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]602; GFX6-NEXT:    s_cbranch_execz .LBB2_4603; GFX6-NEXT:  ; %bb.3:604; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xd605; GFX6-NEXT:    v_mov_b32_e32 v0, s2606; GFX6-NEXT:    v_mov_b32_e32 v2, 0607; GFX6-NEXT:    s_waitcnt lgkmcnt(0)608; GFX6-NEXT:    buffer_atomic_add v0, v2, s[8:11], 0 idxen glc609; GFX6-NEXT:  .LBB2_4:610; GFX6-NEXT:    s_or_b64 exec, exec, s[0:1]611; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9612; GFX6-NEXT:    s_mov_b32 s3, 0xf000613; GFX6-NEXT:    s_mov_b32 s2, -1614; GFX6-NEXT:    s_waitcnt vmcnt(0)615; GFX6-NEXT:    v_readfirstlane_b32 s4, v0616; GFX6-NEXT:    s_waitcnt expcnt(0)617; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s4, v1618; GFX6-NEXT:    s_waitcnt lgkmcnt(0)619; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0620; GFX6-NEXT:    s_endpgm621;622; GFX8-LABEL: add_i32_varying_vdata:623; GFX8:       ; %bb.0: ; %entry624; GFX8-NEXT:    s_mov_b64 s[0:1], exec625; GFX8-NEXT:    s_mov_b32 s2, 0626; GFX8-NEXT:    ; implicit-def: $vgpr1627; GFX8-NEXT:  .LBB2_1: ; %ComputeLoop628; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1629; GFX8-NEXT:    s_ff1_i32_b64 s3, s[0:1]630; GFX8-NEXT:    s_mov_b32 m0, s3631; GFX8-NEXT:    v_readlane_b32 s6, v0, s3632; GFX8-NEXT:    v_writelane_b32 v1, s2, m0633; GFX8-NEXT:    s_add_i32 s2, s2, s6634; GFX8-NEXT:    s_lshl_b64 s[6:7], 1, s3635; GFX8-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[6:7]636; GFX8-NEXT:    s_cbranch_scc1 .LBB2_1637; GFX8-NEXT:  ; %bb.2: ; %ComputeEnd638; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0639; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0640; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0641; GFX8-NEXT:    ; implicit-def: $vgpr0642; GFX8-NEXT:    s_and_saveexec_b64 s[0:1], vcc643; GFX8-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]644; GFX8-NEXT:    s_cbranch_execz .LBB2_4645; GFX8-NEXT:  ; %bb.3:646; GFX8-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34647; GFX8-NEXT:    v_mov_b32_e32 v0, s2648; GFX8-NEXT:    v_mov_b32_e32 v2, 0649; GFX8-NEXT:    s_waitcnt lgkmcnt(0)650; GFX8-NEXT:    buffer_atomic_add v0, v2, s[8:11], 0 idxen glc651; GFX8-NEXT:  .LBB2_4:652; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]653; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24654; GFX8-NEXT:    s_waitcnt vmcnt(0)655; GFX8-NEXT:    v_readfirstlane_b32 s2, v0656; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s2, v1657; GFX8-NEXT:    s_waitcnt lgkmcnt(0)658; GFX8-NEXT:    v_mov_b32_e32 v0, s0659; GFX8-NEXT:    v_mov_b32_e32 v1, s1660; GFX8-NEXT:    flat_store_dword v[0:1], v2661; GFX8-NEXT:    s_endpgm662;663; GFX9-LABEL: add_i32_varying_vdata:664; GFX9:       ; %bb.0: ; %entry665; GFX9-NEXT:    s_mov_b64 s[0:1], exec666; GFX9-NEXT:    s_mov_b32 s2, 0667; GFX9-NEXT:    ; implicit-def: $vgpr1668; GFX9-NEXT:  .LBB2_1: ; %ComputeLoop669; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1670; GFX9-NEXT:    s_ff1_i32_b64 s3, s[0:1]671; GFX9-NEXT:    s_mov_b32 m0, s3672; GFX9-NEXT:    v_readlane_b32 s6, v0, s3673; GFX9-NEXT:    v_writelane_b32 v1, s2, m0674; GFX9-NEXT:    s_add_i32 s2, s2, s6675; GFX9-NEXT:    s_lshl_b64 s[6:7], 1, s3676; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[6:7]677; GFX9-NEXT:    s_cbranch_scc1 .LBB2_1678; GFX9-NEXT:  ; %bb.2: ; %ComputeEnd679; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0680; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0681; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0682; GFX9-NEXT:    ; implicit-def: $vgpr0683; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc684; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]685; GFX9-NEXT:    s_cbranch_execz .LBB2_4686; GFX9-NEXT:  ; %bb.3:687; GFX9-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34688; GFX9-NEXT:    v_mov_b32_e32 v0, s2689; GFX9-NEXT:    v_mov_b32_e32 v2, 0690; GFX9-NEXT:    s_waitcnt lgkmcnt(0)691; GFX9-NEXT:    buffer_atomic_add v0, v2, s[8:11], 0 idxen glc692; GFX9-NEXT:  .LBB2_4:693; GFX9-NEXT:    s_or_b64 exec, exec, s[0:1]694; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24695; GFX9-NEXT:    s_waitcnt vmcnt(0)696; GFX9-NEXT:    v_readfirstlane_b32 s2, v0697; GFX9-NEXT:    v_mov_b32_e32 v2, 0698; GFX9-NEXT:    v_add_u32_e32 v0, s2, v1699; GFX9-NEXT:    s_waitcnt lgkmcnt(0)700; GFX9-NEXT:    global_store_dword v2, v0, s[0:1]701; GFX9-NEXT:    s_endpgm702;703; GFX10W64-LABEL: add_i32_varying_vdata:704; GFX10W64:       ; %bb.0: ; %entry705; GFX10W64-NEXT:    s_mov_b64 s[0:1], exec706; GFX10W64-NEXT:    s_mov_b32 s2, 0707; GFX10W64-NEXT:    ; implicit-def: $vgpr1708; GFX10W64-NEXT:  .LBB2_1: ; %ComputeLoop709; GFX10W64-NEXT:    ; =>This Inner Loop Header: Depth=1710; GFX10W64-NEXT:    s_ff1_i32_b64 s3, s[0:1]711; GFX10W64-NEXT:    v_readlane_b32 s8, v0, s3712; GFX10W64-NEXT:    v_writelane_b32 v1, s2, s3713; GFX10W64-NEXT:    s_lshl_b64 s[6:7], 1, s3714; GFX10W64-NEXT:    s_add_i32 s2, s2, s8715; GFX10W64-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[6:7]716; GFX10W64-NEXT:    s_cbranch_scc1 .LBB2_1717; GFX10W64-NEXT:  ; %bb.2: ; %ComputeEnd718; GFX10W64-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0719; GFX10W64-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0720; GFX10W64-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0721; GFX10W64-NEXT:    ; implicit-def: $vgpr0722; GFX10W64-NEXT:    s_and_saveexec_b64 s[0:1], vcc723; GFX10W64-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]724; GFX10W64-NEXT:    s_cbranch_execz .LBB2_4725; GFX10W64-NEXT:  ; %bb.3:726; GFX10W64-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34727; GFX10W64-NEXT:    v_mov_b32_e32 v0, s2728; GFX10W64-NEXT:    v_mov_b32_e32 v2, 0729; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)730; GFX10W64-NEXT:    buffer_atomic_add v0, v2, s[8:11], 0 idxen glc731; GFX10W64-NEXT:  .LBB2_4:732; GFX10W64-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)733; GFX10W64-NEXT:    s_or_b64 exec, exec, s[0:1]734; GFX10W64-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24735; GFX10W64-NEXT:    s_waitcnt vmcnt(0)736; GFX10W64-NEXT:    v_readfirstlane_b32 s2, v0737; GFX10W64-NEXT:    v_mov_b32_e32 v0, 0738; GFX10W64-NEXT:    v_add_nc_u32_e32 v1, s2, v1739; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)740; GFX10W64-NEXT:    global_store_dword v0, v1, s[0:1]741; GFX10W64-NEXT:    s_endpgm742;743; GFX10W32-LABEL: add_i32_varying_vdata:744; GFX10W32:       ; %bb.0: ; %entry745; GFX10W32-NEXT:    s_mov_b32 s1, exec_lo746; GFX10W32-NEXT:    s_mov_b32 s0, 0747; GFX10W32-NEXT:    ; implicit-def: $vgpr1748; GFX10W32-NEXT:  .LBB2_1: ; %ComputeLoop749; GFX10W32-NEXT:    ; =>This Inner Loop Header: Depth=1750; GFX10W32-NEXT:    s_ff1_i32_b32 s2, s1751; GFX10W32-NEXT:    v_readlane_b32 s3, v0, s2752; GFX10W32-NEXT:    v_writelane_b32 v1, s0, s2753; GFX10W32-NEXT:    s_lshl_b32 s2, 1, s2754; GFX10W32-NEXT:    s_add_i32 s0, s0, s3755; GFX10W32-NEXT:    s_andn2_b32 s1, s1, s2756; GFX10W32-NEXT:    s_cbranch_scc1 .LBB2_1757; GFX10W32-NEXT:  ; %bb.2: ; %ComputeEnd758; GFX10W32-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0759; GFX10W32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0760; GFX10W32-NEXT:    ; implicit-def: $vgpr0761; GFX10W32-NEXT:    s_and_saveexec_b32 s1, vcc_lo762; GFX10W32-NEXT:    s_xor_b32 s1, exec_lo, s1763; GFX10W32-NEXT:    s_cbranch_execz .LBB2_4764; GFX10W32-NEXT:  ; %bb.3:765; GFX10W32-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x34766; GFX10W32-NEXT:    v_mov_b32_e32 v0, s0767; GFX10W32-NEXT:    v_mov_b32_e32 v2, 0768; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)769; GFX10W32-NEXT:    buffer_atomic_add v0, v2, s[8:11], 0 idxen glc770; GFX10W32-NEXT:  .LBB2_4:771; GFX10W32-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)772; GFX10W32-NEXT:    s_or_b32 exec_lo, exec_lo, s1773; GFX10W32-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24774; GFX10W32-NEXT:    s_waitcnt vmcnt(0)775; GFX10W32-NEXT:    v_readfirstlane_b32 s2, v0776; GFX10W32-NEXT:    v_mov_b32_e32 v0, 0777; GFX10W32-NEXT:    v_add_nc_u32_e32 v1, s2, v1778; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)779; GFX10W32-NEXT:    global_store_dword v0, v1, s[0:1]780; GFX10W32-NEXT:    s_endpgm781;782; GFX11W64-LABEL: add_i32_varying_vdata:783; GFX11W64:       ; %bb.0: ; %entry784; GFX11W64-NEXT:    v_and_b32_e32 v1, 0x3ff, v0785; GFX11W64-NEXT:    s_mov_b64 s[0:1], exec786; GFX11W64-NEXT:    s_mov_b32 s2, 0787; GFX11W64-NEXT:    ; implicit-def: $vgpr0788; GFX11W64-NEXT:  .LBB2_1: ; %ComputeLoop789; GFX11W64-NEXT:    ; =>This Inner Loop Header: Depth=1790; GFX11W64-NEXT:    s_ctz_i32_b64 s3, s[0:1]791; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)792; GFX11W64-NEXT:    v_readlane_b32 s8, v1, s3793; GFX11W64-NEXT:    v_writelane_b32 v0, s2, s3794; GFX11W64-NEXT:    s_lshl_b64 s[6:7], 1, s3795; GFX11W64-NEXT:    s_add_i32 s2, s2, s8796; GFX11W64-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[6:7]797; GFX11W64-NEXT:    s_cbranch_scc1 .LBB2_1798; GFX11W64-NEXT:  ; %bb.2: ; %ComputeEnd799; GFX11W64-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0800; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)801; GFX11W64-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1802; GFX11W64-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1803; GFX11W64-NEXT:    ; implicit-def: $vgpr1804; GFX11W64-NEXT:    s_and_saveexec_b64 s[0:1], vcc805; GFX11W64-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)806; GFX11W64-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]807; GFX11W64-NEXT:    s_cbranch_execz .LBB2_4808; GFX11W64-NEXT:  ; %bb.3:809; GFX11W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34810; GFX11W64-NEXT:    v_mov_b32_e32 v1, s2811; GFX11W64-NEXT:    v_mov_b32_e32 v2, 0812; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)813; GFX11W64-NEXT:    buffer_atomic_add_u32 v1, v2, s[8:11], 0 idxen glc814; GFX11W64-NEXT:  .LBB2_4:815; GFX11W64-NEXT:    s_or_b64 exec, exec, s[0:1]816; GFX11W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24817; GFX11W64-NEXT:    s_waitcnt vmcnt(0)818; GFX11W64-NEXT:    v_readfirstlane_b32 s2, v1819; GFX11W64-NEXT:    v_mov_b32_e32 v1, 0820; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)821; GFX11W64-NEXT:    v_add_nc_u32_e32 v0, s2, v0822; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)823; GFX11W64-NEXT:    global_store_b32 v1, v0, s[0:1]824; GFX11W64-NEXT:    s_endpgm825;826; GFX11W32-LABEL: add_i32_varying_vdata:827; GFX11W32:       ; %bb.0: ; %entry828; GFX11W32-NEXT:    v_and_b32_e32 v1, 0x3ff, v0829; GFX11W32-NEXT:    s_mov_b32 s1, exec_lo830; GFX11W32-NEXT:    s_mov_b32 s0, 0831; GFX11W32-NEXT:    ; implicit-def: $vgpr0832; GFX11W32-NEXT:  .LBB2_1: ; %ComputeLoop833; GFX11W32-NEXT:    ; =>This Inner Loop Header: Depth=1834; GFX11W32-NEXT:    s_ctz_i32_b32 s2, s1835; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)836; GFX11W32-NEXT:    v_readlane_b32 s3, v1, s2837; GFX11W32-NEXT:    v_writelane_b32 v0, s0, s2838; GFX11W32-NEXT:    s_lshl_b32 s2, 1, s2839; GFX11W32-NEXT:    s_add_i32 s0, s0, s3840; GFX11W32-NEXT:    s_and_not1_b32 s1, s1, s2841; GFX11W32-NEXT:    s_cbranch_scc1 .LBB2_1842; GFX11W32-NEXT:  ; %bb.2: ; %ComputeEnd843; GFX11W32-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0844; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)845; GFX11W32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1846; GFX11W32-NEXT:    ; implicit-def: $vgpr1847; GFX11W32-NEXT:    s_and_saveexec_b32 s1, vcc_lo848; GFX11W32-NEXT:    s_xor_b32 s1, exec_lo, s1849; GFX11W32-NEXT:    s_cbranch_execz .LBB2_4850; GFX11W32-NEXT:  ; %bb.3:851; GFX11W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34852; GFX11W32-NEXT:    v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v2, 0853; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)854; GFX11W32-NEXT:    buffer_atomic_add_u32 v1, v2, s[8:11], 0 idxen glc855; GFX11W32-NEXT:  .LBB2_4:856; GFX11W32-NEXT:    s_or_b32 exec_lo, exec_lo, s1857; GFX11W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24858; GFX11W32-NEXT:    s_waitcnt vmcnt(0)859; GFX11W32-NEXT:    v_readfirstlane_b32 s2, v1860; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)861; GFX11W32-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, s2, v0862; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)863; GFX11W32-NEXT:    global_store_b32 v1, v0, s[0:1]864; GFX11W32-NEXT:    s_endpgm865;866; GFX12W64-LABEL: add_i32_varying_vdata:867; GFX12W64:       ; %bb.0: ; %entry868; GFX12W64-NEXT:    v_and_b32_e32 v1, 0x3ff, v0869; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec870; GFX12W64-NEXT:    s_mov_b32 s2, 0871; GFX12W64-NEXT:    ; implicit-def: $vgpr0872; GFX12W64-NEXT:  .LBB2_1: ; %ComputeLoop873; GFX12W64-NEXT:    ; =>This Inner Loop Header: Depth=1874; GFX12W64-NEXT:    s_ctz_i32_b64 s3, s[0:1]875; GFX12W64-NEXT:    s_wait_alu depctr_sa_sdst(0)876; GFX12W64-NEXT:    v_readlane_b32 s8, v1, s3877; GFX12W64-NEXT:    v_writelane_b32 v0, s2, s3878; GFX12W64-NEXT:    s_lshl_b64 s[6:7], 1, s3879; GFX12W64-NEXT:    s_add_co_i32 s2, s2, s8880; GFX12W64-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[6:7]881; GFX12W64-NEXT:    s_cbranch_scc1 .LBB2_1882; GFX12W64-NEXT:  ; %bb.2: ; %ComputeEnd883; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0884; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)885; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1886; GFX12W64-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1887; GFX12W64-NEXT:    ; implicit-def: $vgpr1888; GFX12W64-NEXT:    s_and_saveexec_b64 s[0:1], vcc889; GFX12W64-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)890; GFX12W64-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]891; GFX12W64-NEXT:    s_cbranch_execz .LBB2_4892; GFX12W64-NEXT:  ; %bb.3:893; GFX12W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34894; GFX12W64-NEXT:    v_mov_b32_e32 v2, 0895; GFX12W64-NEXT:    v_mov_b32_e32 v1, s2896; GFX12W64-NEXT:    s_wait_kmcnt 0x0897; GFX12W64-NEXT:    buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN898; GFX12W64-NEXT:  .LBB2_4:899; GFX12W64-NEXT:    s_or_b64 exec, exec, s[0:1]900; GFX12W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24901; GFX12W64-NEXT:    s_wait_loadcnt 0x0902; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1903; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0904; GFX12W64-NEXT:    s_wait_alu depctr_va_sdst(0)905; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)906; GFX12W64-NEXT:    v_add_nc_u32_e32 v0, s2, v0907; GFX12W64-NEXT:    s_wait_kmcnt 0x0908; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]909; GFX12W64-NEXT:    s_endpgm910;911; GFX12W32-LABEL: add_i32_varying_vdata:912; GFX12W32:       ; %bb.0: ; %entry913; GFX12W32-NEXT:    v_and_b32_e32 v1, 0x3ff, v0914; GFX12W32-NEXT:    s_mov_b32 s1, exec_lo915; GFX12W32-NEXT:    s_mov_b32 s0, 0916; GFX12W32-NEXT:    ; implicit-def: $vgpr0917; GFX12W32-NEXT:  .LBB2_1: ; %ComputeLoop918; GFX12W32-NEXT:    ; =>This Inner Loop Header: Depth=1919; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)920; GFX12W32-NEXT:    s_ctz_i32_b32 s2, s1921; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)922; GFX12W32-NEXT:    v_readlane_b32 s3, v1, s2923; GFX12W32-NEXT:    v_writelane_b32 v0, s0, s2924; GFX12W32-NEXT:    s_lshl_b32 s2, 1, s2925; GFX12W32-NEXT:    s_add_co_i32 s0, s0, s3926; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)927; GFX12W32-NEXT:    s_and_not1_b32 s1, s1, s2928; GFX12W32-NEXT:    s_cbranch_scc1 .LBB2_1929; GFX12W32-NEXT:  ; %bb.2: ; %ComputeEnd930; GFX12W32-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0931; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)932; GFX12W32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1933; GFX12W32-NEXT:    ; implicit-def: $vgpr1934; GFX12W32-NEXT:    s_and_saveexec_b32 s1, vcc_lo935; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)936; GFX12W32-NEXT:    s_xor_b32 s1, exec_lo, s1937; GFX12W32-NEXT:    s_cbranch_execz .LBB2_4938; GFX12W32-NEXT:  ; %bb.3:939; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34940; GFX12W32-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s0941; GFX12W32-NEXT:    s_wait_kmcnt 0x0942; GFX12W32-NEXT:    buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN943; GFX12W32-NEXT:  .LBB2_4:944; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)945; GFX12W32-NEXT:    s_or_b32 exec_lo, exec_lo, s1946; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24947; GFX12W32-NEXT:    s_wait_loadcnt 0x0948; GFX12W32-NEXT:    v_readfirstlane_b32 s2, v1949; GFX12W32-NEXT:    s_wait_alu depctr_va_sdst(0)950; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)951; GFX12W32-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, s2, v0952; GFX12W32-NEXT:    s_wait_kmcnt 0x0953; GFX12W32-NEXT:    global_store_b32 v1, v0, s[0:1]954; GFX12W32-NEXT:    s_endpgm955entry:956  %lane = call i32 @llvm.amdgcn.workitem.id.x()957  %old = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add(i32 %lane, ptr addrspace(8) %inout, i32 0, i32 0, i32 0, i32 0)958  store i32 %old, ptr addrspace(1) %out959  ret void960}961 962define amdgpu_kernel void @add_i32_varying_vindex(ptr addrspace(1) %out, ptr addrspace(8) %inout) {963; GFX6-LABEL: add_i32_varying_vindex:964; GFX6:       ; %bb.0: ; %entry965; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xd966; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9967; GFX6-NEXT:    v_mov_b32_e32 v1, 1968; GFX6-NEXT:    s_waitcnt lgkmcnt(0)969; GFX6-NEXT:    buffer_atomic_add v1, v0, s[0:3], 0 idxen glc970; GFX6-NEXT:    s_mov_b32 s7, 0xf000971; GFX6-NEXT:    s_mov_b32 s6, -1972; GFX6-NEXT:    s_waitcnt vmcnt(0)973; GFX6-NEXT:    buffer_store_dword v1, off, s[4:7], 0974; GFX6-NEXT:    s_endpgm975;976; GFX8-LABEL: add_i32_varying_vindex:977; GFX8:       ; %bb.0: ; %entry978; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34979; GFX8-NEXT:    v_mov_b32_e32 v2, 1980; GFX8-NEXT:    s_waitcnt lgkmcnt(0)981; GFX8-NEXT:    buffer_atomic_add v2, v0, s[0:3], 0 idxen glc982; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24983; GFX8-NEXT:    s_waitcnt lgkmcnt(0)984; GFX8-NEXT:    v_mov_b32_e32 v0, s0985; GFX8-NEXT:    v_mov_b32_e32 v1, s1986; GFX8-NEXT:    s_waitcnt vmcnt(0)987; GFX8-NEXT:    flat_store_dword v[0:1], v2988; GFX8-NEXT:    s_endpgm989;990; GFX9-LABEL: add_i32_varying_vindex:991; GFX9:       ; %bb.0: ; %entry992; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34993; GFX9-NEXT:    v_mov_b32_e32 v1, 1994; GFX9-NEXT:    s_waitcnt lgkmcnt(0)995; GFX9-NEXT:    buffer_atomic_add v1, v0, s[0:3], 0 idxen glc996; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24997; GFX9-NEXT:    v_mov_b32_e32 v0, 0998; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)999; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]1000; GFX9-NEXT:    s_endpgm1001;1002; GFX10-LABEL: add_i32_varying_vindex:1003; GFX10:       ; %bb.0: ; %entry1004; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x341005; GFX10-NEXT:    v_mov_b32_e32 v1, 11006; GFX10-NEXT:    s_waitcnt lgkmcnt(0)1007; GFX10-NEXT:    buffer_atomic_add v1, v0, s[0:3], 0 idxen glc1008; GFX10-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)1009; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241010; GFX10-NEXT:    v_mov_b32_e32 v0, 01011; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1012; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]1013; GFX10-NEXT:    s_endpgm1014;1015; GFX11W64-LABEL: add_i32_varying_vindex:1016; GFX11W64:       ; %bb.0: ; %entry1017; GFX11W64-NEXT:    s_load_b128 s[0:3], s[4:5], 0x341018; GFX11W64-NEXT:    v_and_b32_e32 v0, 0x3ff, v01019; GFX11W64-NEXT:    v_mov_b32_e32 v1, 11020; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)1021; GFX11W64-NEXT:    buffer_atomic_add_u32 v1, v0, s[0:3], 0 idxen glc1022; GFX11W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241023; GFX11W64-NEXT:    v_mov_b32_e32 v0, 01024; GFX11W64-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1025; GFX11W64-NEXT:    global_store_b32 v0, v1, s[0:1]1026; GFX11W64-NEXT:    s_endpgm1027;1028; GFX11W32-LABEL: add_i32_varying_vindex:1029; GFX11W32:       ; %bb.0: ; %entry1030; GFX11W32-NEXT:    s_load_b128 s[0:3], s[4:5], 0x341031; GFX11W32-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v01032; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)1033; GFX11W32-NEXT:    buffer_atomic_add_u32 v1, v0, s[0:3], 0 idxen glc1034; GFX11W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241035; GFX11W32-NEXT:    v_mov_b32_e32 v0, 01036; GFX11W32-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1037; GFX11W32-NEXT:    global_store_b32 v0, v1, s[0:1]1038; GFX11W32-NEXT:    s_endpgm1039;1040; GFX12W64-LABEL: add_i32_varying_vindex:1041; GFX12W64:       ; %bb.0: ; %entry1042; GFX12W64-NEXT:    s_load_b128 s[0:3], s[4:5], 0x341043; GFX12W64-NEXT:    v_and_b32_e32 v0, 0x3ff, v01044; GFX12W64-NEXT:    v_mov_b32_e32 v1, 11045; GFX12W64-NEXT:    s_wait_kmcnt 0x01046; GFX12W64-NEXT:    buffer_atomic_add_u32 v1, v0, s[0:3], null idxen th:TH_ATOMIC_RETURN1047; GFX12W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241048; GFX12W64-NEXT:    v_mov_b32_e32 v0, 01049; GFX12W64-NEXT:    s_wait_loadcnt 0x01050; GFX12W64-NEXT:    s_wait_kmcnt 0x01051; GFX12W64-NEXT:    global_store_b32 v0, v1, s[0:1]1052; GFX12W64-NEXT:    s_endpgm1053;1054; GFX12W32-LABEL: add_i32_varying_vindex:1055; GFX12W32:       ; %bb.0: ; %entry1056; GFX12W32-NEXT:    s_load_b128 s[0:3], s[4:5], 0x341057; GFX12W32-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v01058; GFX12W32-NEXT:    s_wait_kmcnt 0x01059; GFX12W32-NEXT:    buffer_atomic_add_u32 v1, v0, s[0:3], null idxen th:TH_ATOMIC_RETURN1060; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241061; GFX12W32-NEXT:    v_mov_b32_e32 v0, 01062; GFX12W32-NEXT:    s_wait_loadcnt 0x01063; GFX12W32-NEXT:    s_wait_kmcnt 0x01064; GFX12W32-NEXT:    global_store_b32 v0, v1, s[0:1]1065; GFX12W32-NEXT:    s_endpgm1066entry:1067  %lane = call i32 @llvm.amdgcn.workitem.id.x()1068  %old = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add(i32 1, ptr addrspace(8) %inout, i32 %lane, i32 0, i32 0, i32 0)1069  store i32 %old, ptr addrspace(1) %out1070  ret void1071}1072 1073define amdgpu_kernel void @add_i32_varying_offset(ptr addrspace(1) %out, ptr addrspace(8) %inout) {1074; GFX6-LABEL: add_i32_varying_offset:1075; GFX6:       ; %bb.0: ; %entry1076; GFX6-NEXT:    v_mov_b32_e32 v1, v01077; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xd1078; GFX6-NEXT:    s_mov_b32 s6, 01079; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x91080; GFX6-NEXT:    v_mov_b32_e32 v0, s61081; GFX6-NEXT:    v_mov_b32_e32 v2, 11082; GFX6-NEXT:    s_waitcnt lgkmcnt(0)1083; GFX6-NEXT:    buffer_atomic_add v2, v[0:1], s[0:3], 0 idxen offen glc1084; GFX6-NEXT:    s_mov_b32 s7, 0xf0001085; GFX6-NEXT:    s_mov_b32 s6, -11086; GFX6-NEXT:    s_waitcnt vmcnt(0)1087; GFX6-NEXT:    buffer_store_dword v2, off, s[4:7], 01088; GFX6-NEXT:    s_endpgm1089;1090; GFX8-LABEL: add_i32_varying_offset:1091; GFX8:       ; %bb.0: ; %entry1092; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x341093; GFX8-NEXT:    s_mov_b32 s6, 01094; GFX8-NEXT:    v_mov_b32_e32 v1, v01095; GFX8-NEXT:    v_mov_b32_e32 v0, s61096; GFX8-NEXT:    v_mov_b32_e32 v2, 11097; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1098; GFX8-NEXT:    buffer_atomic_add v2, v[0:1], s[0:3], 0 idxen offen glc1099; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241100; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1101; GFX8-NEXT:    v_mov_b32_e32 v0, s01102; GFX8-NEXT:    v_mov_b32_e32 v1, s11103; GFX8-NEXT:    s_waitcnt vmcnt(0)1104; GFX8-NEXT:    flat_store_dword v[0:1], v21105; GFX8-NEXT:    s_endpgm1106;1107; GFX9-LABEL: add_i32_varying_offset:1108; GFX9:       ; %bb.0: ; %entry1109; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x341110; GFX9-NEXT:    s_mov_b32 s6, 01111; GFX9-NEXT:    v_mov_b32_e32 v1, v01112; GFX9-NEXT:    v_mov_b32_e32 v0, s61113; GFX9-NEXT:    v_mov_b32_e32 v2, 11114; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1115; GFX9-NEXT:    buffer_atomic_add v2, v[0:1], s[0:3], 0 idxen offen glc1116; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241117; GFX9-NEXT:    v_mov_b32_e32 v0, 01118; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1119; GFX9-NEXT:    global_store_dword v0, v2, s[0:1]1120; GFX9-NEXT:    s_endpgm1121;1122; GFX10-LABEL: add_i32_varying_offset:1123; GFX10:       ; %bb.0: ; %entry1124; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x341125; GFX10-NEXT:    s_mov_b32 s6, 01126; GFX10-NEXT:    v_mov_b32_e32 v1, v01127; GFX10-NEXT:    v_mov_b32_e32 v0, s61128; GFX10-NEXT:    v_mov_b32_e32 v2, 11129; GFX10-NEXT:    s_waitcnt lgkmcnt(0)1130; GFX10-NEXT:    buffer_atomic_add v2, v[0:1], s[0:3], 0 idxen offen glc1131; GFX10-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)1132; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241133; GFX10-NEXT:    v_mov_b32_e32 v0, 01134; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1135; GFX10-NEXT:    global_store_dword v0, v2, s[0:1]1136; GFX10-NEXT:    s_endpgm1137;1138; GFX11W64-LABEL: add_i32_varying_offset:1139; GFX11W64:       ; %bb.0: ; %entry1140; GFX11W64-NEXT:    s_load_b128 s[0:3], s[4:5], 0x341141; GFX11W64-NEXT:    s_mov_b32 s6, 01142; GFX11W64-NEXT:    v_and_b32_e32 v1, 0x3ff, v01143; GFX11W64-NEXT:    v_mov_b32_e32 v0, s61144; GFX11W64-NEXT:    v_mov_b32_e32 v2, 11145; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)1146; GFX11W64-NEXT:    buffer_atomic_add_u32 v2, v[0:1], s[0:3], 0 idxen offen glc1147; GFX11W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241148; GFX11W64-NEXT:    v_mov_b32_e32 v0, 01149; GFX11W64-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1150; GFX11W64-NEXT:    global_store_b32 v0, v2, s[0:1]1151; GFX11W64-NEXT:    s_endpgm1152;1153; GFX11W32-LABEL: add_i32_varying_offset:1154; GFX11W32:       ; %bb.0: ; %entry1155; GFX11W32-NEXT:    s_load_b128 s[0:3], s[4:5], 0x341156; GFX11W32-NEXT:    s_mov_b32 s6, 01157; GFX11W32-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)1158; GFX11W32-NEXT:    v_dual_mov_b32 v0, s6 :: v_dual_and_b32 v1, 0x3ff, v01159; GFX11W32-NEXT:    v_mov_b32_e32 v2, 11160; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)1161; GFX11W32-NEXT:    buffer_atomic_add_u32 v2, v[0:1], s[0:3], 0 idxen offen glc1162; GFX11W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241163; GFX11W32-NEXT:    v_mov_b32_e32 v0, 01164; GFX11W32-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1165; GFX11W32-NEXT:    global_store_b32 v0, v2, s[0:1]1166; GFX11W32-NEXT:    s_endpgm1167;1168; GFX12W64-LABEL: add_i32_varying_offset:1169; GFX12W64:       ; %bb.0: ; %entry1170; GFX12W64-NEXT:    s_load_b128 s[0:3], s[4:5], 0x341171; GFX12W64-NEXT:    v_and_b32_e32 v1, 0x3ff, v01172; GFX12W64-NEXT:    v_mov_b32_e32 v0, 01173; GFX12W64-NEXT:    v_mov_b32_e32 v2, 11174; GFX12W64-NEXT:    s_wait_kmcnt 0x01175; GFX12W64-NEXT:    buffer_atomic_add_u32 v2, v[0:1], s[0:3], null idxen offen th:TH_ATOMIC_RETURN1176; GFX12W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241177; GFX12W64-NEXT:    s_wait_loadcnt 0x01178; GFX12W64-NEXT:    s_wait_kmcnt 0x01179; GFX12W64-NEXT:    global_store_b32 v0, v2, s[0:1]1180; GFX12W64-NEXT:    s_endpgm1181;1182; GFX12W32-LABEL: add_i32_varying_offset:1183; GFX12W32:       ; %bb.0: ; %entry1184; GFX12W32-NEXT:    s_load_b128 s[0:3], s[4:5], 0x341185; GFX12W32-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_and_b32 v1, 0x3ff, v01186; GFX12W32-NEXT:    v_mov_b32_e32 v2, 11187; GFX12W32-NEXT:    s_wait_kmcnt 0x01188; GFX12W32-NEXT:    buffer_atomic_add_u32 v2, v[0:1], s[0:3], null idxen offen th:TH_ATOMIC_RETURN1189; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241190; GFX12W32-NEXT:    s_wait_loadcnt 0x01191; GFX12W32-NEXT:    s_wait_kmcnt 0x01192; GFX12W32-NEXT:    global_store_b32 v0, v2, s[0:1]1193; GFX12W32-NEXT:    s_endpgm1194entry:1195  %lane = call i32 @llvm.amdgcn.workitem.id.x()1196  %old = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add(i32 1, ptr addrspace(8) %inout, i32 0, i32 %lane, i32 0, i32 0)1197  store i32 %old, ptr addrspace(1) %out1198  ret void1199}1200 1201define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace(8) %inout) {1202; GFX6-LABEL: sub_i32_constant:1203; GFX6:       ; %bb.0: ; %entry1204; GFX6-NEXT:    s_mov_b64 s[2:3], exec1205; GFX6-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 01206; GFX6-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v01207; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01208; GFX6-NEXT:    ; implicit-def: $vgpr11209; GFX6-NEXT:    s_and_saveexec_b64 s[0:1], vcc1210; GFX6-NEXT:    s_cbranch_execz .LBB5_21211; GFX6-NEXT:  ; %bb.1:1212; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xd1213; GFX6-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1214; GFX6-NEXT:    s_mul_i32 s2, s2, 51215; GFX6-NEXT:    v_mov_b32_e32 v1, s21216; GFX6-NEXT:    v_mov_b32_e32 v2, 01217; GFX6-NEXT:    s_waitcnt lgkmcnt(0)1218; GFX6-NEXT:    buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc1219; GFX6-NEXT:  .LBB5_2:1220; GFX6-NEXT:    s_or_b64 exec, exec, s[0:1]1221; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x91222; GFX6-NEXT:    s_mov_b32 s3, 0xf0001223; GFX6-NEXT:    s_mov_b32 s2, -11224; GFX6-NEXT:    s_waitcnt vmcnt(0)1225; GFX6-NEXT:    v_readfirstlane_b32 s4, v11226; GFX6-NEXT:    v_mul_u32_u24_e32 v0, 5, v01227; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s4, v01228; GFX6-NEXT:    s_waitcnt lgkmcnt(0)1229; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 01230; GFX6-NEXT:    s_endpgm1231;1232; GFX8-LABEL: sub_i32_constant:1233; GFX8:       ; %bb.0: ; %entry1234; GFX8-NEXT:    s_mov_b64 s[2:3], exec1235; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01236; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v01237; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01238; GFX8-NEXT:    ; implicit-def: $vgpr11239; GFX8-NEXT:    s_and_saveexec_b64 s[0:1], vcc1240; GFX8-NEXT:    s_cbranch_execz .LBB5_21241; GFX8-NEXT:  ; %bb.1:1242; GFX8-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341243; GFX8-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1244; GFX8-NEXT:    s_mul_i32 s2, s2, 51245; GFX8-NEXT:    v_mov_b32_e32 v1, s21246; GFX8-NEXT:    v_mov_b32_e32 v2, 01247; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1248; GFX8-NEXT:    buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc1249; GFX8-NEXT:  .LBB5_2:1250; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]1251; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241252; GFX8-NEXT:    s_waitcnt vmcnt(0)1253; GFX8-NEXT:    v_readfirstlane_b32 s2, v11254; GFX8-NEXT:    v_mul_u32_u24_e32 v0, 5, v01255; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, s2, v01256; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1257; GFX8-NEXT:    v_mov_b32_e32 v0, s01258; GFX8-NEXT:    v_mov_b32_e32 v1, s11259; GFX8-NEXT:    flat_store_dword v[0:1], v21260; GFX8-NEXT:    s_endpgm1261;1262; GFX9-LABEL: sub_i32_constant:1263; GFX9:       ; %bb.0: ; %entry1264; GFX9-NEXT:    s_mov_b64 s[2:3], exec1265; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01266; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v01267; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01268; GFX9-NEXT:    ; implicit-def: $vgpr11269; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc1270; GFX9-NEXT:    s_cbranch_execz .LBB5_21271; GFX9-NEXT:  ; %bb.1:1272; GFX9-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341273; GFX9-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1274; GFX9-NEXT:    s_mul_i32 s2, s2, 51275; GFX9-NEXT:    v_mov_b32_e32 v1, s21276; GFX9-NEXT:    v_mov_b32_e32 v2, 01277; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1278; GFX9-NEXT:    buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc1279; GFX9-NEXT:  .LBB5_2:1280; GFX9-NEXT:    s_or_b64 exec, exec, s[0:1]1281; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241282; GFX9-NEXT:    s_waitcnt vmcnt(0)1283; GFX9-NEXT:    v_readfirstlane_b32 s2, v11284; GFX9-NEXT:    v_mul_u32_u24_e32 v0, 5, v01285; GFX9-NEXT:    v_mov_b32_e32 v2, 01286; GFX9-NEXT:    v_sub_u32_e32 v0, s2, v01287; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1288; GFX9-NEXT:    global_store_dword v2, v0, s[0:1]1289; GFX9-NEXT:    s_endpgm1290;1291; GFX10W64-LABEL: sub_i32_constant:1292; GFX10W64:       ; %bb.0: ; %entry1293; GFX10W64-NEXT:    s_mov_b64 s[2:3], exec1294; GFX10W64-NEXT:    ; implicit-def: $vgpr11295; GFX10W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01296; GFX10W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v01297; GFX10W64-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01298; GFX10W64-NEXT:    s_and_saveexec_b64 s[0:1], vcc1299; GFX10W64-NEXT:    s_cbranch_execz .LBB5_21300; GFX10W64-NEXT:  ; %bb.1:1301; GFX10W64-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341302; GFX10W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1303; GFX10W64-NEXT:    v_mov_b32_e32 v2, 01304; GFX10W64-NEXT:    s_mul_i32 s2, s2, 51305; GFX10W64-NEXT:    v_mov_b32_e32 v1, s21306; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)1307; GFX10W64-NEXT:    buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc1308; GFX10W64-NEXT:  .LBB5_2:1309; GFX10W64-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)1310; GFX10W64-NEXT:    s_or_b64 exec, exec, s[0:1]1311; GFX10W64-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241312; GFX10W64-NEXT:    s_waitcnt vmcnt(0)1313; GFX10W64-NEXT:    v_readfirstlane_b32 s2, v11314; GFX10W64-NEXT:    v_mul_u32_u24_e32 v0, 5, v01315; GFX10W64-NEXT:    v_mov_b32_e32 v1, 01316; GFX10W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v01317; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)1318; GFX10W64-NEXT:    global_store_dword v1, v0, s[0:1]1319; GFX10W64-NEXT:    s_endpgm1320;1321; GFX10W32-LABEL: sub_i32_constant:1322; GFX10W32:       ; %bb.0: ; %entry1323; GFX10W32-NEXT:    s_mov_b32 s1, exec_lo1324; GFX10W32-NEXT:    ; implicit-def: $vgpr11325; GFX10W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s1, 01326; GFX10W32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v01327; GFX10W32-NEXT:    s_and_saveexec_b32 s0, vcc_lo1328; GFX10W32-NEXT:    s_cbranch_execz .LBB5_21329; GFX10W32-NEXT:  ; %bb.1:1330; GFX10W32-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341331; GFX10W32-NEXT:    s_bcnt1_i32_b32 s1, s11332; GFX10W32-NEXT:    v_mov_b32_e32 v2, 01333; GFX10W32-NEXT:    s_mul_i32 s1, s1, 51334; GFX10W32-NEXT:    v_mov_b32_e32 v1, s11335; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)1336; GFX10W32-NEXT:    buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc1337; GFX10W32-NEXT:  .LBB5_2:1338; GFX10W32-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)1339; GFX10W32-NEXT:    s_or_b32 exec_lo, exec_lo, s01340; GFX10W32-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241341; GFX10W32-NEXT:    s_waitcnt vmcnt(0)1342; GFX10W32-NEXT:    v_readfirstlane_b32 s2, v11343; GFX10W32-NEXT:    v_mul_u32_u24_e32 v0, 5, v01344; GFX10W32-NEXT:    v_mov_b32_e32 v1, 01345; GFX10W32-NEXT:    v_sub_nc_u32_e32 v0, s2, v01346; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)1347; GFX10W32-NEXT:    global_store_dword v1, v0, s[0:1]1348; GFX10W32-NEXT:    s_endpgm1349;1350; GFX11W64-LABEL: sub_i32_constant:1351; GFX11W64:       ; %bb.0: ; %entry1352; GFX11W64-NEXT:    s_mov_b64 s[2:3], exec1353; GFX11W64-NEXT:    s_mov_b64 s[0:1], exec1354; GFX11W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01355; GFX11W64-NEXT:    ; implicit-def: $vgpr11356; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1357; GFX11W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v01358; GFX11W64-NEXT:    v_cmpx_eq_u32_e32 0, v01359; GFX11W64-NEXT:    s_cbranch_execz .LBB5_21360; GFX11W64-NEXT:  ; %bb.1:1361; GFX11W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x341362; GFX11W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1363; GFX11W64-NEXT:    v_mov_b32_e32 v2, 01364; GFX11W64-NEXT:    s_mul_i32 s2, s2, 51365; GFX11W64-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)1366; GFX11W64-NEXT:    v_mov_b32_e32 v1, s21367; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)1368; GFX11W64-NEXT:    buffer_atomic_sub_u32 v1, v2, s[8:11], 0 idxen glc1369; GFX11W64-NEXT:  .LBB5_2:1370; GFX11W64-NEXT:    s_or_b64 exec, exec, s[0:1]1371; GFX11W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241372; GFX11W64-NEXT:    s_waitcnt vmcnt(0)1373; GFX11W64-NEXT:    v_readfirstlane_b32 s2, v11374; GFX11W64-NEXT:    v_mul_u32_u24_e32 v0, 5, v01375; GFX11W64-NEXT:    v_mov_b32_e32 v1, 01376; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)1377; GFX11W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v01378; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)1379; GFX11W64-NEXT:    global_store_b32 v1, v0, s[0:1]1380; GFX11W64-NEXT:    s_endpgm1381;1382; GFX11W32-LABEL: sub_i32_constant:1383; GFX11W32:       ; %bb.0: ; %entry1384; GFX11W32-NEXT:    s_mov_b32 s1, exec_lo1385; GFX11W32-NEXT:    s_mov_b32 s0, exec_lo1386; GFX11W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s1, 01387; GFX11W32-NEXT:    ; implicit-def: $vgpr11388; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)1389; GFX11W32-NEXT:    v_cmpx_eq_u32_e32 0, v01390; GFX11W32-NEXT:    s_cbranch_execz .LBB5_21391; GFX11W32-NEXT:  ; %bb.1:1392; GFX11W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x341393; GFX11W32-NEXT:    s_bcnt1_i32_b32 s1, s11394; GFX11W32-NEXT:    v_mov_b32_e32 v2, 01395; GFX11W32-NEXT:    s_mul_i32 s1, s1, 51396; GFX11W32-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)1397; GFX11W32-NEXT:    v_mov_b32_e32 v1, s11398; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)1399; GFX11W32-NEXT:    buffer_atomic_sub_u32 v1, v2, s[8:11], 0 idxen glc1400; GFX11W32-NEXT:  .LBB5_2:1401; GFX11W32-NEXT:    s_or_b32 exec_lo, exec_lo, s01402; GFX11W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241403; GFX11W32-NEXT:    s_waitcnt vmcnt(0)1404; GFX11W32-NEXT:    v_readfirstlane_b32 s2, v11405; GFX11W32-NEXT:    v_mul_u32_u24_e32 v0, 5, v01406; GFX11W32-NEXT:    v_mov_b32_e32 v1, 01407; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)1408; GFX11W32-NEXT:    v_sub_nc_u32_e32 v0, s2, v01409; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)1410; GFX11W32-NEXT:    global_store_b32 v1, v0, s[0:1]1411; GFX11W32-NEXT:    s_endpgm1412;1413; GFX12W64-LABEL: sub_i32_constant:1414; GFX12W64:       ; %bb.0: ; %entry1415; GFX12W64-NEXT:    s_mov_b64 s[2:3], exec1416; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec1417; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01418; GFX12W64-NEXT:    ; implicit-def: $vgpr11419; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1420; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v01421; GFX12W64-NEXT:    v_cmpx_eq_u32_e32 0, v01422; GFX12W64-NEXT:    s_cbranch_execz .LBB5_21423; GFX12W64-NEXT:  ; %bb.1:1424; GFX12W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x341425; GFX12W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1426; GFX12W64-NEXT:    v_mov_b32_e32 v2, 01427; GFX12W64-NEXT:    s_wait_alu depctr_sa_sdst(0)1428; GFX12W64-NEXT:    s_mul_i32 s2, s2, 51429; GFX12W64-NEXT:    s_wait_alu depctr_sa_sdst(0)1430; GFX12W64-NEXT:    v_mov_b32_e32 v1, s21431; GFX12W64-NEXT:    s_wait_kmcnt 0x01432; GFX12W64-NEXT:    buffer_atomic_sub_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN1433; GFX12W64-NEXT:  .LBB5_2:1434; GFX12W64-NEXT:    s_or_b64 exec, exec, s[0:1]1435; GFX12W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241436; GFX12W64-NEXT:    s_wait_loadcnt 0x01437; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v11438; GFX12W64-NEXT:    v_mul_u32_u24_e32 v0, 5, v01439; GFX12W64-NEXT:    v_mov_b32_e32 v1, 01440; GFX12W64-NEXT:    s_wait_alu depctr_va_sdst(0)1441; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)1442; GFX12W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v01443; GFX12W64-NEXT:    s_wait_kmcnt 0x01444; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]1445; GFX12W64-NEXT:    s_endpgm1446;1447; GFX12W32-LABEL: sub_i32_constant:1448; GFX12W32:       ; %bb.0: ; %entry1449; GFX12W32-NEXT:    s_mov_b32 s1, exec_lo1450; GFX12W32-NEXT:    s_mov_b32 s0, exec_lo1451; GFX12W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s1, 01452; GFX12W32-NEXT:    ; implicit-def: $vgpr11453; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)1454; GFX12W32-NEXT:    v_cmpx_eq_u32_e32 0, v01455; GFX12W32-NEXT:    s_cbranch_execz .LBB5_21456; GFX12W32-NEXT:  ; %bb.1:1457; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x341458; GFX12W32-NEXT:    s_bcnt1_i32_b32 s1, s11459; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)1460; GFX12W32-NEXT:    s_mul_i32 s1, s1, 51461; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)1462; GFX12W32-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s11463; GFX12W32-NEXT:    s_wait_kmcnt 0x01464; GFX12W32-NEXT:    buffer_atomic_sub_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN1465; GFX12W32-NEXT:  .LBB5_2:1466; GFX12W32-NEXT:    s_or_b32 exec_lo, exec_lo, s01467; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241468; GFX12W32-NEXT:    s_wait_loadcnt 0x01469; GFX12W32-NEXT:    v_readfirstlane_b32 s2, v11470; GFX12W32-NEXT:    v_mul_u32_u24_e32 v0, 5, v01471; GFX12W32-NEXT:    v_mov_b32_e32 v1, 01472; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)1473; GFX12W32-NEXT:    v_sub_nc_u32_e32 v0, s2, v01474; GFX12W32-NEXT:    s_wait_kmcnt 0x01475; GFX12W32-NEXT:    global_store_b32 v1, v0, s[0:1]1476; GFX12W32-NEXT:    s_endpgm1477entry:1478  %old = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.sub(i32 5, ptr addrspace(8) %inout, i32 0, i32 0, i32 0, i32 0)1479  store i32 %old, ptr addrspace(1) %out1480  ret void1481}1482 1483define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(8) %inout, i32 %subitive) {1484; GFX6-LABEL: sub_i32_uniform:1485; GFX6:       ; %bb.0: ; %entry1486; GFX6-NEXT:    s_mov_b64 s[2:3], exec1487; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x111488; GFX6-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 01489; GFX6-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v01490; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01491; GFX6-NEXT:    ; implicit-def: $vgpr11492; GFX6-NEXT:    s_and_saveexec_b64 s[0:1], vcc1493; GFX6-NEXT:    s_cbranch_execz .LBB6_21494; GFX6-NEXT:  ; %bb.1:1495; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xd1496; GFX6-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1497; GFX6-NEXT:    s_waitcnt lgkmcnt(0)1498; GFX6-NEXT:    s_mul_i32 s2, s6, s21499; GFX6-NEXT:    v_mov_b32_e32 v1, s21500; GFX6-NEXT:    v_mov_b32_e32 v2, 01501; GFX6-NEXT:    buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc1502; GFX6-NEXT:  .LBB6_2:1503; GFX6-NEXT:    s_or_b64 exec, exec, s[0:1]1504; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x91505; GFX6-NEXT:    s_mov_b32 s3, 0xf0001506; GFX6-NEXT:    s_mov_b32 s2, -11507; GFX6-NEXT:    s_waitcnt vmcnt(0)1508; GFX6-NEXT:    v_readfirstlane_b32 s4, v11509; GFX6-NEXT:    s_waitcnt lgkmcnt(0)1510; GFX6-NEXT:    v_mul_lo_u32 v0, s6, v01511; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s4, v01512; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 01513; GFX6-NEXT:    s_endpgm1514;1515; GFX8-LABEL: sub_i32_uniform:1516; GFX8:       ; %bb.0: ; %entry1517; GFX8-NEXT:    s_load_dword s6, s[4:5], 0x441518; GFX8-NEXT:    s_mov_b64 s[2:3], exec1519; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01520; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v01521; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01522; GFX8-NEXT:    ; implicit-def: $vgpr11523; GFX8-NEXT:    s_and_saveexec_b64 s[0:1], vcc1524; GFX8-NEXT:    s_cbranch_execz .LBB6_21525; GFX8-NEXT:  ; %bb.1:1526; GFX8-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341527; GFX8-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1528; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1529; GFX8-NEXT:    s_mul_i32 s2, s6, s21530; GFX8-NEXT:    v_mov_b32_e32 v1, s21531; GFX8-NEXT:    v_mov_b32_e32 v2, 01532; GFX8-NEXT:    buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc1533; GFX8-NEXT:  .LBB6_2:1534; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]1535; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241536; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1537; GFX8-NEXT:    v_mul_lo_u32 v0, s6, v01538; GFX8-NEXT:    s_waitcnt vmcnt(0)1539; GFX8-NEXT:    v_readfirstlane_b32 s2, v11540; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, s2, v01541; GFX8-NEXT:    v_mov_b32_e32 v0, s01542; GFX8-NEXT:    v_mov_b32_e32 v1, s11543; GFX8-NEXT:    flat_store_dword v[0:1], v21544; GFX8-NEXT:    s_endpgm1545;1546; GFX9-LABEL: sub_i32_uniform:1547; GFX9:       ; %bb.0: ; %entry1548; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x441549; GFX9-NEXT:    s_mov_b64 s[2:3], exec1550; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01551; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v01552; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01553; GFX9-NEXT:    ; implicit-def: $vgpr11554; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc1555; GFX9-NEXT:    s_cbranch_execz .LBB6_21556; GFX9-NEXT:  ; %bb.1:1557; GFX9-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341558; GFX9-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1559; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1560; GFX9-NEXT:    s_mul_i32 s2, s6, s21561; GFX9-NEXT:    v_mov_b32_e32 v1, s21562; GFX9-NEXT:    v_mov_b32_e32 v2, 01563; GFX9-NEXT:    buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc1564; GFX9-NEXT:  .LBB6_2:1565; GFX9-NEXT:    s_or_b64 exec, exec, s[0:1]1566; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241567; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1568; GFX9-NEXT:    v_mul_lo_u32 v0, s6, v01569; GFX9-NEXT:    s_waitcnt vmcnt(0)1570; GFX9-NEXT:    v_readfirstlane_b32 s2, v11571; GFX9-NEXT:    v_mov_b32_e32 v2, 01572; GFX9-NEXT:    v_sub_u32_e32 v0, s2, v01573; GFX9-NEXT:    global_store_dword v2, v0, s[0:1]1574; GFX9-NEXT:    s_endpgm1575;1576; GFX10W64-LABEL: sub_i32_uniform:1577; GFX10W64:       ; %bb.0: ; %entry1578; GFX10W64-NEXT:    s_load_dword s6, s[4:5], 0x441579; GFX10W64-NEXT:    s_mov_b64 s[2:3], exec1580; GFX10W64-NEXT:    ; implicit-def: $vgpr11581; GFX10W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01582; GFX10W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v01583; GFX10W64-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01584; GFX10W64-NEXT:    s_and_saveexec_b64 s[0:1], vcc1585; GFX10W64-NEXT:    s_cbranch_execz .LBB6_21586; GFX10W64-NEXT:  ; %bb.1:1587; GFX10W64-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341588; GFX10W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1589; GFX10W64-NEXT:    v_mov_b32_e32 v2, 01590; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)1591; GFX10W64-NEXT:    s_mul_i32 s2, s6, s21592; GFX10W64-NEXT:    v_mov_b32_e32 v1, s21593; GFX10W64-NEXT:    buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc1594; GFX10W64-NEXT:  .LBB6_2:1595; GFX10W64-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)1596; GFX10W64-NEXT:    s_or_b64 exec, exec, s[0:1]1597; GFX10W64-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241598; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)1599; GFX10W64-NEXT:    v_mul_lo_u32 v0, s6, v01600; GFX10W64-NEXT:    s_waitcnt vmcnt(0)1601; GFX10W64-NEXT:    v_readfirstlane_b32 s2, v11602; GFX10W64-NEXT:    v_mov_b32_e32 v1, 01603; GFX10W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v01604; GFX10W64-NEXT:    global_store_dword v1, v0, s[0:1]1605; GFX10W64-NEXT:    s_endpgm1606;1607; GFX10W32-LABEL: sub_i32_uniform:1608; GFX10W32:       ; %bb.0: ; %entry1609; GFX10W32-NEXT:    s_load_dword s0, s[4:5], 0x441610; GFX10W32-NEXT:    s_mov_b32 s2, exec_lo1611; GFX10W32-NEXT:    ; implicit-def: $vgpr11612; GFX10W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01613; GFX10W32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v01614; GFX10W32-NEXT:    s_and_saveexec_b32 s1, vcc_lo1615; GFX10W32-NEXT:    s_cbranch_execz .LBB6_21616; GFX10W32-NEXT:  ; %bb.1:1617; GFX10W32-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341618; GFX10W32-NEXT:    s_bcnt1_i32_b32 s2, s21619; GFX10W32-NEXT:    v_mov_b32_e32 v2, 01620; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)1621; GFX10W32-NEXT:    s_mul_i32 s2, s0, s21622; GFX10W32-NEXT:    v_mov_b32_e32 v1, s21623; GFX10W32-NEXT:    buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc1624; GFX10W32-NEXT:  .LBB6_2:1625; GFX10W32-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)1626; GFX10W32-NEXT:    s_or_b32 exec_lo, exec_lo, s11627; GFX10W32-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x241628; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)1629; GFX10W32-NEXT:    v_mul_lo_u32 v0, s0, v01630; GFX10W32-NEXT:    s_waitcnt vmcnt(0)1631; GFX10W32-NEXT:    v_readfirstlane_b32 s0, v11632; GFX10W32-NEXT:    v_mov_b32_e32 v1, 01633; GFX10W32-NEXT:    v_sub_nc_u32_e32 v0, s0, v01634; GFX10W32-NEXT:    global_store_dword v1, v0, s[2:3]1635; GFX10W32-NEXT:    s_endpgm1636;1637; GFX11W64-LABEL: sub_i32_uniform:1638; GFX11W64:       ; %bb.0: ; %entry1639; GFX11W64-NEXT:    s_load_b32 s6, s[4:5], 0x441640; GFX11W64-NEXT:    s_mov_b64 s[2:3], exec1641; GFX11W64-NEXT:    s_mov_b64 s[0:1], exec1642; GFX11W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01643; GFX11W64-NEXT:    ; implicit-def: $vgpr11644; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1645; GFX11W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v01646; GFX11W64-NEXT:    v_cmpx_eq_u32_e32 0, v01647; GFX11W64-NEXT:    s_cbranch_execz .LBB6_21648; GFX11W64-NEXT:  ; %bb.1:1649; GFX11W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x341650; GFX11W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1651; GFX11W64-NEXT:    v_mov_b32_e32 v2, 01652; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)1653; GFX11W64-NEXT:    s_mul_i32 s2, s6, s21654; GFX11W64-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)1655; GFX11W64-NEXT:    v_mov_b32_e32 v1, s21656; GFX11W64-NEXT:    buffer_atomic_sub_u32 v1, v2, s[8:11], 0 idxen glc1657; GFX11W64-NEXT:  .LBB6_2:1658; GFX11W64-NEXT:    s_or_b64 exec, exec, s[0:1]1659; GFX11W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241660; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)1661; GFX11W64-NEXT:    v_mul_lo_u32 v0, s6, v01662; GFX11W64-NEXT:    s_waitcnt vmcnt(0)1663; GFX11W64-NEXT:    v_readfirstlane_b32 s2, v11664; GFX11W64-NEXT:    v_mov_b32_e32 v1, 01665; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)1666; GFX11W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v01667; GFX11W64-NEXT:    global_store_b32 v1, v0, s[0:1]1668; GFX11W64-NEXT:    s_endpgm1669;1670; GFX11W32-LABEL: sub_i32_uniform:1671; GFX11W32:       ; %bb.0: ; %entry1672; GFX11W32-NEXT:    s_load_b32 s0, s[4:5], 0x441673; GFX11W32-NEXT:    s_mov_b32 s2, exec_lo1674; GFX11W32-NEXT:    s_mov_b32 s1, exec_lo1675; GFX11W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01676; GFX11W32-NEXT:    ; implicit-def: $vgpr11677; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)1678; GFX11W32-NEXT:    v_cmpx_eq_u32_e32 0, v01679; GFX11W32-NEXT:    s_cbranch_execz .LBB6_21680; GFX11W32-NEXT:  ; %bb.1:1681; GFX11W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x341682; GFX11W32-NEXT:    s_bcnt1_i32_b32 s2, s21683; GFX11W32-NEXT:    v_mov_b32_e32 v2, 01684; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)1685; GFX11W32-NEXT:    s_mul_i32 s2, s0, s21686; GFX11W32-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)1687; GFX11W32-NEXT:    v_mov_b32_e32 v1, s21688; GFX11W32-NEXT:    buffer_atomic_sub_u32 v1, v2, s[8:11], 0 idxen glc1689; GFX11W32-NEXT:  .LBB6_2:1690; GFX11W32-NEXT:    s_or_b32 exec_lo, exec_lo, s11691; GFX11W32-NEXT:    s_load_b64 s[2:3], s[4:5], 0x241692; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)1693; GFX11W32-NEXT:    v_mul_lo_u32 v0, s0, v01694; GFX11W32-NEXT:    s_waitcnt vmcnt(0)1695; GFX11W32-NEXT:    v_readfirstlane_b32 s0, v11696; GFX11W32-NEXT:    v_mov_b32_e32 v1, 01697; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)1698; GFX11W32-NEXT:    v_sub_nc_u32_e32 v0, s0, v01699; GFX11W32-NEXT:    global_store_b32 v1, v0, s[2:3]1700; GFX11W32-NEXT:    s_endpgm1701;1702; GFX12W64-LABEL: sub_i32_uniform:1703; GFX12W64:       ; %bb.0: ; %entry1704; GFX12W64-NEXT:    s_load_b32 s6, s[4:5], 0x441705; GFX12W64-NEXT:    s_mov_b64 s[2:3], exec1706; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec1707; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01708; GFX12W64-NEXT:    ; implicit-def: $vgpr11709; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1710; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v01711; GFX12W64-NEXT:    v_cmpx_eq_u32_e32 0, v01712; GFX12W64-NEXT:    s_cbranch_execz .LBB6_21713; GFX12W64-NEXT:  ; %bb.1:1714; GFX12W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x341715; GFX12W64-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]1716; GFX12W64-NEXT:    v_mov_b32_e32 v2, 01717; GFX12W64-NEXT:    s_wait_kmcnt 0x01718; GFX12W64-NEXT:    s_wait_alu depctr_sa_sdst(0)1719; GFX12W64-NEXT:    s_mul_i32 s2, s6, s21720; GFX12W64-NEXT:    s_wait_alu depctr_sa_sdst(0)1721; GFX12W64-NEXT:    v_mov_b32_e32 v1, s21722; GFX12W64-NEXT:    buffer_atomic_sub_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN1723; GFX12W64-NEXT:  .LBB6_2:1724; GFX12W64-NEXT:    s_or_b64 exec, exec, s[0:1]1725; GFX12W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241726; GFX12W64-NEXT:    s_wait_kmcnt 0x01727; GFX12W64-NEXT:    v_mul_lo_u32 v0, s6, v01728; GFX12W64-NEXT:    s_wait_loadcnt 0x01729; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v11730; GFX12W64-NEXT:    v_mov_b32_e32 v1, 01731; GFX12W64-NEXT:    s_wait_alu depctr_va_sdst(0)1732; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)1733; GFX12W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v01734; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]1735; GFX12W64-NEXT:    s_endpgm1736;1737; GFX12W32-LABEL: sub_i32_uniform:1738; GFX12W32:       ; %bb.0: ; %entry1739; GFX12W32-NEXT:    s_load_b32 s0, s[4:5], 0x441740; GFX12W32-NEXT:    s_mov_b32 s2, exec_lo1741; GFX12W32-NEXT:    s_mov_b32 s1, exec_lo1742; GFX12W32-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 01743; GFX12W32-NEXT:    ; implicit-def: $vgpr11744; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)1745; GFX12W32-NEXT:    v_cmpx_eq_u32_e32 0, v01746; GFX12W32-NEXT:    s_cbranch_execz .LBB6_21747; GFX12W32-NEXT:  ; %bb.1:1748; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x341749; GFX12W32-NEXT:    s_bcnt1_i32_b32 s2, s21750; GFX12W32-NEXT:    s_wait_kmcnt 0x01751; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)1752; GFX12W32-NEXT:    s_mul_i32 s2, s0, s21753; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)1754; GFX12W32-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s21755; GFX12W32-NEXT:    buffer_atomic_sub_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN1756; GFX12W32-NEXT:  .LBB6_2:1757; GFX12W32-NEXT:    s_or_b32 exec_lo, exec_lo, s11758; GFX12W32-NEXT:    s_load_b64 s[2:3], s[4:5], 0x241759; GFX12W32-NEXT:    s_wait_kmcnt 0x01760; GFX12W32-NEXT:    v_mul_lo_u32 v0, s0, v01761; GFX12W32-NEXT:    s_wait_loadcnt 0x01762; GFX12W32-NEXT:    v_readfirstlane_b32 s0, v11763; GFX12W32-NEXT:    v_mov_b32_e32 v1, 01764; GFX12W32-NEXT:    s_wait_alu depctr_va_sdst(0)1765; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)1766; GFX12W32-NEXT:    v_sub_nc_u32_e32 v0, s0, v01767; GFX12W32-NEXT:    global_store_b32 v1, v0, s[2:3]1768; GFX12W32-NEXT:    s_endpgm1769entry:1770  %old = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.sub(i32 %subitive, ptr addrspace(8) %inout, i32 0, i32 0, i32 0, i32 0)1771  store i32 %old, ptr addrspace(1) %out1772  ret void1773}1774 1775define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addrspace(8) %inout) {1776; GFX6-LABEL: sub_i32_varying_vdata:1777; GFX6:       ; %bb.0: ; %entry1778; GFX6-NEXT:    s_mov_b64 s[0:1], exec1779; GFX6-NEXT:    s_mov_b32 s2, 01780; GFX6-NEXT:    ; implicit-def: $vgpr11781; GFX6-NEXT:  .LBB7_1: ; %ComputeLoop1782; GFX6-NEXT:    ; =>This Inner Loop Header: Depth=11783; GFX6-NEXT:    s_ff1_i32_b64 s3, s[0:1]1784; GFX6-NEXT:    s_mov_b32 m0, s31785; GFX6-NEXT:    v_readlane_b32 s8, v0, s31786; GFX6-NEXT:    v_writelane_b32 v1, s2, m01787; GFX6-NEXT:    s_lshl_b64 s[6:7], 1, s31788; GFX6-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[6:7]1789; GFX6-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 01790; GFX6-NEXT:    s_and_b64 vcc, exec, s[6:7]1791; GFX6-NEXT:    s_add_i32 s2, s2, s81792; GFX6-NEXT:    s_cbranch_vccnz .LBB7_11793; GFX6-NEXT:  ; %bb.2: ; %ComputeEnd1794; GFX6-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 01795; GFX6-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v01796; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01797; GFX6-NEXT:    ; implicit-def: $vgpr01798; GFX6-NEXT:    s_and_saveexec_b64 s[0:1], vcc1799; GFX6-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]1800; GFX6-NEXT:    s_cbranch_execz .LBB7_41801; GFX6-NEXT:  ; %bb.3:1802; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0xd1803; GFX6-NEXT:    v_mov_b32_e32 v0, s21804; GFX6-NEXT:    v_mov_b32_e32 v2, 01805; GFX6-NEXT:    s_waitcnt lgkmcnt(0)1806; GFX6-NEXT:    buffer_atomic_sub v0, v2, s[8:11], 0 idxen glc1807; GFX6-NEXT:  .LBB7_4:1808; GFX6-NEXT:    s_or_b64 exec, exec, s[0:1]1809; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x91810; GFX6-NEXT:    s_mov_b32 s3, 0xf0001811; GFX6-NEXT:    s_mov_b32 s2, -11812; GFX6-NEXT:    s_waitcnt vmcnt(0)1813; GFX6-NEXT:    v_readfirstlane_b32 s4, v01814; GFX6-NEXT:    s_waitcnt expcnt(0)1815; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s4, v11816; GFX6-NEXT:    s_waitcnt lgkmcnt(0)1817; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 01818; GFX6-NEXT:    s_endpgm1819;1820; GFX8-LABEL: sub_i32_varying_vdata:1821; GFX8:       ; %bb.0: ; %entry1822; GFX8-NEXT:    s_mov_b64 s[0:1], exec1823; GFX8-NEXT:    s_mov_b32 s2, 01824; GFX8-NEXT:    ; implicit-def: $vgpr11825; GFX8-NEXT:  .LBB7_1: ; %ComputeLoop1826; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=11827; GFX8-NEXT:    s_ff1_i32_b64 s3, s[0:1]1828; GFX8-NEXT:    s_mov_b32 m0, s31829; GFX8-NEXT:    v_readlane_b32 s6, v0, s31830; GFX8-NEXT:    v_writelane_b32 v1, s2, m01831; GFX8-NEXT:    s_add_i32 s2, s2, s61832; GFX8-NEXT:    s_lshl_b64 s[6:7], 1, s31833; GFX8-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[6:7]1834; GFX8-NEXT:    s_cbranch_scc1 .LBB7_11835; GFX8-NEXT:  ; %bb.2: ; %ComputeEnd1836; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 01837; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v01838; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01839; GFX8-NEXT:    ; implicit-def: $vgpr01840; GFX8-NEXT:    s_and_saveexec_b64 s[0:1], vcc1841; GFX8-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]1842; GFX8-NEXT:    s_cbranch_execz .LBB7_41843; GFX8-NEXT:  ; %bb.3:1844; GFX8-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341845; GFX8-NEXT:    v_mov_b32_e32 v0, s21846; GFX8-NEXT:    v_mov_b32_e32 v2, 01847; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1848; GFX8-NEXT:    buffer_atomic_sub v0, v2, s[8:11], 0 idxen glc1849; GFX8-NEXT:  .LBB7_4:1850; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]1851; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241852; GFX8-NEXT:    s_waitcnt vmcnt(0)1853; GFX8-NEXT:    v_readfirstlane_b32 s2, v01854; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, s2, v11855; GFX8-NEXT:    s_waitcnt lgkmcnt(0)1856; GFX8-NEXT:    v_mov_b32_e32 v0, s01857; GFX8-NEXT:    v_mov_b32_e32 v1, s11858; GFX8-NEXT:    flat_store_dword v[0:1], v21859; GFX8-NEXT:    s_endpgm1860;1861; GFX9-LABEL: sub_i32_varying_vdata:1862; GFX9:       ; %bb.0: ; %entry1863; GFX9-NEXT:    s_mov_b64 s[0:1], exec1864; GFX9-NEXT:    s_mov_b32 s2, 01865; GFX9-NEXT:    ; implicit-def: $vgpr11866; GFX9-NEXT:  .LBB7_1: ; %ComputeLoop1867; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=11868; GFX9-NEXT:    s_ff1_i32_b64 s3, s[0:1]1869; GFX9-NEXT:    s_mov_b32 m0, s31870; GFX9-NEXT:    v_readlane_b32 s6, v0, s31871; GFX9-NEXT:    v_writelane_b32 v1, s2, m01872; GFX9-NEXT:    s_add_i32 s2, s2, s61873; GFX9-NEXT:    s_lshl_b64 s[6:7], 1, s31874; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[6:7]1875; GFX9-NEXT:    s_cbranch_scc1 .LBB7_11876; GFX9-NEXT:  ; %bb.2: ; %ComputeEnd1877; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 01878; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v01879; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01880; GFX9-NEXT:    ; implicit-def: $vgpr01881; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc1882; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]1883; GFX9-NEXT:    s_cbranch_execz .LBB7_41884; GFX9-NEXT:  ; %bb.3:1885; GFX9-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341886; GFX9-NEXT:    v_mov_b32_e32 v0, s21887; GFX9-NEXT:    v_mov_b32_e32 v2, 01888; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1889; GFX9-NEXT:    buffer_atomic_sub v0, v2, s[8:11], 0 idxen glc1890; GFX9-NEXT:  .LBB7_4:1891; GFX9-NEXT:    s_or_b64 exec, exec, s[0:1]1892; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241893; GFX9-NEXT:    s_waitcnt vmcnt(0)1894; GFX9-NEXT:    v_readfirstlane_b32 s2, v01895; GFX9-NEXT:    v_mov_b32_e32 v2, 01896; GFX9-NEXT:    v_sub_u32_e32 v0, s2, v11897; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1898; GFX9-NEXT:    global_store_dword v2, v0, s[0:1]1899; GFX9-NEXT:    s_endpgm1900;1901; GFX10W64-LABEL: sub_i32_varying_vdata:1902; GFX10W64:       ; %bb.0: ; %entry1903; GFX10W64-NEXT:    s_mov_b64 s[0:1], exec1904; GFX10W64-NEXT:    s_mov_b32 s2, 01905; GFX10W64-NEXT:    ; implicit-def: $vgpr11906; GFX10W64-NEXT:  .LBB7_1: ; %ComputeLoop1907; GFX10W64-NEXT:    ; =>This Inner Loop Header: Depth=11908; GFX10W64-NEXT:    s_ff1_i32_b64 s3, s[0:1]1909; GFX10W64-NEXT:    v_readlane_b32 s8, v0, s31910; GFX10W64-NEXT:    v_writelane_b32 v1, s2, s31911; GFX10W64-NEXT:    s_lshl_b64 s[6:7], 1, s31912; GFX10W64-NEXT:    s_add_i32 s2, s2, s81913; GFX10W64-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[6:7]1914; GFX10W64-NEXT:    s_cbranch_scc1 .LBB7_11915; GFX10W64-NEXT:  ; %bb.2: ; %ComputeEnd1916; GFX10W64-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 01917; GFX10W64-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v01918; GFX10W64-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01919; GFX10W64-NEXT:    ; implicit-def: $vgpr01920; GFX10W64-NEXT:    s_and_saveexec_b64 s[0:1], vcc1921; GFX10W64-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]1922; GFX10W64-NEXT:    s_cbranch_execz .LBB7_41923; GFX10W64-NEXT:  ; %bb.3:1924; GFX10W64-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341925; GFX10W64-NEXT:    v_mov_b32_e32 v0, s21926; GFX10W64-NEXT:    v_mov_b32_e32 v2, 01927; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)1928; GFX10W64-NEXT:    buffer_atomic_sub v0, v2, s[8:11], 0 idxen glc1929; GFX10W64-NEXT:  .LBB7_4:1930; GFX10W64-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)1931; GFX10W64-NEXT:    s_or_b64 exec, exec, s[0:1]1932; GFX10W64-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241933; GFX10W64-NEXT:    s_waitcnt vmcnt(0)1934; GFX10W64-NEXT:    v_readfirstlane_b32 s2, v01935; GFX10W64-NEXT:    v_mov_b32_e32 v0, 01936; GFX10W64-NEXT:    v_sub_nc_u32_e32 v1, s2, v11937; GFX10W64-NEXT:    s_waitcnt lgkmcnt(0)1938; GFX10W64-NEXT:    global_store_dword v0, v1, s[0:1]1939; GFX10W64-NEXT:    s_endpgm1940;1941; GFX10W32-LABEL: sub_i32_varying_vdata:1942; GFX10W32:       ; %bb.0: ; %entry1943; GFX10W32-NEXT:    s_mov_b32 s1, exec_lo1944; GFX10W32-NEXT:    s_mov_b32 s0, 01945; GFX10W32-NEXT:    ; implicit-def: $vgpr11946; GFX10W32-NEXT:  .LBB7_1: ; %ComputeLoop1947; GFX10W32-NEXT:    ; =>This Inner Loop Header: Depth=11948; GFX10W32-NEXT:    s_ff1_i32_b32 s2, s11949; GFX10W32-NEXT:    v_readlane_b32 s3, v0, s21950; GFX10W32-NEXT:    v_writelane_b32 v1, s0, s21951; GFX10W32-NEXT:    s_lshl_b32 s2, 1, s21952; GFX10W32-NEXT:    s_add_i32 s0, s0, s31953; GFX10W32-NEXT:    s_andn2_b32 s1, s1, s21954; GFX10W32-NEXT:    s_cbranch_scc1 .LBB7_11955; GFX10W32-NEXT:  ; %bb.2: ; %ComputeEnd1956; GFX10W32-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 01957; GFX10W32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v01958; GFX10W32-NEXT:    ; implicit-def: $vgpr01959; GFX10W32-NEXT:    s_and_saveexec_b32 s1, vcc_lo1960; GFX10W32-NEXT:    s_xor_b32 s1, exec_lo, s11961; GFX10W32-NEXT:    s_cbranch_execz .LBB7_41962; GFX10W32-NEXT:  ; %bb.3:1963; GFX10W32-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x341964; GFX10W32-NEXT:    v_mov_b32_e32 v0, s01965; GFX10W32-NEXT:    v_mov_b32_e32 v2, 01966; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)1967; GFX10W32-NEXT:    buffer_atomic_sub v0, v2, s[8:11], 0 idxen glc1968; GFX10W32-NEXT:  .LBB7_4:1969; GFX10W32-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)1970; GFX10W32-NEXT:    s_or_b32 exec_lo, exec_lo, s11971; GFX10W32-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241972; GFX10W32-NEXT:    s_waitcnt vmcnt(0)1973; GFX10W32-NEXT:    v_readfirstlane_b32 s2, v01974; GFX10W32-NEXT:    v_mov_b32_e32 v0, 01975; GFX10W32-NEXT:    v_sub_nc_u32_e32 v1, s2, v11976; GFX10W32-NEXT:    s_waitcnt lgkmcnt(0)1977; GFX10W32-NEXT:    global_store_dword v0, v1, s[0:1]1978; GFX10W32-NEXT:    s_endpgm1979;1980; GFX11W64-LABEL: sub_i32_varying_vdata:1981; GFX11W64:       ; %bb.0: ; %entry1982; GFX11W64-NEXT:    v_and_b32_e32 v1, 0x3ff, v01983; GFX11W64-NEXT:    s_mov_b64 s[0:1], exec1984; GFX11W64-NEXT:    s_mov_b32 s2, 01985; GFX11W64-NEXT:    ; implicit-def: $vgpr01986; GFX11W64-NEXT:  .LBB7_1: ; %ComputeLoop1987; GFX11W64-NEXT:    ; =>This Inner Loop Header: Depth=11988; GFX11W64-NEXT:    s_ctz_i32_b64 s3, s[0:1]1989; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)1990; GFX11W64-NEXT:    v_readlane_b32 s8, v1, s31991; GFX11W64-NEXT:    v_writelane_b32 v0, s2, s31992; GFX11W64-NEXT:    s_lshl_b64 s[6:7], 1, s31993; GFX11W64-NEXT:    s_add_i32 s2, s2, s81994; GFX11W64-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[6:7]1995; GFX11W64-NEXT:    s_cbranch_scc1 .LBB7_11996; GFX11W64-NEXT:  ; %bb.2: ; %ComputeEnd1997; GFX11W64-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 01998; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1999; GFX11W64-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v12000; GFX11W64-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v12001; GFX11W64-NEXT:    ; implicit-def: $vgpr12002; GFX11W64-NEXT:    s_and_saveexec_b64 s[0:1], vcc2003; GFX11W64-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)2004; GFX11W64-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]2005; GFX11W64-NEXT:    s_cbranch_execz .LBB7_42006; GFX11W64-NEXT:  ; %bb.3:2007; GFX11W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x342008; GFX11W64-NEXT:    v_mov_b32_e32 v1, s22009; GFX11W64-NEXT:    v_mov_b32_e32 v2, 02010; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)2011; GFX11W64-NEXT:    buffer_atomic_sub_u32 v1, v2, s[8:11], 0 idxen glc2012; GFX11W64-NEXT:  .LBB7_4:2013; GFX11W64-NEXT:    s_or_b64 exec, exec, s[0:1]2014; GFX11W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x242015; GFX11W64-NEXT:    s_waitcnt vmcnt(0)2016; GFX11W64-NEXT:    v_readfirstlane_b32 s2, v12017; GFX11W64-NEXT:    v_mov_b32_e32 v1, 02018; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)2019; GFX11W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v02020; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)2021; GFX11W64-NEXT:    global_store_b32 v1, v0, s[0:1]2022; GFX11W64-NEXT:    s_endpgm2023;2024; GFX11W32-LABEL: sub_i32_varying_vdata:2025; GFX11W32:       ; %bb.0: ; %entry2026; GFX11W32-NEXT:    v_and_b32_e32 v1, 0x3ff, v02027; GFX11W32-NEXT:    s_mov_b32 s1, exec_lo2028; GFX11W32-NEXT:    s_mov_b32 s0, 02029; GFX11W32-NEXT:    ; implicit-def: $vgpr02030; GFX11W32-NEXT:  .LBB7_1: ; %ComputeLoop2031; GFX11W32-NEXT:    ; =>This Inner Loop Header: Depth=12032; GFX11W32-NEXT:    s_ctz_i32_b32 s2, s12033; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)2034; GFX11W32-NEXT:    v_readlane_b32 s3, v1, s22035; GFX11W32-NEXT:    v_writelane_b32 v0, s0, s22036; GFX11W32-NEXT:    s_lshl_b32 s2, 1, s22037; GFX11W32-NEXT:    s_add_i32 s0, s0, s32038; GFX11W32-NEXT:    s_and_not1_b32 s1, s1, s22039; GFX11W32-NEXT:    s_cbranch_scc1 .LBB7_12040; GFX11W32-NEXT:  ; %bb.2: ; %ComputeEnd2041; GFX11W32-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 02042; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)2043; GFX11W32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v12044; GFX11W32-NEXT:    ; implicit-def: $vgpr12045; GFX11W32-NEXT:    s_and_saveexec_b32 s1, vcc_lo2046; GFX11W32-NEXT:    s_xor_b32 s1, exec_lo, s12047; GFX11W32-NEXT:    s_cbranch_execz .LBB7_42048; GFX11W32-NEXT:  ; %bb.3:2049; GFX11W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x342050; GFX11W32-NEXT:    v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v2, 02051; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)2052; GFX11W32-NEXT:    buffer_atomic_sub_u32 v1, v2, s[8:11], 0 idxen glc2053; GFX11W32-NEXT:  .LBB7_4:2054; GFX11W32-NEXT:    s_or_b32 exec_lo, exec_lo, s12055; GFX11W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x242056; GFX11W32-NEXT:    s_waitcnt vmcnt(0)2057; GFX11W32-NEXT:    v_readfirstlane_b32 s2, v12058; GFX11W32-NEXT:    v_mov_b32_e32 v1, 02059; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)2060; GFX11W32-NEXT:    v_sub_nc_u32_e32 v0, s2, v02061; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)2062; GFX11W32-NEXT:    global_store_b32 v1, v0, s[0:1]2063; GFX11W32-NEXT:    s_endpgm2064;2065; GFX12W64-LABEL: sub_i32_varying_vdata:2066; GFX12W64:       ; %bb.0: ; %entry2067; GFX12W64-NEXT:    v_and_b32_e32 v1, 0x3ff, v02068; GFX12W64-NEXT:    s_mov_b64 s[0:1], exec2069; GFX12W64-NEXT:    s_mov_b32 s2, 02070; GFX12W64-NEXT:    ; implicit-def: $vgpr02071; GFX12W64-NEXT:  .LBB7_1: ; %ComputeLoop2072; GFX12W64-NEXT:    ; =>This Inner Loop Header: Depth=12073; GFX12W64-NEXT:    s_ctz_i32_b64 s3, s[0:1]2074; GFX12W64-NEXT:    s_wait_alu depctr_sa_sdst(0)2075; GFX12W64-NEXT:    v_readlane_b32 s8, v1, s32076; GFX12W64-NEXT:    v_writelane_b32 v0, s2, s32077; GFX12W64-NEXT:    s_lshl_b64 s[6:7], 1, s32078; GFX12W64-NEXT:    s_add_co_i32 s2, s2, s82079; GFX12W64-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[6:7]2080; GFX12W64-NEXT:    s_cbranch_scc1 .LBB7_12081; GFX12W64-NEXT:  ; %bb.2: ; %ComputeEnd2082; GFX12W64-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 02083; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2084; GFX12W64-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v12085; GFX12W64-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v12086; GFX12W64-NEXT:    ; implicit-def: $vgpr12087; GFX12W64-NEXT:    s_and_saveexec_b64 s[0:1], vcc2088; GFX12W64-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)2089; GFX12W64-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]2090; GFX12W64-NEXT:    s_cbranch_execz .LBB7_42091; GFX12W64-NEXT:  ; %bb.3:2092; GFX12W64-NEXT:    s_load_b128 s[8:11], s[4:5], 0x342093; GFX12W64-NEXT:    v_mov_b32_e32 v2, 02094; GFX12W64-NEXT:    v_mov_b32_e32 v1, s22095; GFX12W64-NEXT:    s_wait_kmcnt 0x02096; GFX12W64-NEXT:    buffer_atomic_sub_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN2097; GFX12W64-NEXT:  .LBB7_4:2098; GFX12W64-NEXT:    s_or_b64 exec, exec, s[0:1]2099; GFX12W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x242100; GFX12W64-NEXT:    s_wait_loadcnt 0x02101; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v12102; GFX12W64-NEXT:    v_mov_b32_e32 v1, 02103; GFX12W64-NEXT:    s_wait_alu depctr_va_sdst(0)2104; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)2105; GFX12W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v02106; GFX12W64-NEXT:    s_wait_kmcnt 0x02107; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]2108; GFX12W64-NEXT:    s_endpgm2109;2110; GFX12W32-LABEL: sub_i32_varying_vdata:2111; GFX12W32:       ; %bb.0: ; %entry2112; GFX12W32-NEXT:    v_and_b32_e32 v1, 0x3ff, v02113; GFX12W32-NEXT:    s_mov_b32 s1, exec_lo2114; GFX12W32-NEXT:    s_mov_b32 s0, 02115; GFX12W32-NEXT:    ; implicit-def: $vgpr02116; GFX12W32-NEXT:  .LBB7_1: ; %ComputeLoop2117; GFX12W32-NEXT:    ; =>This Inner Loop Header: Depth=12118; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)2119; GFX12W32-NEXT:    s_ctz_i32_b32 s2, s12120; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)2121; GFX12W32-NEXT:    v_readlane_b32 s3, v1, s22122; GFX12W32-NEXT:    v_writelane_b32 v0, s0, s22123; GFX12W32-NEXT:    s_lshl_b32 s2, 1, s22124; GFX12W32-NEXT:    s_add_co_i32 s0, s0, s32125; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)2126; GFX12W32-NEXT:    s_and_not1_b32 s1, s1, s22127; GFX12W32-NEXT:    s_cbranch_scc1 .LBB7_12128; GFX12W32-NEXT:  ; %bb.2: ; %ComputeEnd2129; GFX12W32-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 02130; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)2131; GFX12W32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v12132; GFX12W32-NEXT:    ; implicit-def: $vgpr12133; GFX12W32-NEXT:    s_and_saveexec_b32 s1, vcc_lo2134; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)2135; GFX12W32-NEXT:    s_xor_b32 s1, exec_lo, s12136; GFX12W32-NEXT:    s_cbranch_execz .LBB7_42137; GFX12W32-NEXT:  ; %bb.3:2138; GFX12W32-NEXT:    s_load_b128 s[8:11], s[4:5], 0x342139; GFX12W32-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s02140; GFX12W32-NEXT:    s_wait_kmcnt 0x02141; GFX12W32-NEXT:    buffer_atomic_sub_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN2142; GFX12W32-NEXT:  .LBB7_4:2143; GFX12W32-NEXT:    s_wait_alu depctr_sa_sdst(0)2144; GFX12W32-NEXT:    s_or_b32 exec_lo, exec_lo, s12145; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x242146; GFX12W32-NEXT:    s_wait_loadcnt 0x02147; GFX12W32-NEXT:    v_readfirstlane_b32 s2, v12148; GFX12W32-NEXT:    v_mov_b32_e32 v1, 02149; GFX12W32-NEXT:    s_wait_alu depctr_va_sdst(0)2150; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)2151; GFX12W32-NEXT:    v_sub_nc_u32_e32 v0, s2, v02152; GFX12W32-NEXT:    s_wait_kmcnt 0x02153; GFX12W32-NEXT:    global_store_b32 v1, v0, s[0:1]2154; GFX12W32-NEXT:    s_endpgm2155entry:2156  %lane = call i32 @llvm.amdgcn.workitem.id.x()2157  %old = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.sub(i32 %lane, ptr addrspace(8) %inout, i32 0, i32 0, i32 0, i32 0)2158  store i32 %old, ptr addrspace(1) %out2159  ret void2160}2161 2162define amdgpu_kernel void @sub_i32_varying_vindex(ptr addrspace(1) %out, ptr addrspace(8) %inout) {2163; GFX6-LABEL: sub_i32_varying_vindex:2164; GFX6:       ; %bb.0: ; %entry2165; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xd2166; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x92167; GFX6-NEXT:    v_mov_b32_e32 v1, 12168; GFX6-NEXT:    s_waitcnt lgkmcnt(0)2169; GFX6-NEXT:    buffer_atomic_sub v1, v0, s[0:3], 0 idxen glc2170; GFX6-NEXT:    s_mov_b32 s7, 0xf0002171; GFX6-NEXT:    s_mov_b32 s6, -12172; GFX6-NEXT:    s_waitcnt vmcnt(0)2173; GFX6-NEXT:    buffer_store_dword v1, off, s[4:7], 02174; GFX6-NEXT:    s_endpgm2175;2176; GFX8-LABEL: sub_i32_varying_vindex:2177; GFX8:       ; %bb.0: ; %entry2178; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x342179; GFX8-NEXT:    v_mov_b32_e32 v2, 12180; GFX8-NEXT:    s_waitcnt lgkmcnt(0)2181; GFX8-NEXT:    buffer_atomic_sub v2, v0, s[0:3], 0 idxen glc2182; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x242183; GFX8-NEXT:    s_waitcnt lgkmcnt(0)2184; GFX8-NEXT:    v_mov_b32_e32 v0, s02185; GFX8-NEXT:    v_mov_b32_e32 v1, s12186; GFX8-NEXT:    s_waitcnt vmcnt(0)2187; GFX8-NEXT:    flat_store_dword v[0:1], v22188; GFX8-NEXT:    s_endpgm2189;2190; GFX9-LABEL: sub_i32_varying_vindex:2191; GFX9:       ; %bb.0: ; %entry2192; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x342193; GFX9-NEXT:    v_mov_b32_e32 v1, 12194; GFX9-NEXT:    s_waitcnt lgkmcnt(0)2195; GFX9-NEXT:    buffer_atomic_sub v1, v0, s[0:3], 0 idxen glc2196; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x242197; GFX9-NEXT:    v_mov_b32_e32 v0, 02198; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2199; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]2200; GFX9-NEXT:    s_endpgm2201;2202; GFX10-LABEL: sub_i32_varying_vindex:2203; GFX10:       ; %bb.0: ; %entry2204; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x342205; GFX10-NEXT:    v_mov_b32_e32 v1, 12206; GFX10-NEXT:    s_waitcnt lgkmcnt(0)2207; GFX10-NEXT:    buffer_atomic_sub v1, v0, s[0:3], 0 idxen glc2208; GFX10-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)2209; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x242210; GFX10-NEXT:    v_mov_b32_e32 v0, 02211; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2212; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]2213; GFX10-NEXT:    s_endpgm2214;2215; GFX11W64-LABEL: sub_i32_varying_vindex:2216; GFX11W64:       ; %bb.0: ; %entry2217; GFX11W64-NEXT:    s_load_b128 s[0:3], s[4:5], 0x342218; GFX11W64-NEXT:    v_and_b32_e32 v0, 0x3ff, v02219; GFX11W64-NEXT:    v_mov_b32_e32 v1, 12220; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)2221; GFX11W64-NEXT:    buffer_atomic_sub_u32 v1, v0, s[0:3], 0 idxen glc2222; GFX11W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x242223; GFX11W64-NEXT:    v_mov_b32_e32 v0, 02224; GFX11W64-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2225; GFX11W64-NEXT:    global_store_b32 v0, v1, s[0:1]2226; GFX11W64-NEXT:    s_endpgm2227;2228; GFX11W32-LABEL: sub_i32_varying_vindex:2229; GFX11W32:       ; %bb.0: ; %entry2230; GFX11W32-NEXT:    s_load_b128 s[0:3], s[4:5], 0x342231; GFX11W32-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v02232; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)2233; GFX11W32-NEXT:    buffer_atomic_sub_u32 v1, v0, s[0:3], 0 idxen glc2234; GFX11W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x242235; GFX11W32-NEXT:    v_mov_b32_e32 v0, 02236; GFX11W32-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2237; GFX11W32-NEXT:    global_store_b32 v0, v1, s[0:1]2238; GFX11W32-NEXT:    s_endpgm2239;2240; GFX12W64-LABEL: sub_i32_varying_vindex:2241; GFX12W64:       ; %bb.0: ; %entry2242; GFX12W64-NEXT:    s_load_b128 s[0:3], s[4:5], 0x342243; GFX12W64-NEXT:    v_and_b32_e32 v0, 0x3ff, v02244; GFX12W64-NEXT:    v_mov_b32_e32 v1, 12245; GFX12W64-NEXT:    s_wait_kmcnt 0x02246; GFX12W64-NEXT:    buffer_atomic_sub_u32 v1, v0, s[0:3], null idxen th:TH_ATOMIC_RETURN2247; GFX12W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x242248; GFX12W64-NEXT:    v_mov_b32_e32 v0, 02249; GFX12W64-NEXT:    s_wait_loadcnt 0x02250; GFX12W64-NEXT:    s_wait_kmcnt 0x02251; GFX12W64-NEXT:    global_store_b32 v0, v1, s[0:1]2252; GFX12W64-NEXT:    s_endpgm2253;2254; GFX12W32-LABEL: sub_i32_varying_vindex:2255; GFX12W32:       ; %bb.0: ; %entry2256; GFX12W32-NEXT:    s_load_b128 s[0:3], s[4:5], 0x342257; GFX12W32-NEXT:    v_dual_mov_b32 v1, 1 :: v_dual_and_b32 v0, 0x3ff, v02258; GFX12W32-NEXT:    s_wait_kmcnt 0x02259; GFX12W32-NEXT:    buffer_atomic_sub_u32 v1, v0, s[0:3], null idxen th:TH_ATOMIC_RETURN2260; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x242261; GFX12W32-NEXT:    v_mov_b32_e32 v0, 02262; GFX12W32-NEXT:    s_wait_loadcnt 0x02263; GFX12W32-NEXT:    s_wait_kmcnt 0x02264; GFX12W32-NEXT:    global_store_b32 v0, v1, s[0:1]2265; GFX12W32-NEXT:    s_endpgm2266entry:2267  %lane = call i32 @llvm.amdgcn.workitem.id.x()2268  %old = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.sub(i32 1, ptr addrspace(8) %inout, i32 %lane, i32 0, i32 0, i32 0)2269  store i32 %old, ptr addrspace(1) %out2270  ret void2271}2272 2273define amdgpu_kernel void @sub_i32_varying_offset(ptr addrspace(1) %out, ptr addrspace(8) %inout) {2274; GFX6-LABEL: sub_i32_varying_offset:2275; GFX6:       ; %bb.0: ; %entry2276; GFX6-NEXT:    v_mov_b32_e32 v1, v02277; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xd2278; GFX6-NEXT:    s_mov_b32 s6, 02279; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x92280; GFX6-NEXT:    v_mov_b32_e32 v0, s62281; GFX6-NEXT:    v_mov_b32_e32 v2, 12282; GFX6-NEXT:    s_waitcnt lgkmcnt(0)2283; GFX6-NEXT:    buffer_atomic_sub v2, v[0:1], s[0:3], 0 idxen offen glc2284; GFX6-NEXT:    s_mov_b32 s7, 0xf0002285; GFX6-NEXT:    s_mov_b32 s6, -12286; GFX6-NEXT:    s_waitcnt vmcnt(0)2287; GFX6-NEXT:    buffer_store_dword v2, off, s[4:7], 02288; GFX6-NEXT:    s_endpgm2289;2290; GFX8-LABEL: sub_i32_varying_offset:2291; GFX8:       ; %bb.0: ; %entry2292; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x342293; GFX8-NEXT:    s_mov_b32 s6, 02294; GFX8-NEXT:    v_mov_b32_e32 v1, v02295; GFX8-NEXT:    v_mov_b32_e32 v0, s62296; GFX8-NEXT:    v_mov_b32_e32 v2, 12297; GFX8-NEXT:    s_waitcnt lgkmcnt(0)2298; GFX8-NEXT:    buffer_atomic_sub v2, v[0:1], s[0:3], 0 idxen offen glc2299; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x242300; GFX8-NEXT:    s_waitcnt lgkmcnt(0)2301; GFX8-NEXT:    v_mov_b32_e32 v0, s02302; GFX8-NEXT:    v_mov_b32_e32 v1, s12303; GFX8-NEXT:    s_waitcnt vmcnt(0)2304; GFX8-NEXT:    flat_store_dword v[0:1], v22305; GFX8-NEXT:    s_endpgm2306;2307; GFX9-LABEL: sub_i32_varying_offset:2308; GFX9:       ; %bb.0: ; %entry2309; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x342310; GFX9-NEXT:    s_mov_b32 s6, 02311; GFX9-NEXT:    v_mov_b32_e32 v1, v02312; GFX9-NEXT:    v_mov_b32_e32 v0, s62313; GFX9-NEXT:    v_mov_b32_e32 v2, 12314; GFX9-NEXT:    s_waitcnt lgkmcnt(0)2315; GFX9-NEXT:    buffer_atomic_sub v2, v[0:1], s[0:3], 0 idxen offen glc2316; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x242317; GFX9-NEXT:    v_mov_b32_e32 v0, 02318; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2319; GFX9-NEXT:    global_store_dword v0, v2, s[0:1]2320; GFX9-NEXT:    s_endpgm2321;2322; GFX10-LABEL: sub_i32_varying_offset:2323; GFX10:       ; %bb.0: ; %entry2324; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x342325; GFX10-NEXT:    s_mov_b32 s6, 02326; GFX10-NEXT:    v_mov_b32_e32 v1, v02327; GFX10-NEXT:    v_mov_b32_e32 v0, s62328; GFX10-NEXT:    v_mov_b32_e32 v2, 12329; GFX10-NEXT:    s_waitcnt lgkmcnt(0)2330; GFX10-NEXT:    buffer_atomic_sub v2, v[0:1], s[0:3], 0 idxen offen glc2331; GFX10-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)2332; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x242333; GFX10-NEXT:    v_mov_b32_e32 v0, 02334; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2335; GFX10-NEXT:    global_store_dword v0, v2, s[0:1]2336; GFX10-NEXT:    s_endpgm2337;2338; GFX11W64-LABEL: sub_i32_varying_offset:2339; GFX11W64:       ; %bb.0: ; %entry2340; GFX11W64-NEXT:    s_load_b128 s[0:3], s[4:5], 0x342341; GFX11W64-NEXT:    s_mov_b32 s6, 02342; GFX11W64-NEXT:    v_and_b32_e32 v1, 0x3ff, v02343; GFX11W64-NEXT:    v_mov_b32_e32 v0, s62344; GFX11W64-NEXT:    v_mov_b32_e32 v2, 12345; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)2346; GFX11W64-NEXT:    buffer_atomic_sub_u32 v2, v[0:1], s[0:3], 0 idxen offen glc2347; GFX11W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x242348; GFX11W64-NEXT:    v_mov_b32_e32 v0, 02349; GFX11W64-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2350; GFX11W64-NEXT:    global_store_b32 v0, v2, s[0:1]2351; GFX11W64-NEXT:    s_endpgm2352;2353; GFX11W32-LABEL: sub_i32_varying_offset:2354; GFX11W32:       ; %bb.0: ; %entry2355; GFX11W32-NEXT:    s_load_b128 s[0:3], s[4:5], 0x342356; GFX11W32-NEXT:    s_mov_b32 s6, 02357; GFX11W32-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)2358; GFX11W32-NEXT:    v_dual_mov_b32 v0, s6 :: v_dual_and_b32 v1, 0x3ff, v02359; GFX11W32-NEXT:    v_mov_b32_e32 v2, 12360; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)2361; GFX11W32-NEXT:    buffer_atomic_sub_u32 v2, v[0:1], s[0:3], 0 idxen offen glc2362; GFX11W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x242363; GFX11W32-NEXT:    v_mov_b32_e32 v0, 02364; GFX11W32-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2365; GFX11W32-NEXT:    global_store_b32 v0, v2, s[0:1]2366; GFX11W32-NEXT:    s_endpgm2367;2368; GFX12W64-LABEL: sub_i32_varying_offset:2369; GFX12W64:       ; %bb.0: ; %entry2370; GFX12W64-NEXT:    s_load_b128 s[0:3], s[4:5], 0x342371; GFX12W64-NEXT:    v_and_b32_e32 v1, 0x3ff, v02372; GFX12W64-NEXT:    v_mov_b32_e32 v0, 02373; GFX12W64-NEXT:    v_mov_b32_e32 v2, 12374; GFX12W64-NEXT:    s_wait_kmcnt 0x02375; GFX12W64-NEXT:    buffer_atomic_sub_u32 v2, v[0:1], s[0:3], null idxen offen th:TH_ATOMIC_RETURN2376; GFX12W64-NEXT:    s_load_b64 s[0:1], s[4:5], 0x242377; GFX12W64-NEXT:    s_wait_loadcnt 0x02378; GFX12W64-NEXT:    s_wait_kmcnt 0x02379; GFX12W64-NEXT:    global_store_b32 v0, v2, s[0:1]2380; GFX12W64-NEXT:    s_endpgm2381;2382; GFX12W32-LABEL: sub_i32_varying_offset:2383; GFX12W32:       ; %bb.0: ; %entry2384; GFX12W32-NEXT:    s_load_b128 s[0:3], s[4:5], 0x342385; GFX12W32-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_and_b32 v1, 0x3ff, v02386; GFX12W32-NEXT:    v_mov_b32_e32 v2, 12387; GFX12W32-NEXT:    s_wait_kmcnt 0x02388; GFX12W32-NEXT:    buffer_atomic_sub_u32 v2, v[0:1], s[0:3], null idxen offen th:TH_ATOMIC_RETURN2389; GFX12W32-NEXT:    s_load_b64 s[0:1], s[4:5], 0x242390; GFX12W32-NEXT:    s_wait_loadcnt 0x02391; GFX12W32-NEXT:    s_wait_kmcnt 0x02392; GFX12W32-NEXT:    global_store_b32 v0, v2, s[0:1]2393; GFX12W32-NEXT:    s_endpgm2394entry:2395  %lane = call i32 @llvm.amdgcn.workitem.id.x()2396  %old = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.sub(i32 1, ptr addrspace(8) %inout, i32 0, i32 %lane, i32 0, i32 0)2397  store i32 %old, ptr addrspace(1) %out2398  ret void2399}2400;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:2401; GFX11: {{.*}}2402; GFX12: {{.*}}2403