brintos / llvm-project-archived public Read only

1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -enable-var-scope -check-prefixes=SI %s3; RUN: llc -mtriple=amdgcn -mcpu=tonga -amdgpu-atomic-optimizer-strategy=None -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=VI %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s5; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-atomic-optimizer-strategy=None -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-TRUE16 %s6; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-atomic-optimizer-strategy=None -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-FAKE16 %s7 8define amdgpu_kernel void @atomic_add_i32_offset(ptr addrspace(1) %out, i32 %in) {9; SI-LABEL: atomic_add_i32_offset:10; SI:       ; %bb.0: ; %entry11; SI-NEXT:    s_load_dword s6, s[4:5], 0xb12; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x913; SI-NEXT:    s_mov_b32 s3, 0xf00014; SI-NEXT:    s_mov_b32 s2, -115; SI-NEXT:    s_waitcnt lgkmcnt(0)16; SI-NEXT:    v_mov_b32_e32 v0, s617; SI-NEXT:    buffer_atomic_add v0, off, s[0:3], 0 offset:1618; SI-NEXT:    s_waitcnt vmcnt(0)19; SI-NEXT:    buffer_wbinvl120; SI-NEXT:    s_endpgm21;22; VI-LABEL: atomic_add_i32_offset:23; VI:       ; %bb.0: ; %entry24; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c25; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2426; VI-NEXT:    s_mov_b32 s3, 0xf00027; VI-NEXT:    s_mov_b32 s2, -128; VI-NEXT:    s_waitcnt lgkmcnt(0)29; VI-NEXT:    v_mov_b32_e32 v0, s630; VI-NEXT:    buffer_atomic_add v0, off, s[0:3], 0 offset:1631; VI-NEXT:    s_waitcnt vmcnt(0)32; VI-NEXT:    buffer_wbinvl1_vol33; VI-NEXT:    s_endpgm34;35; GFX9-LABEL: atomic_add_i32_offset:36; GFX9:       ; %bb.0: ; %entry37; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c38; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2439; GFX9-NEXT:    v_mov_b32_e32 v0, 040; GFX9-NEXT:    s_waitcnt lgkmcnt(0)41; GFX9-NEXT:    v_mov_b32_e32 v1, s242; GFX9-NEXT:    global_atomic_add v0, v1, s[0:1] offset:1643; GFX9-NEXT:    s_waitcnt vmcnt(0)44; GFX9-NEXT:    buffer_wbinvl1_vol45; GFX9-NEXT:    s_endpgm46;47; GFX11-LABEL: atomic_add_i32_offset:48; GFX11:       ; %bb.0: ; %entry49; GFX11-NEXT:    s_clause 0x150; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c51; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2452; GFX11-NEXT:    s_waitcnt lgkmcnt(0)53; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s254; GFX11-NEXT:    global_atomic_add_u32 v0, v1, s[0:1] offset:1655; GFX11-NEXT:    s_waitcnt_vscnt null, 0x056; GFX11-NEXT:    buffer_gl1_inv57; GFX11-NEXT:    buffer_gl0_inv58; GFX11-NEXT:    s_endpgm59entry:60  %gep = getelementptr i32, ptr addrspace(1) %out, i64 461  %val = atomicrmw volatile add ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst62  ret void63}64 65define amdgpu_kernel void @atomic_add_i32_max_neg_offset(ptr addrspace(1) %out, i32 %in) {66; SI-LABEL: atomic_add_i32_max_neg_offset:67; SI:       ; %bb.0: ; %entry68; SI-NEXT:    s_load_dword s6, s[4:5], 0xb69; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x970; SI-NEXT:    s_mov_b32 s3, 0xf00071; SI-NEXT:    s_mov_b32 s2, 072; SI-NEXT:    v_mov_b32_e32 v0, 0xfffff00073; SI-NEXT:    v_mov_b32_e32 v1, -174; SI-NEXT:    s_waitcnt lgkmcnt(0)75; SI-NEXT:    v_mov_b32_e32 v2, s676; SI-NEXT:    buffer_atomic_add v2, v[0:1], s[0:3], 0 addr6477; SI-NEXT:    s_waitcnt vmcnt(0)78; SI-NEXT:    buffer_wbinvl179; SI-NEXT:    s_endpgm80;81; VI-LABEL: atomic_add_i32_max_neg_offset:82; VI:       ; %bb.0: ; %entry83; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2484; VI-NEXT:    s_load_dword s2, s[4:5], 0x2c85; VI-NEXT:    s_waitcnt lgkmcnt(0)86; VI-NEXT:    s_add_u32 s0, s0, 0xfffff00087; VI-NEXT:    s_addc_u32 s1, s1, -188; VI-NEXT:    v_mov_b32_e32 v0, s089; VI-NEXT:    v_mov_b32_e32 v1, s190; VI-NEXT:    v_mov_b32_e32 v2, s291; VI-NEXT:    flat_atomic_add v[0:1], v292; VI-NEXT:    s_waitcnt vmcnt(0)93; VI-NEXT:    buffer_wbinvl1_vol94; VI-NEXT:    s_endpgm95;96; GFX9-LABEL: atomic_add_i32_max_neg_offset:97; GFX9:       ; %bb.0: ; %entry98; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c99; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24100; GFX9-NEXT:    v_mov_b32_e32 v0, 0101; GFX9-NEXT:    s_waitcnt lgkmcnt(0)102; GFX9-NEXT:    v_mov_b32_e32 v1, s2103; GFX9-NEXT:    global_atomic_add v0, v1, s[0:1] offset:-4096104; GFX9-NEXT:    s_waitcnt vmcnt(0)105; GFX9-NEXT:    buffer_wbinvl1_vol106; GFX9-NEXT:    s_endpgm107;108; GFX11-LABEL: atomic_add_i32_max_neg_offset:109; GFX11:       ; %bb.0: ; %entry110; GFX11-NEXT:    s_clause 0x1111; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c112; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24113; GFX11-NEXT:    s_waitcnt lgkmcnt(0)114; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2115; GFX11-NEXT:    global_atomic_add_u32 v0, v1, s[0:1] offset:-4096116; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0117; GFX11-NEXT:    buffer_gl1_inv118; GFX11-NEXT:    buffer_gl0_inv119; GFX11-NEXT:    s_endpgm120entry:121  %gep = getelementptr i32, ptr addrspace(1) %out, i64 -1024122  %val = atomicrmw volatile add ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst123  ret void124}125 126define amdgpu_kernel void @atomic_add_i32_soffset(ptr addrspace(1) %out, i32 %in) {127; SI-LABEL: atomic_add_i32_soffset:128; SI:       ; %bb.0: ; %entry129; SI-NEXT:    s_load_dword s6, s[4:5], 0xb130; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9131; SI-NEXT:    s_mov_b32 s3, 0xf000132; SI-NEXT:    s_mov_b32 s2, -1133; SI-NEXT:    s_mov_b32 s4, 0x8ca0134; SI-NEXT:    s_waitcnt lgkmcnt(0)135; SI-NEXT:    v_mov_b32_e32 v0, s6136; SI-NEXT:    buffer_atomic_add v0, off, s[0:3], s4137; SI-NEXT:    s_waitcnt vmcnt(0)138; SI-NEXT:    buffer_wbinvl1139; SI-NEXT:    s_endpgm140;141; VI-LABEL: atomic_add_i32_soffset:142; VI:       ; %bb.0: ; %entry143; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c144; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24145; VI-NEXT:    s_mov_b32 s3, 0xf000146; VI-NEXT:    s_mov_b32 s2, -1147; VI-NEXT:    s_mov_b32 s4, 0x8ca0148; VI-NEXT:    s_waitcnt lgkmcnt(0)149; VI-NEXT:    v_mov_b32_e32 v0, s6150; VI-NEXT:    buffer_atomic_add v0, off, s[0:3], s4151; VI-NEXT:    s_waitcnt vmcnt(0)152; VI-NEXT:    buffer_wbinvl1_vol153; VI-NEXT:    s_endpgm154;155; GFX9-LABEL: atomic_add_i32_soffset:156; GFX9:       ; %bb.0: ; %entry157; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c158; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24159; GFX9-NEXT:    v_mov_b32_e32 v0, 0x8000160; GFX9-NEXT:    s_waitcnt lgkmcnt(0)161; GFX9-NEXT:    v_mov_b32_e32 v1, s2162; GFX9-NEXT:    global_atomic_add v0, v1, s[0:1] offset:3232163; GFX9-NEXT:    s_waitcnt vmcnt(0)164; GFX9-NEXT:    buffer_wbinvl1_vol165; GFX9-NEXT:    s_endpgm166;167; GFX11-LABEL: atomic_add_i32_soffset:168; GFX11:       ; %bb.0: ; %entry169; GFX11-NEXT:    s_clause 0x1170; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c171; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24172; GFX11-NEXT:    s_waitcnt lgkmcnt(0)173; GFX11-NEXT:    v_dual_mov_b32 v0, 0x8000 :: v_dual_mov_b32 v1, s2174; GFX11-NEXT:    global_atomic_add_u32 v0, v1, s[0:1] offset:3232175; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0176; GFX11-NEXT:    buffer_gl1_inv177; GFX11-NEXT:    buffer_gl0_inv178; GFX11-NEXT:    s_endpgm179entry:180  %gep = getelementptr i32, ptr addrspace(1) %out, i64 9000181  %val = atomicrmw volatile add ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst182  ret void183}184 185define amdgpu_kernel void @atomic_add_i32_huge_offset(ptr addrspace(1) %out, i32 %in) {186; SI-LABEL: atomic_add_i32_huge_offset:187; SI:       ; %bb.0: ; %entry188; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9189; SI-NEXT:    s_load_dword s4, s[4:5], 0xb190; SI-NEXT:    s_mov_b32 s3, 0xf000191; SI-NEXT:    s_mov_b32 s2, 0192; SI-NEXT:    v_mov_b32_e32 v0, 0xdeac193; SI-NEXT:    v_mov_b32_e32 v1, 0xabcd194; SI-NEXT:    s_waitcnt lgkmcnt(0)195; SI-NEXT:    v_mov_b32_e32 v2, s4196; SI-NEXT:    buffer_atomic_add v2, v[0:1], s[0:3], 0 addr64197; SI-NEXT:    s_waitcnt vmcnt(0)198; SI-NEXT:    buffer_wbinvl1199; SI-NEXT:    s_endpgm200;201; VI-LABEL: atomic_add_i32_huge_offset:202; VI:       ; %bb.0: ; %entry203; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24204; VI-NEXT:    s_load_dword s2, s[4:5], 0x2c205; VI-NEXT:    s_waitcnt lgkmcnt(0)206; VI-NEXT:    s_add_u32 s0, s0, 0xdeac207; VI-NEXT:    s_addc_u32 s1, s1, 0xabcd208; VI-NEXT:    v_mov_b32_e32 v0, s0209; VI-NEXT:    v_mov_b32_e32 v1, s1210; VI-NEXT:    v_mov_b32_e32 v2, s2211; VI-NEXT:    flat_atomic_add v[0:1], v2212; VI-NEXT:    s_waitcnt vmcnt(0)213; VI-NEXT:    buffer_wbinvl1_vol214; VI-NEXT:    s_endpgm215;216; GFX9-LABEL: atomic_add_i32_huge_offset:217; GFX9:       ; %bb.0: ; %entry218; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24219; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c220; GFX9-NEXT:    v_mov_b32_e32 v0, 0221; GFX9-NEXT:    s_waitcnt lgkmcnt(0)222; GFX9-NEXT:    s_add_u32 s0, s0, 0xdeac223; GFX9-NEXT:    s_addc_u32 s1, s1, 0xabcd224; GFX9-NEXT:    v_mov_b32_e32 v1, s2225; GFX9-NEXT:    global_atomic_add v0, v1, s[0:1]226; GFX9-NEXT:    s_waitcnt vmcnt(0)227; GFX9-NEXT:    buffer_wbinvl1_vol228; GFX9-NEXT:    s_endpgm229;230; GFX11-LABEL: atomic_add_i32_huge_offset:231; GFX11:       ; %bb.0: ; %entry232; GFX11-NEXT:    s_clause 0x1233; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c234; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24235; GFX11-NEXT:    s_waitcnt lgkmcnt(0)236; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2237; GFX11-NEXT:    s_add_u32 s0, s0, 0xdeac238; GFX11-NEXT:    s_addc_u32 s1, s1, 0xabcd239; GFX11-NEXT:    global_atomic_add_u32 v0, v1, s[0:1]240; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0241; GFX11-NEXT:    buffer_gl1_inv242; GFX11-NEXT:    buffer_gl0_inv243; GFX11-NEXT:    s_endpgm244entry:245  %gep = getelementptr i32, ptr addrspace(1) %out, i64 47224239175595246 247  %val = atomicrmw volatile add ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst248  ret void249}250 251define amdgpu_kernel void @atomic_add_i32_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) {252; SI-LABEL: atomic_add_i32_ret_offset:253; SI:       ; %bb.0: ; %entry254; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9255; SI-NEXT:    s_load_dword s8, s[4:5], 0xd256; SI-NEXT:    s_mov_b32 s7, 0xf000257; SI-NEXT:    s_mov_b32 s6, -1258; SI-NEXT:    s_waitcnt lgkmcnt(0)259; SI-NEXT:    s_mov_b32 s4, s2260; SI-NEXT:    s_mov_b32 s5, s3261; SI-NEXT:    s_mov_b32 s2, s6262; SI-NEXT:    s_mov_b32 s3, s7263; SI-NEXT:    v_mov_b32_e32 v0, s8264; SI-NEXT:    buffer_atomic_add v0, off, s[0:3], 0 offset:16 glc265; SI-NEXT:    s_waitcnt vmcnt(0)266; SI-NEXT:    buffer_wbinvl1267; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0268; SI-NEXT:    s_endpgm269;270; VI-LABEL: atomic_add_i32_ret_offset:271; VI:       ; %bb.0: ; %entry272; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24273; VI-NEXT:    s_load_dword s8, s[4:5], 0x34274; VI-NEXT:    s_mov_b32 s7, 0xf000275; VI-NEXT:    s_mov_b32 s6, -1276; VI-NEXT:    s_waitcnt lgkmcnt(0)277; VI-NEXT:    s_mov_b32 s4, s2278; VI-NEXT:    s_mov_b32 s5, s3279; VI-NEXT:    s_mov_b32 s2, s6280; VI-NEXT:    s_mov_b32 s3, s7281; VI-NEXT:    v_mov_b32_e32 v0, s8282; VI-NEXT:    buffer_atomic_add v0, off, s[0:3], 0 offset:16 glc283; VI-NEXT:    s_waitcnt vmcnt(0)284; VI-NEXT:    buffer_wbinvl1_vol285; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0286; VI-NEXT:    s_endpgm287;288; GFX9-LABEL: atomic_add_i32_ret_offset:289; GFX9:       ; %bb.0: ; %entry290; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x34291; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24292; GFX9-NEXT:    v_mov_b32_e32 v0, 0293; GFX9-NEXT:    s_waitcnt lgkmcnt(0)294; GFX9-NEXT:    v_mov_b32_e32 v1, s6295; GFX9-NEXT:    global_atomic_add v1, v0, v1, s[0:1] offset:16 glc296; GFX9-NEXT:    s_waitcnt vmcnt(0)297; GFX9-NEXT:    buffer_wbinvl1_vol298; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]299; GFX9-NEXT:    s_endpgm300;301; GFX11-LABEL: atomic_add_i32_ret_offset:302; GFX11:       ; %bb.0: ; %entry303; GFX11-NEXT:    s_clause 0x1304; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34305; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24306; GFX11-NEXT:    s_waitcnt lgkmcnt(0)307; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6308; GFX11-NEXT:    global_atomic_add_u32 v1, v0, v1, s[0:1] offset:16 glc309; GFX11-NEXT:    s_waitcnt vmcnt(0)310; GFX11-NEXT:    buffer_gl1_inv311; GFX11-NEXT:    buffer_gl0_inv312; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]313; GFX11-NEXT:    s_endpgm314entry:315  %gep = getelementptr i32, ptr addrspace(1) %out, i64 4316  %val = atomicrmw volatile add ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst317  store i32 %val, ptr addrspace(1) %out2318  ret void319}320 321define amdgpu_kernel void @atomic_add_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {322; SI-LABEL: atomic_add_i32_addr64_offset:323; SI:       ; %bb.0: ; %entry324; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd325; SI-NEXT:    s_load_dword s8, s[4:5], 0xb326; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9327; SI-NEXT:    s_mov_b32 s3, 0xf000328; SI-NEXT:    s_waitcnt lgkmcnt(0)329; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2330; SI-NEXT:    s_mov_b32 s2, 0331; SI-NEXT:    v_mov_b32_e32 v2, s8332; SI-NEXT:    v_mov_b32_e32 v0, s4333; SI-NEXT:    v_mov_b32_e32 v1, s5334; SI-NEXT:    buffer_atomic_add v2, v[0:1], s[0:3], 0 addr64 offset:16335; SI-NEXT:    s_waitcnt vmcnt(0)336; SI-NEXT:    buffer_wbinvl1337; SI-NEXT:    s_endpgm338;339; VI-LABEL: atomic_add_i32_addr64_offset:340; VI:       ; %bb.0: ; %entry341; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34342; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24343; VI-NEXT:    s_load_dword s4, s[4:5], 0x2c344; VI-NEXT:    s_waitcnt lgkmcnt(0)345; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2346; VI-NEXT:    s_add_u32 s0, s2, s0347; VI-NEXT:    s_addc_u32 s1, s3, s1348; VI-NEXT:    s_add_u32 s0, s0, 16349; VI-NEXT:    s_addc_u32 s1, s1, 0350; VI-NEXT:    v_mov_b32_e32 v0, s0351; VI-NEXT:    v_mov_b32_e32 v1, s1352; VI-NEXT:    v_mov_b32_e32 v2, s4353; VI-NEXT:    flat_atomic_add v[0:1], v2354; VI-NEXT:    s_waitcnt vmcnt(0)355; VI-NEXT:    buffer_wbinvl1_vol356; VI-NEXT:    s_endpgm357;358; GFX9-LABEL: atomic_add_i32_addr64_offset:359; GFX9:       ; %bb.0: ; %entry360; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34361; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24362; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c363; GFX9-NEXT:    v_mov_b32_e32 v0, 0364; GFX9-NEXT:    s_waitcnt lgkmcnt(0)365; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2366; GFX9-NEXT:    s_add_u32 s0, s2, s0367; GFX9-NEXT:    s_addc_u32 s1, s3, s1368; GFX9-NEXT:    v_mov_b32_e32 v1, s6369; GFX9-NEXT:    global_atomic_add v0, v1, s[0:1] offset:16370; GFX9-NEXT:    s_waitcnt vmcnt(0)371; GFX9-NEXT:    buffer_wbinvl1_vol372; GFX9-NEXT:    s_endpgm373;374; GFX11-LABEL: atomic_add_i32_addr64_offset:375; GFX11:       ; %bb.0: ; %entry376; GFX11-NEXT:    s_clause 0x2377; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34378; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c379; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24380; GFX11-NEXT:    v_mov_b32_e32 v0, 0381; GFX11-NEXT:    s_waitcnt lgkmcnt(0)382; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2383; GFX11-NEXT:    v_mov_b32_e32 v1, s6384; GFX11-NEXT:    s_add_u32 s0, s2, s0385; GFX11-NEXT:    s_addc_u32 s1, s3, s1386; GFX11-NEXT:    global_atomic_add_u32 v0, v1, s[0:1] offset:16387; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0388; GFX11-NEXT:    buffer_gl1_inv389; GFX11-NEXT:    buffer_gl0_inv390; GFX11-NEXT:    s_endpgm391entry:392  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index393  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4394  %val = atomicrmw volatile add ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst395  ret void396}397 398define amdgpu_kernel void @atomic_add_i32_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) {399; SI-LABEL: atomic_add_i32_ret_addr64_offset:400; SI:       ; %bb.0: ; %entry401; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9402; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf403; SI-NEXT:    s_load_dword s6, s[4:5], 0xd404; SI-NEXT:    s_mov_b32 s7, 0xf000405; SI-NEXT:    s_waitcnt lgkmcnt(0)406; SI-NEXT:    s_mov_b32 s4, s2407; SI-NEXT:    s_mov_b32 s5, s3408; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 2409; SI-NEXT:    s_mov_b32 s2, 0410; SI-NEXT:    s_mov_b32 s3, s7411; SI-NEXT:    v_mov_b32_e32 v2, s6412; SI-NEXT:    v_mov_b32_e32 v0, s8413; SI-NEXT:    v_mov_b32_e32 v1, s9414; SI-NEXT:    buffer_atomic_add v2, v[0:1], s[0:3], 0 addr64 offset:16 glc415; SI-NEXT:    s_waitcnt vmcnt(0)416; SI-NEXT:    buffer_wbinvl1417; SI-NEXT:    s_mov_b32 s6, -1418; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 0419; SI-NEXT:    s_endpgm420;421; VI-LABEL: atomic_add_i32_ret_addr64_offset:422; VI:       ; %bb.0: ; %entry423; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c424; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24425; VI-NEXT:    s_load_dword s8, s[4:5], 0x34426; VI-NEXT:    s_waitcnt lgkmcnt(0)427; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2428; VI-NEXT:    s_add_u32 s0, s0, s4429; VI-NEXT:    s_addc_u32 s1, s1, s5430; VI-NEXT:    s_add_u32 s0, s0, 16431; VI-NEXT:    s_addc_u32 s1, s1, 0432; VI-NEXT:    v_mov_b32_e32 v0, s0433; VI-NEXT:    v_mov_b32_e32 v1, s1434; VI-NEXT:    v_mov_b32_e32 v2, s8435; VI-NEXT:    flat_atomic_add v0, v[0:1], v2 glc436; VI-NEXT:    s_waitcnt vmcnt(0)437; VI-NEXT:    buffer_wbinvl1_vol438; VI-NEXT:    s_mov_b32 s7, 0xf000439; VI-NEXT:    s_mov_b32 s6, -1440; VI-NEXT:    s_mov_b32 s4, s2441; VI-NEXT:    s_mov_b32 s5, s3442; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0443; VI-NEXT:    s_endpgm444;445; GFX9-LABEL: atomic_add_i32_ret_addr64_offset:446; GFX9:       ; %bb.0: ; %entry447; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c448; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24449; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x34450; GFX9-NEXT:    v_mov_b32_e32 v0, 0451; GFX9-NEXT:    s_waitcnt lgkmcnt(0)452; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2453; GFX9-NEXT:    s_add_u32 s0, s0, s4454; GFX9-NEXT:    s_addc_u32 s1, s1, s5455; GFX9-NEXT:    v_mov_b32_e32 v1, s8456; GFX9-NEXT:    global_atomic_add v1, v0, v1, s[0:1] offset:16 glc457; GFX9-NEXT:    s_waitcnt vmcnt(0)458; GFX9-NEXT:    buffer_wbinvl1_vol459; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]460; GFX9-NEXT:    s_endpgm461;462; GFX11-LABEL: atomic_add_i32_ret_addr64_offset:463; GFX11:       ; %bb.0: ; %entry464; GFX11-NEXT:    s_clause 0x2465; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c466; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34467; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24468; GFX11-NEXT:    v_mov_b32_e32 v0, 0469; GFX11-NEXT:    s_waitcnt lgkmcnt(0)470; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2471; GFX11-NEXT:    v_mov_b32_e32 v1, s8472; GFX11-NEXT:    s_add_u32 s0, s0, s4473; GFX11-NEXT:    s_addc_u32 s1, s1, s5474; GFX11-NEXT:    global_atomic_add_u32 v1, v0, v1, s[0:1] offset:16 glc475; GFX11-NEXT:    s_waitcnt vmcnt(0)476; GFX11-NEXT:    buffer_gl1_inv477; GFX11-NEXT:    buffer_gl0_inv478; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]479; GFX11-NEXT:    s_endpgm480entry:481  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index482  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4483  %val = atomicrmw volatile add ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst484  store i32 %val, ptr addrspace(1) %out2485  ret void486}487 488define amdgpu_kernel void @atomic_add_i32(ptr addrspace(1) %out, i32 %in) {489; SI-LABEL: atomic_add_i32:490; SI:       ; %bb.0: ; %entry491; SI-NEXT:    s_load_dword s6, s[4:5], 0xb492; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9493; SI-NEXT:    s_mov_b32 s3, 0xf000494; SI-NEXT:    s_mov_b32 s2, -1495; SI-NEXT:    s_waitcnt lgkmcnt(0)496; SI-NEXT:    v_mov_b32_e32 v0, s6497; SI-NEXT:    buffer_atomic_add v0, off, s[0:3], 0498; SI-NEXT:    s_waitcnt vmcnt(0)499; SI-NEXT:    buffer_wbinvl1500; SI-NEXT:    s_endpgm501;502; VI-LABEL: atomic_add_i32:503; VI:       ; %bb.0: ; %entry504; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c505; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24506; VI-NEXT:    s_mov_b32 s3, 0xf000507; VI-NEXT:    s_mov_b32 s2, -1508; VI-NEXT:    s_waitcnt lgkmcnt(0)509; VI-NEXT:    v_mov_b32_e32 v0, s6510; VI-NEXT:    buffer_atomic_add v0, off, s[0:3], 0511; VI-NEXT:    s_waitcnt vmcnt(0)512; VI-NEXT:    buffer_wbinvl1_vol513; VI-NEXT:    s_endpgm514;515; GFX9-LABEL: atomic_add_i32:516; GFX9:       ; %bb.0: ; %entry517; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c518; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24519; GFX9-NEXT:    v_mov_b32_e32 v0, 0520; GFX9-NEXT:    s_waitcnt lgkmcnt(0)521; GFX9-NEXT:    v_mov_b32_e32 v1, s2522; GFX9-NEXT:    global_atomic_add v0, v1, s[0:1]523; GFX9-NEXT:    s_waitcnt vmcnt(0)524; GFX9-NEXT:    buffer_wbinvl1_vol525; GFX9-NEXT:    s_endpgm526;527; GFX11-LABEL: atomic_add_i32:528; GFX11:       ; %bb.0: ; %entry529; GFX11-NEXT:    s_clause 0x1530; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c531; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24532; GFX11-NEXT:    s_waitcnt lgkmcnt(0)533; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2534; GFX11-NEXT:    global_atomic_add_u32 v0, v1, s[0:1]535; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0536; GFX11-NEXT:    buffer_gl1_inv537; GFX11-NEXT:    buffer_gl0_inv538; GFX11-NEXT:    s_endpgm539entry:540  %val = atomicrmw volatile add ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst541  ret void542}543 544define amdgpu_kernel void @atomic_add_i32_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) {545; SI-LABEL: atomic_add_i32_ret:546; SI:       ; %bb.0: ; %entry547; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9548; SI-NEXT:    s_load_dword s8, s[4:5], 0xd549; SI-NEXT:    s_mov_b32 s7, 0xf000550; SI-NEXT:    s_mov_b32 s6, -1551; SI-NEXT:    s_waitcnt lgkmcnt(0)552; SI-NEXT:    s_mov_b32 s4, s0553; SI-NEXT:    s_mov_b32 s5, s1554; SI-NEXT:    v_mov_b32_e32 v0, s8555; SI-NEXT:    buffer_atomic_add v0, off, s[4:7], 0 glc556; SI-NEXT:    s_waitcnt vmcnt(0)557; SI-NEXT:    buffer_wbinvl1558; SI-NEXT:    s_mov_b32 s4, s2559; SI-NEXT:    s_mov_b32 s5, s3560; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0561; SI-NEXT:    s_endpgm562;563; VI-LABEL: atomic_add_i32_ret:564; VI:       ; %bb.0: ; %entry565; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24566; VI-NEXT:    s_load_dword s8, s[4:5], 0x34567; VI-NEXT:    s_mov_b32 s7, 0xf000568; VI-NEXT:    s_mov_b32 s6, -1569; VI-NEXT:    s_waitcnt lgkmcnt(0)570; VI-NEXT:    s_mov_b32 s4, s0571; VI-NEXT:    s_mov_b32 s5, s1572; VI-NEXT:    v_mov_b32_e32 v0, s8573; VI-NEXT:    buffer_atomic_add v0, off, s[4:7], 0 glc574; VI-NEXT:    s_waitcnt vmcnt(0)575; VI-NEXT:    buffer_wbinvl1_vol576; VI-NEXT:    s_mov_b32 s4, s2577; VI-NEXT:    s_mov_b32 s5, s3578; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0579; VI-NEXT:    s_endpgm580;581; GFX9-LABEL: atomic_add_i32_ret:582; GFX9:       ; %bb.0: ; %entry583; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x34584; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24585; GFX9-NEXT:    v_mov_b32_e32 v0, 0586; GFX9-NEXT:    s_waitcnt lgkmcnt(0)587; GFX9-NEXT:    v_mov_b32_e32 v1, s6588; GFX9-NEXT:    global_atomic_add v1, v0, v1, s[0:1] glc589; GFX9-NEXT:    s_waitcnt vmcnt(0)590; GFX9-NEXT:    buffer_wbinvl1_vol591; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]592; GFX9-NEXT:    s_endpgm593;594; GFX11-LABEL: atomic_add_i32_ret:595; GFX11:       ; %bb.0: ; %entry596; GFX11-NEXT:    s_clause 0x1597; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34598; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24599; GFX11-NEXT:    s_waitcnt lgkmcnt(0)600; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6601; GFX11-NEXT:    global_atomic_add_u32 v1, v0, v1, s[0:1] glc602; GFX11-NEXT:    s_waitcnt vmcnt(0)603; GFX11-NEXT:    buffer_gl1_inv604; GFX11-NEXT:    buffer_gl0_inv605; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]606; GFX11-NEXT:    s_endpgm607entry:608  %val = atomicrmw volatile add ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst609  store i32 %val, ptr addrspace(1) %out2610  ret void611}612 613define amdgpu_kernel void @atomic_add_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) {614; SI-LABEL: atomic_add_i32_addr64:615; SI:       ; %bb.0: ; %entry616; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd617; SI-NEXT:    s_load_dword s8, s[4:5], 0xb618; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9619; SI-NEXT:    s_mov_b32 s3, 0xf000620; SI-NEXT:    s_waitcnt lgkmcnt(0)621; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2622; SI-NEXT:    s_mov_b32 s2, 0623; SI-NEXT:    v_mov_b32_e32 v2, s8624; SI-NEXT:    v_mov_b32_e32 v0, s4625; SI-NEXT:    v_mov_b32_e32 v1, s5626; SI-NEXT:    buffer_atomic_add v2, v[0:1], s[0:3], 0 addr64627; SI-NEXT:    s_waitcnt vmcnt(0)628; SI-NEXT:    buffer_wbinvl1629; SI-NEXT:    s_endpgm630;631; VI-LABEL: atomic_add_i32_addr64:632; VI:       ; %bb.0: ; %entry633; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34634; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24635; VI-NEXT:    s_load_dword s4, s[4:5], 0x2c636; VI-NEXT:    s_waitcnt lgkmcnt(0)637; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2638; VI-NEXT:    s_add_u32 s0, s2, s0639; VI-NEXT:    s_addc_u32 s1, s3, s1640; VI-NEXT:    v_mov_b32_e32 v0, s0641; VI-NEXT:    v_mov_b32_e32 v1, s1642; VI-NEXT:    v_mov_b32_e32 v2, s4643; VI-NEXT:    flat_atomic_add v[0:1], v2644; VI-NEXT:    s_waitcnt vmcnt(0)645; VI-NEXT:    buffer_wbinvl1_vol646; VI-NEXT:    s_endpgm647;648; GFX9-LABEL: atomic_add_i32_addr64:649; GFX9:       ; %bb.0: ; %entry650; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34651; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24652; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c653; GFX9-NEXT:    v_mov_b32_e32 v0, 0654; GFX9-NEXT:    s_waitcnt lgkmcnt(0)655; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2656; GFX9-NEXT:    s_add_u32 s0, s2, s0657; GFX9-NEXT:    s_addc_u32 s1, s3, s1658; GFX9-NEXT:    v_mov_b32_e32 v1, s6659; GFX9-NEXT:    global_atomic_add v0, v1, s[0:1]660; GFX9-NEXT:    s_waitcnt vmcnt(0)661; GFX9-NEXT:    buffer_wbinvl1_vol662; GFX9-NEXT:    s_endpgm663;664; GFX11-LABEL: atomic_add_i32_addr64:665; GFX11:       ; %bb.0: ; %entry666; GFX11-NEXT:    s_clause 0x2667; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34668; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c669; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24670; GFX11-NEXT:    v_mov_b32_e32 v0, 0671; GFX11-NEXT:    s_waitcnt lgkmcnt(0)672; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2673; GFX11-NEXT:    v_mov_b32_e32 v1, s6674; GFX11-NEXT:    s_add_u32 s0, s2, s0675; GFX11-NEXT:    s_addc_u32 s1, s3, s1676; GFX11-NEXT:    global_atomic_add_u32 v0, v1, s[0:1]677; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0678; GFX11-NEXT:    buffer_gl1_inv679; GFX11-NEXT:    buffer_gl0_inv680; GFX11-NEXT:    s_endpgm681entry:682  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index683  %val = atomicrmw volatile add ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst684  ret void685}686 687define amdgpu_kernel void @atomic_add_i32_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) {688; SI-LABEL: atomic_add_i32_ret_addr64:689; SI:       ; %bb.0: ; %entry690; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9691; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf692; SI-NEXT:    s_load_dword s6, s[4:5], 0xd693; SI-NEXT:    s_mov_b32 s7, 0xf000694; SI-NEXT:    s_waitcnt lgkmcnt(0)695; SI-NEXT:    s_mov_b32 s4, s2696; SI-NEXT:    s_mov_b32 s5, s3697; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 2698; SI-NEXT:    s_mov_b32 s2, 0699; SI-NEXT:    s_mov_b32 s3, s7700; SI-NEXT:    v_mov_b32_e32 v2, s6701; SI-NEXT:    v_mov_b32_e32 v0, s8702; SI-NEXT:    v_mov_b32_e32 v1, s9703; SI-NEXT:    buffer_atomic_add v2, v[0:1], s[0:3], 0 addr64 glc704; SI-NEXT:    s_waitcnt vmcnt(0)705; SI-NEXT:    buffer_wbinvl1706; SI-NEXT:    s_mov_b32 s6, -1707; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 0708; SI-NEXT:    s_endpgm709;710; VI-LABEL: atomic_add_i32_ret_addr64:711; VI:       ; %bb.0: ; %entry712; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c713; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24714; VI-NEXT:    s_load_dword s8, s[4:5], 0x34715; VI-NEXT:    s_waitcnt lgkmcnt(0)716; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2717; VI-NEXT:    s_add_u32 s0, s0, s4718; VI-NEXT:    s_addc_u32 s1, s1, s5719; VI-NEXT:    v_mov_b32_e32 v0, s0720; VI-NEXT:    v_mov_b32_e32 v1, s1721; VI-NEXT:    v_mov_b32_e32 v2, s8722; VI-NEXT:    flat_atomic_add v0, v[0:1], v2 glc723; VI-NEXT:    s_waitcnt vmcnt(0)724; VI-NEXT:    buffer_wbinvl1_vol725; VI-NEXT:    s_mov_b32 s7, 0xf000726; VI-NEXT:    s_mov_b32 s6, -1727; VI-NEXT:    s_mov_b32 s4, s2728; VI-NEXT:    s_mov_b32 s5, s3729; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0730; VI-NEXT:    s_endpgm731;732; GFX9-LABEL: atomic_add_i32_ret_addr64:733; GFX9:       ; %bb.0: ; %entry734; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c735; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24736; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x34737; GFX9-NEXT:    v_mov_b32_e32 v0, 0738; GFX9-NEXT:    s_waitcnt lgkmcnt(0)739; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2740; GFX9-NEXT:    s_add_u32 s0, s0, s4741; GFX9-NEXT:    s_addc_u32 s1, s1, s5742; GFX9-NEXT:    v_mov_b32_e32 v1, s8743; GFX9-NEXT:    global_atomic_add v1, v0, v1, s[0:1] glc744; GFX9-NEXT:    s_waitcnt vmcnt(0)745; GFX9-NEXT:    buffer_wbinvl1_vol746; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]747; GFX9-NEXT:    s_endpgm748;749; GFX11-LABEL: atomic_add_i32_ret_addr64:750; GFX11:       ; %bb.0: ; %entry751; GFX11-NEXT:    s_clause 0x2752; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c753; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x34754; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24755; GFX11-NEXT:    v_mov_b32_e32 v0, 0756; GFX11-NEXT:    s_waitcnt lgkmcnt(0)757; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2758; GFX11-NEXT:    v_mov_b32_e32 v1, s8759; GFX11-NEXT:    s_add_u32 s0, s0, s4760; GFX11-NEXT:    s_addc_u32 s1, s1, s5761; GFX11-NEXT:    global_atomic_add_u32 v1, v0, v1, s[0:1] glc762; GFX11-NEXT:    s_waitcnt vmcnt(0)763; GFX11-NEXT:    buffer_gl1_inv764; GFX11-NEXT:    buffer_gl0_inv765; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]766; GFX11-NEXT:    s_endpgm767entry:768  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index769  %val = atomicrmw volatile add ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst770  store i32 %val, ptr addrspace(1) %out2771  ret void772}773 774define amdgpu_kernel void @atomic_and_i32_offset(ptr addrspace(1) %out, i32 %in) {775; SI-LABEL: atomic_and_i32_offset:776; SI:       ; %bb.0: ; %entry777; SI-NEXT:    s_load_dword s6, s[4:5], 0xb778; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9779; SI-NEXT:    s_mov_b32 s3, 0xf000780; SI-NEXT:    s_mov_b32 s2, -1781; SI-NEXT:    s_waitcnt lgkmcnt(0)782; SI-NEXT:    v_mov_b32_e32 v0, s6783; SI-NEXT:    buffer_atomic_and v0, off, s[0:3], 0 offset:16784; SI-NEXT:    s_waitcnt vmcnt(0)785; SI-NEXT:    buffer_wbinvl1786; SI-NEXT:    s_endpgm787;788; VI-LABEL: atomic_and_i32_offset:789; VI:       ; %bb.0: ; %entry790; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c791; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24792; VI-NEXT:    s_mov_b32 s3, 0xf000793; VI-NEXT:    s_mov_b32 s2, -1794; VI-NEXT:    s_waitcnt lgkmcnt(0)795; VI-NEXT:    v_mov_b32_e32 v0, s6796; VI-NEXT:    buffer_atomic_and v0, off, s[0:3], 0 offset:16797; VI-NEXT:    s_waitcnt vmcnt(0)798; VI-NEXT:    buffer_wbinvl1_vol799; VI-NEXT:    s_endpgm800;801; GFX9-LABEL: atomic_and_i32_offset:802; GFX9:       ; %bb.0: ; %entry803; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c804; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24805; GFX9-NEXT:    v_mov_b32_e32 v0, 0806; GFX9-NEXT:    s_waitcnt lgkmcnt(0)807; GFX9-NEXT:    v_mov_b32_e32 v1, s2808; GFX9-NEXT:    global_atomic_and v0, v1, s[0:1] offset:16809; GFX9-NEXT:    s_waitcnt vmcnt(0)810; GFX9-NEXT:    buffer_wbinvl1_vol811; GFX9-NEXT:    s_endpgm812;813; GFX11-LABEL: atomic_and_i32_offset:814; GFX11:       ; %bb.0: ; %entry815; GFX11-NEXT:    s_clause 0x1816; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c817; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24818; GFX11-NEXT:    s_waitcnt lgkmcnt(0)819; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2820; GFX11-NEXT:    global_atomic_and_b32 v0, v1, s[0:1] offset:16821; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0822; GFX11-NEXT:    buffer_gl1_inv823; GFX11-NEXT:    buffer_gl0_inv824; GFX11-NEXT:    s_endpgm825entry:826  %gep = getelementptr i32, ptr addrspace(1) %out, i64 4827  %val = atomicrmw volatile and ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0828  ret void829}830 831define amdgpu_kernel void @atomic_and_i32_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) {832; SI-LABEL: atomic_and_i32_ret_offset:833; SI:       ; %bb.0: ; %entry834; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9835; SI-NEXT:    s_load_dword s8, s[4:5], 0xd836; SI-NEXT:    s_mov_b32 s7, 0xf000837; SI-NEXT:    s_mov_b32 s6, -1838; SI-NEXT:    s_waitcnt lgkmcnt(0)839; SI-NEXT:    s_mov_b32 s4, s2840; SI-NEXT:    s_mov_b32 s5, s3841; SI-NEXT:    s_mov_b32 s2, s6842; SI-NEXT:    s_mov_b32 s3, s7843; SI-NEXT:    v_mov_b32_e32 v0, s8844; SI-NEXT:    buffer_atomic_and v0, off, s[0:3], 0 offset:16 glc845; SI-NEXT:    s_waitcnt vmcnt(0)846; SI-NEXT:    buffer_wbinvl1847; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0848; SI-NEXT:    s_endpgm849;850; VI-LABEL: atomic_and_i32_ret_offset:851; VI:       ; %bb.0: ; %entry852; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24853; VI-NEXT:    s_load_dword s8, s[4:5], 0x34854; VI-NEXT:    s_mov_b32 s7, 0xf000855; VI-NEXT:    s_mov_b32 s6, -1856; VI-NEXT:    s_waitcnt lgkmcnt(0)857; VI-NEXT:    s_mov_b32 s4, s2858; VI-NEXT:    s_mov_b32 s5, s3859; VI-NEXT:    s_mov_b32 s2, s6860; VI-NEXT:    s_mov_b32 s3, s7861; VI-NEXT:    v_mov_b32_e32 v0, s8862; VI-NEXT:    buffer_atomic_and v0, off, s[0:3], 0 offset:16 glc863; VI-NEXT:    s_waitcnt vmcnt(0)864; VI-NEXT:    buffer_wbinvl1_vol865; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0866; VI-NEXT:    s_endpgm867;868; GFX9-LABEL: atomic_and_i32_ret_offset:869; GFX9:       ; %bb.0: ; %entry870; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x34871; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24872; GFX9-NEXT:    v_mov_b32_e32 v0, 0873; GFX9-NEXT:    s_waitcnt lgkmcnt(0)874; GFX9-NEXT:    v_mov_b32_e32 v1, s6875; GFX9-NEXT:    global_atomic_and v1, v0, v1, s[0:1] offset:16 glc876; GFX9-NEXT:    s_waitcnt vmcnt(0)877; GFX9-NEXT:    buffer_wbinvl1_vol878; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]879; GFX9-NEXT:    s_endpgm880;881; GFX11-LABEL: atomic_and_i32_ret_offset:882; GFX11:       ; %bb.0: ; %entry883; GFX11-NEXT:    s_clause 0x1884; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34885; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24886; GFX11-NEXT:    s_waitcnt lgkmcnt(0)887; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6888; GFX11-NEXT:    global_atomic_and_b32 v1, v0, v1, s[0:1] offset:16 glc889; GFX11-NEXT:    s_waitcnt vmcnt(0)890; GFX11-NEXT:    buffer_gl1_inv891; GFX11-NEXT:    buffer_gl0_inv892; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]893; GFX11-NEXT:    s_endpgm894entry:895  %gep = getelementptr i32, ptr addrspace(1) %out, i64 4896  %val = atomicrmw volatile and ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0897  store i32 %val, ptr addrspace(1) %out2898  ret void899}900 901define amdgpu_kernel void @atomic_and_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {902; SI-LABEL: atomic_and_i32_addr64_offset:903; SI:       ; %bb.0: ; %entry904; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd905; SI-NEXT:    s_load_dword s8, s[4:5], 0xb906; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9907; SI-NEXT:    s_mov_b32 s3, 0xf000908; SI-NEXT:    s_waitcnt lgkmcnt(0)909; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 2910; SI-NEXT:    s_mov_b32 s2, 0911; SI-NEXT:    v_mov_b32_e32 v2, s8912; SI-NEXT:    v_mov_b32_e32 v0, s4913; SI-NEXT:    v_mov_b32_e32 v1, s5914; SI-NEXT:    buffer_atomic_and v2, v[0:1], s[0:3], 0 addr64 offset:16915; SI-NEXT:    s_waitcnt vmcnt(0)916; SI-NEXT:    buffer_wbinvl1917; SI-NEXT:    s_endpgm918;919; VI-LABEL: atomic_and_i32_addr64_offset:920; VI:       ; %bb.0: ; %entry921; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34922; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24923; VI-NEXT:    s_load_dword s4, s[4:5], 0x2c924; VI-NEXT:    s_waitcnt lgkmcnt(0)925; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2926; VI-NEXT:    s_add_u32 s0, s2, s0927; VI-NEXT:    s_addc_u32 s1, s3, s1928; VI-NEXT:    s_add_u32 s0, s0, 16929; VI-NEXT:    s_addc_u32 s1, s1, 0930; VI-NEXT:    v_mov_b32_e32 v0, s0931; VI-NEXT:    v_mov_b32_e32 v1, s1932; VI-NEXT:    v_mov_b32_e32 v2, s4933; VI-NEXT:    flat_atomic_and v[0:1], v2934; VI-NEXT:    s_waitcnt vmcnt(0)935; VI-NEXT:    buffer_wbinvl1_vol936; VI-NEXT:    s_endpgm937;938; GFX9-LABEL: atomic_and_i32_addr64_offset:939; GFX9:       ; %bb.0: ; %entry940; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34941; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24942; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c943; GFX9-NEXT:    v_mov_b32_e32 v0, 0944; GFX9-NEXT:    s_waitcnt lgkmcnt(0)945; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2946; GFX9-NEXT:    s_add_u32 s0, s2, s0947; GFX9-NEXT:    s_addc_u32 s1, s3, s1948; GFX9-NEXT:    v_mov_b32_e32 v1, s6949; GFX9-NEXT:    global_atomic_and v0, v1, s[0:1] offset:16950; GFX9-NEXT:    s_waitcnt vmcnt(0)951; GFX9-NEXT:    buffer_wbinvl1_vol952; GFX9-NEXT:    s_endpgm953;954; GFX11-LABEL: atomic_and_i32_addr64_offset:955; GFX11:       ; %bb.0: ; %entry956; GFX11-NEXT:    s_clause 0x2957; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34958; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c959; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x24960; GFX11-NEXT:    v_mov_b32_e32 v0, 0961; GFX11-NEXT:    s_waitcnt lgkmcnt(0)962; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2963; GFX11-NEXT:    v_mov_b32_e32 v1, s6964; GFX11-NEXT:    s_add_u32 s0, s2, s0965; GFX11-NEXT:    s_addc_u32 s1, s3, s1966; GFX11-NEXT:    global_atomic_and_b32 v0, v1, s[0:1] offset:16967; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0968; GFX11-NEXT:    buffer_gl1_inv969; GFX11-NEXT:    buffer_gl0_inv970; GFX11-NEXT:    s_endpgm971entry:972  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index973  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4974  %val = atomicrmw volatile and ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0975  ret void976}977 978define amdgpu_kernel void @atomic_and_i32_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) {979; SI-LABEL: atomic_and_i32_ret_addr64_offset:980; SI:       ; %bb.0: ; %entry981; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9982; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf983; SI-NEXT:    s_load_dword s6, s[4:5], 0xd984; SI-NEXT:    s_mov_b32 s7, 0xf000985; SI-NEXT:    s_waitcnt lgkmcnt(0)986; SI-NEXT:    s_mov_b32 s4, s2987; SI-NEXT:    s_mov_b32 s5, s3988; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 2989; SI-NEXT:    s_mov_b32 s2, 0990; SI-NEXT:    s_mov_b32 s3, s7991; SI-NEXT:    v_mov_b32_e32 v2, s6992; SI-NEXT:    v_mov_b32_e32 v0, s8993; SI-NEXT:    v_mov_b32_e32 v1, s9994; SI-NEXT:    buffer_atomic_and v2, v[0:1], s[0:3], 0 addr64 offset:16 glc995; SI-NEXT:    s_waitcnt vmcnt(0)996; SI-NEXT:    buffer_wbinvl1997; SI-NEXT:    s_mov_b32 s6, -1998; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 0999; SI-NEXT:    s_endpgm1000;1001; VI-LABEL: atomic_and_i32_ret_addr64_offset:1002; VI:       ; %bb.0: ; %entry1003; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c1004; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241005; VI-NEXT:    s_load_dword s8, s[4:5], 0x341006; VI-NEXT:    s_waitcnt lgkmcnt(0)1007; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 21008; VI-NEXT:    s_add_u32 s0, s0, s41009; VI-NEXT:    s_addc_u32 s1, s1, s51010; VI-NEXT:    s_add_u32 s0, s0, 161011; VI-NEXT:    s_addc_u32 s1, s1, 01012; VI-NEXT:    v_mov_b32_e32 v0, s01013; VI-NEXT:    v_mov_b32_e32 v1, s11014; VI-NEXT:    v_mov_b32_e32 v2, s81015; VI-NEXT:    flat_atomic_and v0, v[0:1], v2 glc1016; VI-NEXT:    s_waitcnt vmcnt(0)1017; VI-NEXT:    buffer_wbinvl1_vol1018; VI-NEXT:    s_mov_b32 s7, 0xf0001019; VI-NEXT:    s_mov_b32 s6, -11020; VI-NEXT:    s_mov_b32 s4, s21021; VI-NEXT:    s_mov_b32 s5, s31022; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 01023; VI-NEXT:    s_endpgm1024;1025; GFX9-LABEL: atomic_and_i32_ret_addr64_offset:1026; GFX9:       ; %bb.0: ; %entry1027; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c1028; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241029; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x341030; GFX9-NEXT:    v_mov_b32_e32 v0, 01031; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1032; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 21033; GFX9-NEXT:    s_add_u32 s0, s0, s41034; GFX9-NEXT:    s_addc_u32 s1, s1, s51035; GFX9-NEXT:    v_mov_b32_e32 v1, s81036; GFX9-NEXT:    global_atomic_and v1, v0, v1, s[0:1] offset:16 glc1037; GFX9-NEXT:    s_waitcnt vmcnt(0)1038; GFX9-NEXT:    buffer_wbinvl1_vol1039; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]1040; GFX9-NEXT:    s_endpgm1041;1042; GFX11-LABEL: atomic_and_i32_ret_addr64_offset:1043; GFX11:       ; %bb.0: ; %entry1044; GFX11-NEXT:    s_clause 0x21045; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c1046; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x341047; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241048; GFX11-NEXT:    v_mov_b32_e32 v0, 01049; GFX11-NEXT:    s_waitcnt lgkmcnt(0)1050; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 21051; GFX11-NEXT:    v_mov_b32_e32 v1, s81052; GFX11-NEXT:    s_add_u32 s0, s0, s41053; GFX11-NEXT:    s_addc_u32 s1, s1, s51054; GFX11-NEXT:    global_atomic_and_b32 v1, v0, v1, s[0:1] offset:16 glc1055; GFX11-NEXT:    s_waitcnt vmcnt(0)1056; GFX11-NEXT:    buffer_gl1_inv1057; GFX11-NEXT:    buffer_gl0_inv1058; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]1059; GFX11-NEXT:    s_endpgm1060entry:1061  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index1062  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 41063  %val = atomicrmw volatile and ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01064  store i32 %val, ptr addrspace(1) %out21065  ret void1066}1067 1068define amdgpu_kernel void @atomic_and_i32(ptr addrspace(1) %out, i32 %in) {1069; SI-LABEL: atomic_and_i32:1070; SI:       ; %bb.0: ; %entry1071; SI-NEXT:    s_load_dword s6, s[4:5], 0xb1072; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x91073; SI-NEXT:    s_mov_b32 s3, 0xf0001074; SI-NEXT:    s_mov_b32 s2, -11075; SI-NEXT:    s_waitcnt lgkmcnt(0)1076; SI-NEXT:    v_mov_b32_e32 v0, s61077; SI-NEXT:    buffer_atomic_and v0, off, s[0:3], 01078; SI-NEXT:    s_waitcnt vmcnt(0)1079; SI-NEXT:    buffer_wbinvl11080; SI-NEXT:    s_endpgm1081;1082; VI-LABEL: atomic_and_i32:1083; VI:       ; %bb.0: ; %entry1084; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c1085; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241086; VI-NEXT:    s_mov_b32 s3, 0xf0001087; VI-NEXT:    s_mov_b32 s2, -11088; VI-NEXT:    s_waitcnt lgkmcnt(0)1089; VI-NEXT:    v_mov_b32_e32 v0, s61090; VI-NEXT:    buffer_atomic_and v0, off, s[0:3], 01091; VI-NEXT:    s_waitcnt vmcnt(0)1092; VI-NEXT:    buffer_wbinvl1_vol1093; VI-NEXT:    s_endpgm1094;1095; GFX9-LABEL: atomic_and_i32:1096; GFX9:       ; %bb.0: ; %entry1097; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c1098; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241099; GFX9-NEXT:    v_mov_b32_e32 v0, 01100; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1101; GFX9-NEXT:    v_mov_b32_e32 v1, s21102; GFX9-NEXT:    global_atomic_and v0, v1, s[0:1]1103; GFX9-NEXT:    s_waitcnt vmcnt(0)1104; GFX9-NEXT:    buffer_wbinvl1_vol1105; GFX9-NEXT:    s_endpgm1106;1107; GFX11-LABEL: atomic_and_i32:1108; GFX11:       ; %bb.0: ; %entry1109; GFX11-NEXT:    s_clause 0x11110; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c1111; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241112; GFX11-NEXT:    s_waitcnt lgkmcnt(0)1113; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s21114; GFX11-NEXT:    global_atomic_and_b32 v0, v1, s[0:1]1115; GFX11-NEXT:    s_waitcnt_vscnt null, 0x01116; GFX11-NEXT:    buffer_gl1_inv1117; GFX11-NEXT:    buffer_gl0_inv1118; GFX11-NEXT:    s_endpgm1119entry:1120  %val = atomicrmw volatile and ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01121  ret void1122}1123 1124define amdgpu_kernel void @atomic_and_i32_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) {1125; SI-LABEL: atomic_and_i32_ret:1126; SI:       ; %bb.0: ; %entry1127; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x91128; SI-NEXT:    s_load_dword s8, s[4:5], 0xd1129; SI-NEXT:    s_mov_b32 s7, 0xf0001130; SI-NEXT:    s_mov_b32 s6, -11131; SI-NEXT:    s_waitcnt lgkmcnt(0)1132; SI-NEXT:    s_mov_b32 s4, s01133; SI-NEXT:    s_mov_b32 s5, s11134; SI-NEXT:    v_mov_b32_e32 v0, s81135; SI-NEXT:    buffer_atomic_and v0, off, s[4:7], 0 glc1136; SI-NEXT:    s_waitcnt vmcnt(0)1137; SI-NEXT:    buffer_wbinvl11138; SI-NEXT:    s_mov_b32 s4, s21139; SI-NEXT:    s_mov_b32 s5, s31140; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 01141; SI-NEXT:    s_endpgm1142;1143; VI-LABEL: atomic_and_i32_ret:1144; VI:       ; %bb.0: ; %entry1145; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241146; VI-NEXT:    s_load_dword s8, s[4:5], 0x341147; VI-NEXT:    s_mov_b32 s7, 0xf0001148; VI-NEXT:    s_mov_b32 s6, -11149; VI-NEXT:    s_waitcnt lgkmcnt(0)1150; VI-NEXT:    s_mov_b32 s4, s01151; VI-NEXT:    s_mov_b32 s5, s11152; VI-NEXT:    v_mov_b32_e32 v0, s81153; VI-NEXT:    buffer_atomic_and v0, off, s[4:7], 0 glc1154; VI-NEXT:    s_waitcnt vmcnt(0)1155; VI-NEXT:    buffer_wbinvl1_vol1156; VI-NEXT:    s_mov_b32 s4, s21157; VI-NEXT:    s_mov_b32 s5, s31158; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 01159; VI-NEXT:    s_endpgm1160;1161; GFX9-LABEL: atomic_and_i32_ret:1162; GFX9:       ; %bb.0: ; %entry1163; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x341164; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241165; GFX9-NEXT:    v_mov_b32_e32 v0, 01166; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1167; GFX9-NEXT:    v_mov_b32_e32 v1, s61168; GFX9-NEXT:    global_atomic_and v1, v0, v1, s[0:1] glc1169; GFX9-NEXT:    s_waitcnt vmcnt(0)1170; GFX9-NEXT:    buffer_wbinvl1_vol1171; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]1172; GFX9-NEXT:    s_endpgm1173;1174; GFX11-LABEL: atomic_and_i32_ret:1175; GFX11:       ; %bb.0: ; %entry1176; GFX11-NEXT:    s_clause 0x11177; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x341178; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241179; GFX11-NEXT:    s_waitcnt lgkmcnt(0)1180; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s61181; GFX11-NEXT:    global_atomic_and_b32 v1, v0, v1, s[0:1] glc1182; GFX11-NEXT:    s_waitcnt vmcnt(0)1183; GFX11-NEXT:    buffer_gl1_inv1184; GFX11-NEXT:    buffer_gl0_inv1185; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]1186; GFX11-NEXT:    s_endpgm1187entry:1188  %val = atomicrmw volatile and ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01189  store i32 %val, ptr addrspace(1) %out21190  ret void1191}1192 1193define amdgpu_kernel void @atomic_and_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) {1194; SI-LABEL: atomic_and_i32_addr64:1195; SI:       ; %bb.0: ; %entry1196; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd1197; SI-NEXT:    s_load_dword s8, s[4:5], 0xb1198; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x91199; SI-NEXT:    s_mov_b32 s3, 0xf0001200; SI-NEXT:    s_waitcnt lgkmcnt(0)1201; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 21202; SI-NEXT:    s_mov_b32 s2, 01203; SI-NEXT:    v_mov_b32_e32 v2, s81204; SI-NEXT:    v_mov_b32_e32 v0, s41205; SI-NEXT:    v_mov_b32_e32 v1, s51206; SI-NEXT:    buffer_atomic_and v2, v[0:1], s[0:3], 0 addr641207; SI-NEXT:    s_waitcnt vmcnt(0)1208; SI-NEXT:    buffer_wbinvl11209; SI-NEXT:    s_endpgm1210;1211; VI-LABEL: atomic_and_i32_addr64:1212; VI:       ; %bb.0: ; %entry1213; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x341214; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x241215; VI-NEXT:    s_load_dword s4, s[4:5], 0x2c1216; VI-NEXT:    s_waitcnt lgkmcnt(0)1217; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 21218; VI-NEXT:    s_add_u32 s0, s2, s01219; VI-NEXT:    s_addc_u32 s1, s3, s11220; VI-NEXT:    v_mov_b32_e32 v0, s01221; VI-NEXT:    v_mov_b32_e32 v1, s11222; VI-NEXT:    v_mov_b32_e32 v2, s41223; VI-NEXT:    flat_atomic_and v[0:1], v21224; VI-NEXT:    s_waitcnt vmcnt(0)1225; VI-NEXT:    buffer_wbinvl1_vol1226; VI-NEXT:    s_endpgm1227;1228; GFX9-LABEL: atomic_and_i32_addr64:1229; GFX9:       ; %bb.0: ; %entry1230; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x341231; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x241232; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c1233; GFX9-NEXT:    v_mov_b32_e32 v0, 01234; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1235; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 21236; GFX9-NEXT:    s_add_u32 s0, s2, s01237; GFX9-NEXT:    s_addc_u32 s1, s3, s11238; GFX9-NEXT:    v_mov_b32_e32 v1, s61239; GFX9-NEXT:    global_atomic_and v0, v1, s[0:1]1240; GFX9-NEXT:    s_waitcnt vmcnt(0)1241; GFX9-NEXT:    buffer_wbinvl1_vol1242; GFX9-NEXT:    s_endpgm1243;1244; GFX11-LABEL: atomic_and_i32_addr64:1245; GFX11:       ; %bb.0: ; %entry1246; GFX11-NEXT:    s_clause 0x21247; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x341248; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c1249; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x241250; GFX11-NEXT:    v_mov_b32_e32 v0, 01251; GFX11-NEXT:    s_waitcnt lgkmcnt(0)1252; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 21253; GFX11-NEXT:    v_mov_b32_e32 v1, s61254; GFX11-NEXT:    s_add_u32 s0, s2, s01255; GFX11-NEXT:    s_addc_u32 s1, s3, s11256; GFX11-NEXT:    global_atomic_and_b32 v0, v1, s[0:1]1257; GFX11-NEXT:    s_waitcnt_vscnt null, 0x01258; GFX11-NEXT:    buffer_gl1_inv1259; GFX11-NEXT:    buffer_gl0_inv1260; GFX11-NEXT:    s_endpgm1261entry:1262  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index1263  %val = atomicrmw volatile and ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01264  ret void1265}1266 1267define amdgpu_kernel void @atomic_and_i32_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) {1268; SI-LABEL: atomic_and_i32_ret_addr64:1269; SI:       ; %bb.0: ; %entry1270; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x91271; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf1272; SI-NEXT:    s_load_dword s6, s[4:5], 0xd1273; SI-NEXT:    s_mov_b32 s7, 0xf0001274; SI-NEXT:    s_waitcnt lgkmcnt(0)1275; SI-NEXT:    s_mov_b32 s4, s21276; SI-NEXT:    s_mov_b32 s5, s31277; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 21278; SI-NEXT:    s_mov_b32 s2, 01279; SI-NEXT:    s_mov_b32 s3, s71280; SI-NEXT:    v_mov_b32_e32 v2, s61281; SI-NEXT:    v_mov_b32_e32 v0, s81282; SI-NEXT:    v_mov_b32_e32 v1, s91283; SI-NEXT:    buffer_atomic_and v2, v[0:1], s[0:3], 0 addr64 glc1284; SI-NEXT:    s_waitcnt vmcnt(0)1285; SI-NEXT:    buffer_wbinvl11286; SI-NEXT:    s_mov_b32 s6, -11287; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 01288; SI-NEXT:    s_endpgm1289;1290; VI-LABEL: atomic_and_i32_ret_addr64:1291; VI:       ; %bb.0: ; %entry1292; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c1293; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241294; VI-NEXT:    s_load_dword s8, s[4:5], 0x341295; VI-NEXT:    s_waitcnt lgkmcnt(0)1296; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 21297; VI-NEXT:    s_add_u32 s0, s0, s41298; VI-NEXT:    s_addc_u32 s1, s1, s51299; VI-NEXT:    v_mov_b32_e32 v0, s01300; VI-NEXT:    v_mov_b32_e32 v1, s11301; VI-NEXT:    v_mov_b32_e32 v2, s81302; VI-NEXT:    flat_atomic_and v0, v[0:1], v2 glc1303; VI-NEXT:    s_waitcnt vmcnt(0)1304; VI-NEXT:    buffer_wbinvl1_vol1305; VI-NEXT:    s_mov_b32 s7, 0xf0001306; VI-NEXT:    s_mov_b32 s6, -11307; VI-NEXT:    s_mov_b32 s4, s21308; VI-NEXT:    s_mov_b32 s5, s31309; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 01310; VI-NEXT:    s_endpgm1311;1312; GFX9-LABEL: atomic_and_i32_ret_addr64:1313; GFX9:       ; %bb.0: ; %entry1314; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c1315; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241316; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x341317; GFX9-NEXT:    v_mov_b32_e32 v0, 01318; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1319; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 21320; GFX9-NEXT:    s_add_u32 s0, s0, s41321; GFX9-NEXT:    s_addc_u32 s1, s1, s51322; GFX9-NEXT:    v_mov_b32_e32 v1, s81323; GFX9-NEXT:    global_atomic_and v1, v0, v1, s[0:1] glc1324; GFX9-NEXT:    s_waitcnt vmcnt(0)1325; GFX9-NEXT:    buffer_wbinvl1_vol1326; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]1327; GFX9-NEXT:    s_endpgm1328;1329; GFX11-LABEL: atomic_and_i32_ret_addr64:1330; GFX11:       ; %bb.0: ; %entry1331; GFX11-NEXT:    s_clause 0x21332; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c1333; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x341334; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241335; GFX11-NEXT:    v_mov_b32_e32 v0, 01336; GFX11-NEXT:    s_waitcnt lgkmcnt(0)1337; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 21338; GFX11-NEXT:    v_mov_b32_e32 v1, s81339; GFX11-NEXT:    s_add_u32 s0, s0, s41340; GFX11-NEXT:    s_addc_u32 s1, s1, s51341; GFX11-NEXT:    global_atomic_and_b32 v1, v0, v1, s[0:1] glc1342; GFX11-NEXT:    s_waitcnt vmcnt(0)1343; GFX11-NEXT:    buffer_gl1_inv1344; GFX11-NEXT:    buffer_gl0_inv1345; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]1346; GFX11-NEXT:    s_endpgm1347entry:1348  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index1349  %val = atomicrmw volatile and ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01350  store i32 %val, ptr addrspace(1) %out21351  ret void1352}1353 1354define amdgpu_kernel void @atomic_sub_i32_offset(ptr addrspace(1) %out, i32 %in) {1355; SI-LABEL: atomic_sub_i32_offset:1356; SI:       ; %bb.0: ; %entry1357; SI-NEXT:    s_load_dword s6, s[4:5], 0xb1358; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x91359; SI-NEXT:    s_mov_b32 s3, 0xf0001360; SI-NEXT:    s_mov_b32 s2, -11361; SI-NEXT:    s_waitcnt lgkmcnt(0)1362; SI-NEXT:    v_mov_b32_e32 v0, s61363; SI-NEXT:    buffer_atomic_sub v0, off, s[0:3], 0 offset:161364; SI-NEXT:    s_waitcnt vmcnt(0)1365; SI-NEXT:    buffer_wbinvl11366; SI-NEXT:    s_endpgm1367;1368; VI-LABEL: atomic_sub_i32_offset:1369; VI:       ; %bb.0: ; %entry1370; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c1371; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241372; VI-NEXT:    s_mov_b32 s3, 0xf0001373; VI-NEXT:    s_mov_b32 s2, -11374; VI-NEXT:    s_waitcnt lgkmcnt(0)1375; VI-NEXT:    v_mov_b32_e32 v0, s61376; VI-NEXT:    buffer_atomic_sub v0, off, s[0:3], 0 offset:161377; VI-NEXT:    s_waitcnt vmcnt(0)1378; VI-NEXT:    buffer_wbinvl1_vol1379; VI-NEXT:    s_endpgm1380;1381; GFX9-LABEL: atomic_sub_i32_offset:1382; GFX9:       ; %bb.0: ; %entry1383; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c1384; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241385; GFX9-NEXT:    v_mov_b32_e32 v0, 01386; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1387; GFX9-NEXT:    v_mov_b32_e32 v1, s21388; GFX9-NEXT:    global_atomic_sub v0, v1, s[0:1] offset:161389; GFX9-NEXT:    s_waitcnt vmcnt(0)1390; GFX9-NEXT:    buffer_wbinvl1_vol1391; GFX9-NEXT:    s_endpgm1392;1393; GFX11-LABEL: atomic_sub_i32_offset:1394; GFX11:       ; %bb.0: ; %entry1395; GFX11-NEXT:    s_clause 0x11396; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c1397; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241398; GFX11-NEXT:    s_waitcnt lgkmcnt(0)1399; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s21400; GFX11-NEXT:    global_atomic_sub_u32 v0, v1, s[0:1] offset:161401; GFX11-NEXT:    s_waitcnt_vscnt null, 0x01402; GFX11-NEXT:    buffer_gl1_inv1403; GFX11-NEXT:    buffer_gl0_inv1404; GFX11-NEXT:    s_endpgm1405entry:1406  %gep = getelementptr i32, ptr addrspace(1) %out, i64 41407  %val = atomicrmw volatile sub ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01408  ret void1409}1410 1411define amdgpu_kernel void @atomic_sub_i32_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) {1412; SI-LABEL: atomic_sub_i32_ret_offset:1413; SI:       ; %bb.0: ; %entry1414; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x91415; SI-NEXT:    s_load_dword s8, s[4:5], 0xd1416; SI-NEXT:    s_mov_b32 s7, 0xf0001417; SI-NEXT:    s_mov_b32 s6, -11418; SI-NEXT:    s_waitcnt lgkmcnt(0)1419; SI-NEXT:    s_mov_b32 s4, s21420; SI-NEXT:    s_mov_b32 s5, s31421; SI-NEXT:    s_mov_b32 s2, s61422; SI-NEXT:    s_mov_b32 s3, s71423; SI-NEXT:    v_mov_b32_e32 v0, s81424; SI-NEXT:    buffer_atomic_sub v0, off, s[0:3], 0 offset:16 glc1425; SI-NEXT:    s_waitcnt vmcnt(0)1426; SI-NEXT:    buffer_wbinvl11427; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 01428; SI-NEXT:    s_endpgm1429;1430; VI-LABEL: atomic_sub_i32_ret_offset:1431; VI:       ; %bb.0: ; %entry1432; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241433; VI-NEXT:    s_load_dword s8, s[4:5], 0x341434; VI-NEXT:    s_mov_b32 s7, 0xf0001435; VI-NEXT:    s_mov_b32 s6, -11436; VI-NEXT:    s_waitcnt lgkmcnt(0)1437; VI-NEXT:    s_mov_b32 s4, s21438; VI-NEXT:    s_mov_b32 s5, s31439; VI-NEXT:    s_mov_b32 s2, s61440; VI-NEXT:    s_mov_b32 s3, s71441; VI-NEXT:    v_mov_b32_e32 v0, s81442; VI-NEXT:    buffer_atomic_sub v0, off, s[0:3], 0 offset:16 glc1443; VI-NEXT:    s_waitcnt vmcnt(0)1444; VI-NEXT:    buffer_wbinvl1_vol1445; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 01446; VI-NEXT:    s_endpgm1447;1448; GFX9-LABEL: atomic_sub_i32_ret_offset:1449; GFX9:       ; %bb.0: ; %entry1450; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x341451; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241452; GFX9-NEXT:    v_mov_b32_e32 v0, 01453; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1454; GFX9-NEXT:    v_mov_b32_e32 v1, s61455; GFX9-NEXT:    global_atomic_sub v1, v0, v1, s[0:1] offset:16 glc1456; GFX9-NEXT:    s_waitcnt vmcnt(0)1457; GFX9-NEXT:    buffer_wbinvl1_vol1458; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]1459; GFX9-NEXT:    s_endpgm1460;1461; GFX11-LABEL: atomic_sub_i32_ret_offset:1462; GFX11:       ; %bb.0: ; %entry1463; GFX11-NEXT:    s_clause 0x11464; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x341465; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241466; GFX11-NEXT:    s_waitcnt lgkmcnt(0)1467; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s61468; GFX11-NEXT:    global_atomic_sub_u32 v1, v0, v1, s[0:1] offset:16 glc1469; GFX11-NEXT:    s_waitcnt vmcnt(0)1470; GFX11-NEXT:    buffer_gl1_inv1471; GFX11-NEXT:    buffer_gl0_inv1472; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]1473; GFX11-NEXT:    s_endpgm1474entry:1475  %gep = getelementptr i32, ptr addrspace(1) %out, i64 41476  %val = atomicrmw volatile sub ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01477  store i32 %val, ptr addrspace(1) %out21478  ret void1479}1480 1481define amdgpu_kernel void @atomic_sub_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {1482; SI-LABEL: atomic_sub_i32_addr64_offset:1483; SI:       ; %bb.0: ; %entry1484; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd1485; SI-NEXT:    s_load_dword s8, s[4:5], 0xb1486; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x91487; SI-NEXT:    s_mov_b32 s3, 0xf0001488; SI-NEXT:    s_waitcnt lgkmcnt(0)1489; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 21490; SI-NEXT:    s_mov_b32 s2, 01491; SI-NEXT:    v_mov_b32_e32 v2, s81492; SI-NEXT:    v_mov_b32_e32 v0, s41493; SI-NEXT:    v_mov_b32_e32 v1, s51494; SI-NEXT:    buffer_atomic_sub v2, v[0:1], s[0:3], 0 addr64 offset:161495; SI-NEXT:    s_waitcnt vmcnt(0)1496; SI-NEXT:    buffer_wbinvl11497; SI-NEXT:    s_endpgm1498;1499; VI-LABEL: atomic_sub_i32_addr64_offset:1500; VI:       ; %bb.0: ; %entry1501; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x341502; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x241503; VI-NEXT:    s_load_dword s4, s[4:5], 0x2c1504; VI-NEXT:    s_waitcnt lgkmcnt(0)1505; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 21506; VI-NEXT:    s_add_u32 s0, s2, s01507; VI-NEXT:    s_addc_u32 s1, s3, s11508; VI-NEXT:    s_add_u32 s0, s0, 161509; VI-NEXT:    s_addc_u32 s1, s1, 01510; VI-NEXT:    v_mov_b32_e32 v0, s01511; VI-NEXT:    v_mov_b32_e32 v1, s11512; VI-NEXT:    v_mov_b32_e32 v2, s41513; VI-NEXT:    flat_atomic_sub v[0:1], v21514; VI-NEXT:    s_waitcnt vmcnt(0)1515; VI-NEXT:    buffer_wbinvl1_vol1516; VI-NEXT:    s_endpgm1517;1518; GFX9-LABEL: atomic_sub_i32_addr64_offset:1519; GFX9:       ; %bb.0: ; %entry1520; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x341521; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x241522; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c1523; GFX9-NEXT:    v_mov_b32_e32 v0, 01524; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1525; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 21526; GFX9-NEXT:    s_add_u32 s0, s2, s01527; GFX9-NEXT:    s_addc_u32 s1, s3, s11528; GFX9-NEXT:    v_mov_b32_e32 v1, s61529; GFX9-NEXT:    global_atomic_sub v0, v1, s[0:1] offset:161530; GFX9-NEXT:    s_waitcnt vmcnt(0)1531; GFX9-NEXT:    buffer_wbinvl1_vol1532; GFX9-NEXT:    s_endpgm1533;1534; GFX11-LABEL: atomic_sub_i32_addr64_offset:1535; GFX11:       ; %bb.0: ; %entry1536; GFX11-NEXT:    s_clause 0x21537; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x341538; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c1539; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x241540; GFX11-NEXT:    v_mov_b32_e32 v0, 01541; GFX11-NEXT:    s_waitcnt lgkmcnt(0)1542; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 21543; GFX11-NEXT:    v_mov_b32_e32 v1, s61544; GFX11-NEXT:    s_add_u32 s0, s2, s01545; GFX11-NEXT:    s_addc_u32 s1, s3, s11546; GFX11-NEXT:    global_atomic_sub_u32 v0, v1, s[0:1] offset:161547; GFX11-NEXT:    s_waitcnt_vscnt null, 0x01548; GFX11-NEXT:    buffer_gl1_inv1549; GFX11-NEXT:    buffer_gl0_inv1550; GFX11-NEXT:    s_endpgm1551entry:1552  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index1553  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 41554  %val = atomicrmw volatile sub ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01555  ret void1556}1557 1558define amdgpu_kernel void @atomic_sub_i32_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) {1559; SI-LABEL: atomic_sub_i32_ret_addr64_offset:1560; SI:       ; %bb.0: ; %entry1561; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x91562; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf1563; SI-NEXT:    s_load_dword s6, s[4:5], 0xd1564; SI-NEXT:    s_mov_b32 s7, 0xf0001565; SI-NEXT:    s_waitcnt lgkmcnt(0)1566; SI-NEXT:    s_mov_b32 s4, s21567; SI-NEXT:    s_mov_b32 s5, s31568; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 21569; SI-NEXT:    s_mov_b32 s2, 01570; SI-NEXT:    s_mov_b32 s3, s71571; SI-NEXT:    v_mov_b32_e32 v2, s61572; SI-NEXT:    v_mov_b32_e32 v0, s81573; SI-NEXT:    v_mov_b32_e32 v1, s91574; SI-NEXT:    buffer_atomic_sub v2, v[0:1], s[0:3], 0 addr64 offset:16 glc1575; SI-NEXT:    s_waitcnt vmcnt(0)1576; SI-NEXT:    buffer_wbinvl11577; SI-NEXT:    s_mov_b32 s6, -11578; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 01579; SI-NEXT:    s_endpgm1580;1581; VI-LABEL: atomic_sub_i32_ret_addr64_offset:1582; VI:       ; %bb.0: ; %entry1583; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c1584; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241585; VI-NEXT:    s_load_dword s8, s[4:5], 0x341586; VI-NEXT:    s_waitcnt lgkmcnt(0)1587; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 21588; VI-NEXT:    s_add_u32 s0, s0, s41589; VI-NEXT:    s_addc_u32 s1, s1, s51590; VI-NEXT:    s_add_u32 s0, s0, 161591; VI-NEXT:    s_addc_u32 s1, s1, 01592; VI-NEXT:    v_mov_b32_e32 v0, s01593; VI-NEXT:    v_mov_b32_e32 v1, s11594; VI-NEXT:    v_mov_b32_e32 v2, s81595; VI-NEXT:    flat_atomic_sub v0, v[0:1], v2 glc1596; VI-NEXT:    s_waitcnt vmcnt(0)1597; VI-NEXT:    buffer_wbinvl1_vol1598; VI-NEXT:    s_mov_b32 s7, 0xf0001599; VI-NEXT:    s_mov_b32 s6, -11600; VI-NEXT:    s_mov_b32 s4, s21601; VI-NEXT:    s_mov_b32 s5, s31602; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 01603; VI-NEXT:    s_endpgm1604;1605; GFX9-LABEL: atomic_sub_i32_ret_addr64_offset:1606; GFX9:       ; %bb.0: ; %entry1607; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c1608; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241609; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x341610; GFX9-NEXT:    v_mov_b32_e32 v0, 01611; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1612; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 21613; GFX9-NEXT:    s_add_u32 s0, s0, s41614; GFX9-NEXT:    s_addc_u32 s1, s1, s51615; GFX9-NEXT:    v_mov_b32_e32 v1, s81616; GFX9-NEXT:    global_atomic_sub v1, v0, v1, s[0:1] offset:16 glc1617; GFX9-NEXT:    s_waitcnt vmcnt(0)1618; GFX9-NEXT:    buffer_wbinvl1_vol1619; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]1620; GFX9-NEXT:    s_endpgm1621;1622; GFX11-LABEL: atomic_sub_i32_ret_addr64_offset:1623; GFX11:       ; %bb.0: ; %entry1624; GFX11-NEXT:    s_clause 0x21625; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c1626; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x341627; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241628; GFX11-NEXT:    v_mov_b32_e32 v0, 01629; GFX11-NEXT:    s_waitcnt lgkmcnt(0)1630; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 21631; GFX11-NEXT:    v_mov_b32_e32 v1, s81632; GFX11-NEXT:    s_add_u32 s0, s0, s41633; GFX11-NEXT:    s_addc_u32 s1, s1, s51634; GFX11-NEXT:    global_atomic_sub_u32 v1, v0, v1, s[0:1] offset:16 glc1635; GFX11-NEXT:    s_waitcnt vmcnt(0)1636; GFX11-NEXT:    buffer_gl1_inv1637; GFX11-NEXT:    buffer_gl0_inv1638; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]1639; GFX11-NEXT:    s_endpgm1640entry:1641  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index1642  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 41643  %val = atomicrmw volatile sub ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01644  store i32 %val, ptr addrspace(1) %out21645  ret void1646}1647 1648define amdgpu_kernel void @atomic_sub_i32(ptr addrspace(1) %out, i32 %in) {1649; SI-LABEL: atomic_sub_i32:1650; SI:       ; %bb.0: ; %entry1651; SI-NEXT:    s_load_dword s6, s[4:5], 0xb1652; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x91653; SI-NEXT:    s_mov_b32 s3, 0xf0001654; SI-NEXT:    s_mov_b32 s2, -11655; SI-NEXT:    s_waitcnt lgkmcnt(0)1656; SI-NEXT:    v_mov_b32_e32 v0, s61657; SI-NEXT:    buffer_atomic_sub v0, off, s[0:3], 01658; SI-NEXT:    s_waitcnt vmcnt(0)1659; SI-NEXT:    buffer_wbinvl11660; SI-NEXT:    s_endpgm1661;1662; VI-LABEL: atomic_sub_i32:1663; VI:       ; %bb.0: ; %entry1664; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c1665; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241666; VI-NEXT:    s_mov_b32 s3, 0xf0001667; VI-NEXT:    s_mov_b32 s2, -11668; VI-NEXT:    s_waitcnt lgkmcnt(0)1669; VI-NEXT:    v_mov_b32_e32 v0, s61670; VI-NEXT:    buffer_atomic_sub v0, off, s[0:3], 01671; VI-NEXT:    s_waitcnt vmcnt(0)1672; VI-NEXT:    buffer_wbinvl1_vol1673; VI-NEXT:    s_endpgm1674;1675; GFX9-LABEL: atomic_sub_i32:1676; GFX9:       ; %bb.0: ; %entry1677; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c1678; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241679; GFX9-NEXT:    v_mov_b32_e32 v0, 01680; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1681; GFX9-NEXT:    v_mov_b32_e32 v1, s21682; GFX9-NEXT:    global_atomic_sub v0, v1, s[0:1]1683; GFX9-NEXT:    s_waitcnt vmcnt(0)1684; GFX9-NEXT:    buffer_wbinvl1_vol1685; GFX9-NEXT:    s_endpgm1686;1687; GFX11-LABEL: atomic_sub_i32:1688; GFX11:       ; %bb.0: ; %entry1689; GFX11-NEXT:    s_clause 0x11690; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c1691; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241692; GFX11-NEXT:    s_waitcnt lgkmcnt(0)1693; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s21694; GFX11-NEXT:    global_atomic_sub_u32 v0, v1, s[0:1]1695; GFX11-NEXT:    s_waitcnt_vscnt null, 0x01696; GFX11-NEXT:    buffer_gl1_inv1697; GFX11-NEXT:    buffer_gl0_inv1698; GFX11-NEXT:    s_endpgm1699entry:1700  %val = atomicrmw volatile sub ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01701  ret void1702}1703 1704define amdgpu_kernel void @atomic_sub_i32_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) {1705; SI-LABEL: atomic_sub_i32_ret:1706; SI:       ; %bb.0: ; %entry1707; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x91708; SI-NEXT:    s_load_dword s8, s[4:5], 0xd1709; SI-NEXT:    s_mov_b32 s7, 0xf0001710; SI-NEXT:    s_mov_b32 s6, -11711; SI-NEXT:    s_waitcnt lgkmcnt(0)1712; SI-NEXT:    s_mov_b32 s4, s01713; SI-NEXT:    s_mov_b32 s5, s11714; SI-NEXT:    v_mov_b32_e32 v0, s81715; SI-NEXT:    buffer_atomic_sub v0, off, s[4:7], 0 glc1716; SI-NEXT:    s_waitcnt vmcnt(0)1717; SI-NEXT:    buffer_wbinvl11718; SI-NEXT:    s_mov_b32 s4, s21719; SI-NEXT:    s_mov_b32 s5, s31720; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 01721; SI-NEXT:    s_endpgm1722;1723; VI-LABEL: atomic_sub_i32_ret:1724; VI:       ; %bb.0: ; %entry1725; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241726; VI-NEXT:    s_load_dword s8, s[4:5], 0x341727; VI-NEXT:    s_mov_b32 s7, 0xf0001728; VI-NEXT:    s_mov_b32 s6, -11729; VI-NEXT:    s_waitcnt lgkmcnt(0)1730; VI-NEXT:    s_mov_b32 s4, s01731; VI-NEXT:    s_mov_b32 s5, s11732; VI-NEXT:    v_mov_b32_e32 v0, s81733; VI-NEXT:    buffer_atomic_sub v0, off, s[4:7], 0 glc1734; VI-NEXT:    s_waitcnt vmcnt(0)1735; VI-NEXT:    buffer_wbinvl1_vol1736; VI-NEXT:    s_mov_b32 s4, s21737; VI-NEXT:    s_mov_b32 s5, s31738; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 01739; VI-NEXT:    s_endpgm1740;1741; GFX9-LABEL: atomic_sub_i32_ret:1742; GFX9:       ; %bb.0: ; %entry1743; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x341744; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241745; GFX9-NEXT:    v_mov_b32_e32 v0, 01746; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1747; GFX9-NEXT:    v_mov_b32_e32 v1, s61748; GFX9-NEXT:    global_atomic_sub v1, v0, v1, s[0:1] glc1749; GFX9-NEXT:    s_waitcnt vmcnt(0)1750; GFX9-NEXT:    buffer_wbinvl1_vol1751; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]1752; GFX9-NEXT:    s_endpgm1753;1754; GFX11-LABEL: atomic_sub_i32_ret:1755; GFX11:       ; %bb.0: ; %entry1756; GFX11-NEXT:    s_clause 0x11757; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x341758; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241759; GFX11-NEXT:    s_waitcnt lgkmcnt(0)1760; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s61761; GFX11-NEXT:    global_atomic_sub_u32 v1, v0, v1, s[0:1] glc1762; GFX11-NEXT:    s_waitcnt vmcnt(0)1763; GFX11-NEXT:    buffer_gl1_inv1764; GFX11-NEXT:    buffer_gl0_inv1765; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]1766; GFX11-NEXT:    s_endpgm1767entry:1768  %val = atomicrmw volatile sub ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01769  store i32 %val, ptr addrspace(1) %out21770  ret void1771}1772 1773define amdgpu_kernel void @atomic_sub_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) {1774; SI-LABEL: atomic_sub_i32_addr64:1775; SI:       ; %bb.0: ; %entry1776; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd1777; SI-NEXT:    s_load_dword s8, s[4:5], 0xb1778; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x91779; SI-NEXT:    s_mov_b32 s3, 0xf0001780; SI-NEXT:    s_waitcnt lgkmcnt(0)1781; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 21782; SI-NEXT:    s_mov_b32 s2, 01783; SI-NEXT:    v_mov_b32_e32 v2, s81784; SI-NEXT:    v_mov_b32_e32 v0, s41785; SI-NEXT:    v_mov_b32_e32 v1, s51786; SI-NEXT:    buffer_atomic_sub v2, v[0:1], s[0:3], 0 addr641787; SI-NEXT:    s_waitcnt vmcnt(0)1788; SI-NEXT:    buffer_wbinvl11789; SI-NEXT:    s_endpgm1790;1791; VI-LABEL: atomic_sub_i32_addr64:1792; VI:       ; %bb.0: ; %entry1793; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x341794; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x241795; VI-NEXT:    s_load_dword s4, s[4:5], 0x2c1796; VI-NEXT:    s_waitcnt lgkmcnt(0)1797; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 21798; VI-NEXT:    s_add_u32 s0, s2, s01799; VI-NEXT:    s_addc_u32 s1, s3, s11800; VI-NEXT:    v_mov_b32_e32 v0, s01801; VI-NEXT:    v_mov_b32_e32 v1, s11802; VI-NEXT:    v_mov_b32_e32 v2, s41803; VI-NEXT:    flat_atomic_sub v[0:1], v21804; VI-NEXT:    s_waitcnt vmcnt(0)1805; VI-NEXT:    buffer_wbinvl1_vol1806; VI-NEXT:    s_endpgm1807;1808; GFX9-LABEL: atomic_sub_i32_addr64:1809; GFX9:       ; %bb.0: ; %entry1810; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x341811; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x241812; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c1813; GFX9-NEXT:    v_mov_b32_e32 v0, 01814; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1815; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 21816; GFX9-NEXT:    s_add_u32 s0, s2, s01817; GFX9-NEXT:    s_addc_u32 s1, s3, s11818; GFX9-NEXT:    v_mov_b32_e32 v1, s61819; GFX9-NEXT:    global_atomic_sub v0, v1, s[0:1]1820; GFX9-NEXT:    s_waitcnt vmcnt(0)1821; GFX9-NEXT:    buffer_wbinvl1_vol1822; GFX9-NEXT:    s_endpgm1823;1824; GFX11-LABEL: atomic_sub_i32_addr64:1825; GFX11:       ; %bb.0: ; %entry1826; GFX11-NEXT:    s_clause 0x21827; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x341828; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c1829; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x241830; GFX11-NEXT:    v_mov_b32_e32 v0, 01831; GFX11-NEXT:    s_waitcnt lgkmcnt(0)1832; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 21833; GFX11-NEXT:    v_mov_b32_e32 v1, s61834; GFX11-NEXT:    s_add_u32 s0, s2, s01835; GFX11-NEXT:    s_addc_u32 s1, s3, s11836; GFX11-NEXT:    global_atomic_sub_u32 v0, v1, s[0:1]1837; GFX11-NEXT:    s_waitcnt_vscnt null, 0x01838; GFX11-NEXT:    buffer_gl1_inv1839; GFX11-NEXT:    buffer_gl0_inv1840; GFX11-NEXT:    s_endpgm1841entry:1842  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index1843  %val = atomicrmw volatile sub ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01844  ret void1845}1846 1847define amdgpu_kernel void @atomic_sub_i32_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) {1848; SI-LABEL: atomic_sub_i32_ret_addr64:1849; SI:       ; %bb.0: ; %entry1850; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x91851; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf1852; SI-NEXT:    s_load_dword s6, s[4:5], 0xd1853; SI-NEXT:    s_mov_b32 s7, 0xf0001854; SI-NEXT:    s_waitcnt lgkmcnt(0)1855; SI-NEXT:    s_mov_b32 s4, s21856; SI-NEXT:    s_mov_b32 s5, s31857; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 21858; SI-NEXT:    s_mov_b32 s2, 01859; SI-NEXT:    s_mov_b32 s3, s71860; SI-NEXT:    v_mov_b32_e32 v2, s61861; SI-NEXT:    v_mov_b32_e32 v0, s81862; SI-NEXT:    v_mov_b32_e32 v1, s91863; SI-NEXT:    buffer_atomic_sub v2, v[0:1], s[0:3], 0 addr64 glc1864; SI-NEXT:    s_waitcnt vmcnt(0)1865; SI-NEXT:    buffer_wbinvl11866; SI-NEXT:    s_mov_b32 s6, -11867; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 01868; SI-NEXT:    s_endpgm1869;1870; VI-LABEL: atomic_sub_i32_ret_addr64:1871; VI:       ; %bb.0: ; %entry1872; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c1873; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241874; VI-NEXT:    s_load_dword s8, s[4:5], 0x341875; VI-NEXT:    s_waitcnt lgkmcnt(0)1876; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 21877; VI-NEXT:    s_add_u32 s0, s0, s41878; VI-NEXT:    s_addc_u32 s1, s1, s51879; VI-NEXT:    v_mov_b32_e32 v0, s01880; VI-NEXT:    v_mov_b32_e32 v1, s11881; VI-NEXT:    v_mov_b32_e32 v2, s81882; VI-NEXT:    flat_atomic_sub v0, v[0:1], v2 glc1883; VI-NEXT:    s_waitcnt vmcnt(0)1884; VI-NEXT:    buffer_wbinvl1_vol1885; VI-NEXT:    s_mov_b32 s7, 0xf0001886; VI-NEXT:    s_mov_b32 s6, -11887; VI-NEXT:    s_mov_b32 s4, s21888; VI-NEXT:    s_mov_b32 s5, s31889; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 01890; VI-NEXT:    s_endpgm1891;1892; GFX9-LABEL: atomic_sub_i32_ret_addr64:1893; GFX9:       ; %bb.0: ; %entry1894; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c1895; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241896; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x341897; GFX9-NEXT:    v_mov_b32_e32 v0, 01898; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1899; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 21900; GFX9-NEXT:    s_add_u32 s0, s0, s41901; GFX9-NEXT:    s_addc_u32 s1, s1, s51902; GFX9-NEXT:    v_mov_b32_e32 v1, s81903; GFX9-NEXT:    global_atomic_sub v1, v0, v1, s[0:1] glc1904; GFX9-NEXT:    s_waitcnt vmcnt(0)1905; GFX9-NEXT:    buffer_wbinvl1_vol1906; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]1907; GFX9-NEXT:    s_endpgm1908;1909; GFX11-LABEL: atomic_sub_i32_ret_addr64:1910; GFX11:       ; %bb.0: ; %entry1911; GFX11-NEXT:    s_clause 0x21912; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c1913; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x341914; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241915; GFX11-NEXT:    v_mov_b32_e32 v0, 01916; GFX11-NEXT:    s_waitcnt lgkmcnt(0)1917; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 21918; GFX11-NEXT:    v_mov_b32_e32 v1, s81919; GFX11-NEXT:    s_add_u32 s0, s0, s41920; GFX11-NEXT:    s_addc_u32 s1, s1, s51921; GFX11-NEXT:    global_atomic_sub_u32 v1, v0, v1, s[0:1] glc1922; GFX11-NEXT:    s_waitcnt vmcnt(0)1923; GFX11-NEXT:    buffer_gl1_inv1924; GFX11-NEXT:    buffer_gl0_inv1925; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]1926; GFX11-NEXT:    s_endpgm1927entry:1928  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index1929  %val = atomicrmw volatile sub ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01930  store i32 %val, ptr addrspace(1) %out21931  ret void1932}1933 1934define amdgpu_kernel void @atomic_max_i32_offset(ptr addrspace(1) %out, i32 %in) {1935; SI-LABEL: atomic_max_i32_offset:1936; SI:       ; %bb.0: ; %entry1937; SI-NEXT:    s_load_dword s6, s[4:5], 0xb1938; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x91939; SI-NEXT:    s_mov_b32 s3, 0xf0001940; SI-NEXT:    s_mov_b32 s2, -11941; SI-NEXT:    s_waitcnt lgkmcnt(0)1942; SI-NEXT:    v_mov_b32_e32 v0, s61943; SI-NEXT:    buffer_atomic_smax v0, off, s[0:3], 0 offset:161944; SI-NEXT:    s_waitcnt vmcnt(0)1945; SI-NEXT:    buffer_wbinvl11946; SI-NEXT:    s_endpgm1947;1948; VI-LABEL: atomic_max_i32_offset:1949; VI:       ; %bb.0: ; %entry1950; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c1951; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241952; VI-NEXT:    s_mov_b32 s3, 0xf0001953; VI-NEXT:    s_mov_b32 s2, -11954; VI-NEXT:    s_waitcnt lgkmcnt(0)1955; VI-NEXT:    v_mov_b32_e32 v0, s61956; VI-NEXT:    buffer_atomic_smax v0, off, s[0:3], 0 offset:161957; VI-NEXT:    s_waitcnt vmcnt(0)1958; VI-NEXT:    buffer_wbinvl1_vol1959; VI-NEXT:    s_endpgm1960;1961; GFX9-LABEL: atomic_max_i32_offset:1962; GFX9:       ; %bb.0: ; %entry1963; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c1964; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241965; GFX9-NEXT:    v_mov_b32_e32 v0, 01966; GFX9-NEXT:    s_waitcnt lgkmcnt(0)1967; GFX9-NEXT:    v_mov_b32_e32 v1, s21968; GFX9-NEXT:    global_atomic_smax v0, v1, s[0:1] offset:161969; GFX9-NEXT:    s_waitcnt vmcnt(0)1970; GFX9-NEXT:    buffer_wbinvl1_vol1971; GFX9-NEXT:    s_endpgm1972;1973; GFX11-LABEL: atomic_max_i32_offset:1974; GFX11:       ; %bb.0: ; %entry1975; GFX11-NEXT:    s_clause 0x11976; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c1977; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241978; GFX11-NEXT:    s_waitcnt lgkmcnt(0)1979; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s21980; GFX11-NEXT:    global_atomic_max_i32 v0, v1, s[0:1] offset:161981; GFX11-NEXT:    s_waitcnt_vscnt null, 0x01982; GFX11-NEXT:    buffer_gl1_inv1983; GFX11-NEXT:    buffer_gl0_inv1984; GFX11-NEXT:    s_endpgm1985entry:1986  %gep = getelementptr i32, ptr addrspace(1) %out, i64 41987  %val = atomicrmw volatile max ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01988  ret void1989}1990 1991define amdgpu_kernel void @atomic_max_i32_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) {1992; SI-LABEL: atomic_max_i32_ret_offset:1993; SI:       ; %bb.0: ; %entry1994; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x91995; SI-NEXT:    s_load_dword s8, s[4:5], 0xd1996; SI-NEXT:    s_mov_b32 s7, 0xf0001997; SI-NEXT:    s_mov_b32 s6, -11998; SI-NEXT:    s_waitcnt lgkmcnt(0)1999; SI-NEXT:    s_mov_b32 s4, s22000; SI-NEXT:    s_mov_b32 s5, s32001; SI-NEXT:    s_mov_b32 s2, s62002; SI-NEXT:    s_mov_b32 s3, s72003; SI-NEXT:    v_mov_b32_e32 v0, s82004; SI-NEXT:    buffer_atomic_smax v0, off, s[0:3], 0 offset:16 glc2005; SI-NEXT:    s_waitcnt vmcnt(0)2006; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 02007; SI-NEXT:    s_endpgm2008;2009; VI-LABEL: atomic_max_i32_ret_offset:2010; VI:       ; %bb.0: ; %entry2011; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242012; VI-NEXT:    s_load_dword s8, s[4:5], 0x342013; VI-NEXT:    s_mov_b32 s7, 0xf0002014; VI-NEXT:    s_mov_b32 s6, -12015; VI-NEXT:    s_waitcnt lgkmcnt(0)2016; VI-NEXT:    s_mov_b32 s4, s22017; VI-NEXT:    s_mov_b32 s5, s32018; VI-NEXT:    s_mov_b32 s2, s62019; VI-NEXT:    s_mov_b32 s3, s72020; VI-NEXT:    v_mov_b32_e32 v0, s82021; VI-NEXT:    buffer_atomic_smax v0, off, s[0:3], 0 offset:16 glc2022; VI-NEXT:    s_waitcnt vmcnt(0)2023; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 02024; VI-NEXT:    s_endpgm2025;2026; GFX9-LABEL: atomic_max_i32_ret_offset:2027; GFX9:       ; %bb.0: ; %entry2028; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x342029; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242030; GFX9-NEXT:    v_mov_b32_e32 v0, 02031; GFX9-NEXT:    s_waitcnt lgkmcnt(0)2032; GFX9-NEXT:    v_mov_b32_e32 v1, s62033; GFX9-NEXT:    global_atomic_smax v1, v0, v1, s[0:1] offset:16 glc2034; GFX9-NEXT:    s_waitcnt vmcnt(0)2035; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]2036; GFX9-NEXT:    s_endpgm2037;2038; GFX11-LABEL: atomic_max_i32_ret_offset:2039; GFX11:       ; %bb.0: ; %entry2040; GFX11-NEXT:    s_clause 0x12041; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x342042; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x242043; GFX11-NEXT:    s_waitcnt lgkmcnt(0)2044; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s62045; GFX11-NEXT:    global_atomic_max_i32 v1, v0, v1, s[0:1] offset:16 glc2046; GFX11-NEXT:    s_waitcnt vmcnt(0)2047; GFX11-NEXT:    buffer_gl0_inv2048; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]2049; GFX11-NEXT:    s_endpgm2050entry:2051  %gep = getelementptr i32, ptr addrspace(1) %out, i64 42052  %val = atomicrmw volatile max ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02053  store i32 %val, ptr addrspace(1) %out22054  ret void2055}2056 2057define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {2058; SI-LABEL: atomic_max_i32_addr64_offset:2059; SI:       ; %bb.0: ; %entry2060; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd2061; SI-NEXT:    s_load_dword s8, s[4:5], 0xb2062; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x92063; SI-NEXT:    s_mov_b32 s3, 0xf0002064; SI-NEXT:    s_waitcnt lgkmcnt(0)2065; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 22066; SI-NEXT:    s_mov_b32 s2, 02067; SI-NEXT:    v_mov_b32_e32 v2, s82068; SI-NEXT:    v_mov_b32_e32 v0, s42069; SI-NEXT:    v_mov_b32_e32 v1, s52070; SI-NEXT:    buffer_atomic_smax v2, v[0:1], s[0:3], 0 addr64 offset:162071; SI-NEXT:    s_endpgm2072;2073; VI-LABEL: atomic_max_i32_addr64_offset:2074; VI:       ; %bb.0: ; %entry2075; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x342076; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x242077; VI-NEXT:    s_load_dword s4, s[4:5], 0x2c2078; VI-NEXT:    s_waitcnt lgkmcnt(0)2079; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 22080; VI-NEXT:    s_add_u32 s0, s2, s02081; VI-NEXT:    s_addc_u32 s1, s3, s12082; VI-NEXT:    s_add_u32 s0, s0, 162083; VI-NEXT:    s_addc_u32 s1, s1, 02084; VI-NEXT:    v_mov_b32_e32 v0, s02085; VI-NEXT:    v_mov_b32_e32 v1, s12086; VI-NEXT:    v_mov_b32_e32 v2, s42087; VI-NEXT:    flat_atomic_smax v[0:1], v22088; VI-NEXT:    s_endpgm2089;2090; GFX9-LABEL: atomic_max_i32_addr64_offset:2091; GFX9:       ; %bb.0: ; %entry2092; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x342093; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x242094; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c2095; GFX9-NEXT:    v_mov_b32_e32 v0, 02096; GFX9-NEXT:    s_waitcnt lgkmcnt(0)2097; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 22098; GFX9-NEXT:    s_add_u32 s0, s2, s02099; GFX9-NEXT:    s_addc_u32 s1, s3, s12100; GFX9-NEXT:    v_mov_b32_e32 v1, s62101; GFX9-NEXT:    global_atomic_smax v0, v1, s[0:1] offset:162102; GFX9-NEXT:    s_endpgm2103;2104; GFX11-LABEL: atomic_max_i32_addr64_offset:2105; GFX11:       ; %bb.0: ; %entry2106; GFX11-NEXT:    s_clause 0x22107; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x342108; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c2109; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x242110; GFX11-NEXT:    v_mov_b32_e32 v0, 02111; GFX11-NEXT:    s_waitcnt lgkmcnt(0)2112; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 22113; GFX11-NEXT:    v_mov_b32_e32 v1, s62114; GFX11-NEXT:    s_add_u32 s0, s2, s02115; GFX11-NEXT:    s_addc_u32 s1, s3, s12116; GFX11-NEXT:    global_atomic_max_i32 v0, v1, s[0:1] offset:162117; GFX11-NEXT:    s_waitcnt_vscnt null, 0x02118; GFX11-NEXT:    buffer_gl0_inv2119; GFX11-NEXT:    s_endpgm2120entry:2121  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index2122  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 42123  %val = atomicrmw volatile max ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02124  ret void2125}2126 2127define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) {2128; SI-LABEL: atomic_max_i32_ret_addr64_offset:2129; SI:       ; %bb.0: ; %entry2130; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x92131; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf2132; SI-NEXT:    s_load_dword s6, s[4:5], 0xd2133; SI-NEXT:    s_mov_b32 s7, 0xf0002134; SI-NEXT:    s_waitcnt lgkmcnt(0)2135; SI-NEXT:    s_mov_b32 s4, s22136; SI-NEXT:    s_mov_b32 s5, s32137; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 22138; SI-NEXT:    s_mov_b32 s2, 02139; SI-NEXT:    s_mov_b32 s3, s72140; SI-NEXT:    v_mov_b32_e32 v2, s62141; SI-NEXT:    v_mov_b32_e32 v0, s82142; SI-NEXT:    v_mov_b32_e32 v1, s92143; SI-NEXT:    buffer_atomic_smax v2, v[0:1], s[0:3], 0 addr64 offset:16 glc2144; SI-NEXT:    s_mov_b32 s6, -12145; SI-NEXT:    s_waitcnt vmcnt(0)2146; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 02147; SI-NEXT:    s_endpgm2148;2149; VI-LABEL: atomic_max_i32_ret_addr64_offset:2150; VI:       ; %bb.0: ; %entry2151; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c2152; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242153; VI-NEXT:    s_load_dword s8, s[4:5], 0x342154; VI-NEXT:    s_waitcnt lgkmcnt(0)2155; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 22156; VI-NEXT:    s_add_u32 s0, s0, s42157; VI-NEXT:    s_addc_u32 s1, s1, s52158; VI-NEXT:    s_add_u32 s0, s0, 162159; VI-NEXT:    s_addc_u32 s1, s1, 02160; VI-NEXT:    v_mov_b32_e32 v0, s02161; VI-NEXT:    v_mov_b32_e32 v1, s12162; VI-NEXT:    v_mov_b32_e32 v2, s82163; VI-NEXT:    flat_atomic_smax v0, v[0:1], v2 glc2164; VI-NEXT:    s_mov_b32 s7, 0xf0002165; VI-NEXT:    s_mov_b32 s6, -12166; VI-NEXT:    s_mov_b32 s4, s22167; VI-NEXT:    s_mov_b32 s5, s32168; VI-NEXT:    s_waitcnt vmcnt(0)2169; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 02170; VI-NEXT:    s_endpgm2171;2172; GFX9-LABEL: atomic_max_i32_ret_addr64_offset:2173; GFX9:       ; %bb.0: ; %entry2174; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c2175; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242176; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x342177; GFX9-NEXT:    v_mov_b32_e32 v0, 02178; GFX9-NEXT:    s_waitcnt lgkmcnt(0)2179; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 22180; GFX9-NEXT:    s_add_u32 s0, s0, s42181; GFX9-NEXT:    s_addc_u32 s1, s1, s52182; GFX9-NEXT:    v_mov_b32_e32 v1, s82183; GFX9-NEXT:    global_atomic_smax v1, v0, v1, s[0:1] offset:16 glc2184; GFX9-NEXT:    s_waitcnt vmcnt(0)2185; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]2186; GFX9-NEXT:    s_endpgm2187;2188; GFX11-LABEL: atomic_max_i32_ret_addr64_offset:2189; GFX11:       ; %bb.0: ; %entry2190; GFX11-NEXT:    s_clause 0x22191; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c2192; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x342193; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x242194; GFX11-NEXT:    v_mov_b32_e32 v0, 02195; GFX11-NEXT:    s_waitcnt lgkmcnt(0)2196; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 22197; GFX11-NEXT:    v_mov_b32_e32 v1, s82198; GFX11-NEXT:    s_add_u32 s0, s0, s42199; GFX11-NEXT:    s_addc_u32 s1, s1, s52200; GFX11-NEXT:    global_atomic_max_i32 v1, v0, v1, s[0:1] offset:16 glc2201; GFX11-NEXT:    s_waitcnt vmcnt(0)2202; GFX11-NEXT:    buffer_gl0_inv2203; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]2204; GFX11-NEXT:    s_endpgm2205entry:2206  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index2207  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 42208  %val = atomicrmw volatile max ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02209  store i32 %val, ptr addrspace(1) %out22210  ret void2211}2212 2213define amdgpu_kernel void @atomic_max_i32(ptr addrspace(1) %out, i32 %in) {2214; SI-LABEL: atomic_max_i32:2215; SI:       ; %bb.0: ; %entry2216; SI-NEXT:    s_load_dword s6, s[4:5], 0xb2217; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x92218; SI-NEXT:    s_mov_b32 s3, 0xf0002219; SI-NEXT:    s_mov_b32 s2, -12220; SI-NEXT:    s_waitcnt lgkmcnt(0)2221; SI-NEXT:    v_mov_b32_e32 v0, s62222; SI-NEXT:    buffer_atomic_smax v0, off, s[0:3], 02223; SI-NEXT:    s_endpgm2224;2225; VI-LABEL: atomic_max_i32:2226; VI:       ; %bb.0: ; %entry2227; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c2228; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x242229; VI-NEXT:    s_mov_b32 s3, 0xf0002230; VI-NEXT:    s_mov_b32 s2, -12231; VI-NEXT:    s_waitcnt lgkmcnt(0)2232; VI-NEXT:    v_mov_b32_e32 v0, s62233; VI-NEXT:    buffer_atomic_smax v0, off, s[0:3], 02234; VI-NEXT:    s_endpgm2235;2236; GFX9-LABEL: atomic_max_i32:2237; GFX9:       ; %bb.0: ; %entry2238; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c2239; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x242240; GFX9-NEXT:    v_mov_b32_e32 v0, 02241; GFX9-NEXT:    s_waitcnt lgkmcnt(0)2242; GFX9-NEXT:    v_mov_b32_e32 v1, s22243; GFX9-NEXT:    global_atomic_smax v0, v1, s[0:1]2244; GFX9-NEXT:    s_endpgm2245;2246; GFX11-LABEL: atomic_max_i32:2247; GFX11:       ; %bb.0: ; %entry2248; GFX11-NEXT:    s_clause 0x12249; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c2250; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x242251; GFX11-NEXT:    s_waitcnt lgkmcnt(0)2252; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s22253; GFX11-NEXT:    global_atomic_max_i32 v0, v1, s[0:1]2254; GFX11-NEXT:    s_waitcnt_vscnt null, 0x02255; GFX11-NEXT:    buffer_gl0_inv2256; GFX11-NEXT:    s_endpgm2257entry:2258  %val = atomicrmw volatile max ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02259  ret void2260}2261 2262define amdgpu_kernel void @atomic_max_i32_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) {2263; SI-LABEL: atomic_max_i32_ret:2264; SI:       ; %bb.0: ; %entry2265; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x92266; SI-NEXT:    s_load_dword s8, s[4:5], 0xd2267; SI-NEXT:    s_mov_b32 s7, 0xf0002268; SI-NEXT:    s_mov_b32 s6, -12269; SI-NEXT:    s_waitcnt lgkmcnt(0)2270; SI-NEXT:    s_mov_b32 s4, s02271; SI-NEXT:    s_mov_b32 s5, s12272; SI-NEXT:    v_mov_b32_e32 v0, s82273; SI-NEXT:    buffer_atomic_smax v0, off, s[4:7], 0 glc2274; SI-NEXT:    s_mov_b32 s4, s22275; SI-NEXT:    s_mov_b32 s5, s32276; SI-NEXT:    s_waitcnt vmcnt(0)2277; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 02278; SI-NEXT:    s_endpgm2279;2280; VI-LABEL: atomic_max_i32_ret:2281; VI:       ; %bb.0: ; %entry2282; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242283; VI-NEXT:    s_load_dword s8, s[4:5], 0x342284; VI-NEXT:    s_mov_b32 s7, 0xf0002285; VI-NEXT:    s_mov_b32 s6, -12286; VI-NEXT:    s_waitcnt lgkmcnt(0)2287; VI-NEXT:    s_mov_b32 s4, s02288; VI-NEXT:    s_mov_b32 s5, s12289; VI-NEXT:    v_mov_b32_e32 v0, s82290; VI-NEXT:    buffer_atomic_smax v0, off, s[4:7], 0 glc2291; VI-NEXT:    s_mov_b32 s4, s22292; VI-NEXT:    s_mov_b32 s5, s32293; VI-NEXT:    s_waitcnt vmcnt(0)2294; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 02295; VI-NEXT:    s_endpgm2296;2297; GFX9-LABEL: atomic_max_i32_ret:2298; GFX9:       ; %bb.0: ; %entry2299; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x342300; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242301; GFX9-NEXT:    v_mov_b32_e32 v0, 02302; GFX9-NEXT:    s_waitcnt lgkmcnt(0)2303; GFX9-NEXT:    v_mov_b32_e32 v1, s62304; GFX9-NEXT:    global_atomic_smax v1, v0, v1, s[0:1] glc2305; GFX9-NEXT:    s_waitcnt vmcnt(0)2306; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]2307; GFX9-NEXT:    s_endpgm2308;2309; GFX11-LABEL: atomic_max_i32_ret:2310; GFX11:       ; %bb.0: ; %entry2311; GFX11-NEXT:    s_clause 0x12312; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x342313; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x242314; GFX11-NEXT:    s_waitcnt lgkmcnt(0)2315; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s62316; GFX11-NEXT:    global_atomic_max_i32 v1, v0, v1, s[0:1] glc2317; GFX11-NEXT:    s_waitcnt vmcnt(0)2318; GFX11-NEXT:    buffer_gl0_inv2319; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]2320; GFX11-NEXT:    s_endpgm2321entry:2322  %val = atomicrmw volatile max ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02323  store i32 %val, ptr addrspace(1) %out22324  ret void2325}2326 2327define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) {2328; SI-LABEL: atomic_max_i32_addr64:2329; SI:       ; %bb.0: ; %entry2330; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd2331; SI-NEXT:    s_load_dword s8, s[4:5], 0xb2332; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x92333; SI-NEXT:    s_mov_b32 s3, 0xf0002334; SI-NEXT:    s_waitcnt lgkmcnt(0)2335; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 22336; SI-NEXT:    s_mov_b32 s2, 02337; SI-NEXT:    v_mov_b32_e32 v2, s82338; SI-NEXT:    v_mov_b32_e32 v0, s42339; SI-NEXT:    v_mov_b32_e32 v1, s52340; SI-NEXT:    buffer_atomic_smax v2, v[0:1], s[0:3], 0 addr642341; SI-NEXT:    s_endpgm2342;2343; VI-LABEL: atomic_max_i32_addr64:2344; VI:       ; %bb.0: ; %entry2345; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x342346; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x242347; VI-NEXT:    s_load_dword s4, s[4:5], 0x2c2348; VI-NEXT:    s_waitcnt lgkmcnt(0)2349; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 22350; VI-NEXT:    s_add_u32 s0, s2, s02351; VI-NEXT:    s_addc_u32 s1, s3, s12352; VI-NEXT:    v_mov_b32_e32 v0, s02353; VI-NEXT:    v_mov_b32_e32 v1, s12354; VI-NEXT:    v_mov_b32_e32 v2, s42355; VI-NEXT:    flat_atomic_smax v[0:1], v22356; VI-NEXT:    s_endpgm2357;2358; GFX9-LABEL: atomic_max_i32_addr64:2359; GFX9:       ; %bb.0: ; %entry2360; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x342361; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x242362; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c2363; GFX9-NEXT:    v_mov_b32_e32 v0, 02364; GFX9-NEXT:    s_waitcnt lgkmcnt(0)2365; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 22366; GFX9-NEXT:    s_add_u32 s0, s2, s02367; GFX9-NEXT:    s_addc_u32 s1, s3, s12368; GFX9-NEXT:    v_mov_b32_e32 v1, s62369; GFX9-NEXT:    global_atomic_smax v0, v1, s[0:1]2370; GFX9-NEXT:    s_endpgm2371;2372; GFX11-LABEL: atomic_max_i32_addr64:2373; GFX11:       ; %bb.0: ; %entry2374; GFX11-NEXT:    s_clause 0x22375; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x342376; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c2377; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x242378; GFX11-NEXT:    v_mov_b32_e32 v0, 02379; GFX11-NEXT:    s_waitcnt lgkmcnt(0)2380; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 22381; GFX11-NEXT:    v_mov_b32_e32 v1, s62382; GFX11-NEXT:    s_add_u32 s0, s2, s02383; GFX11-NEXT:    s_addc_u32 s1, s3, s12384; GFX11-NEXT:    global_atomic_max_i32 v0, v1, s[0:1]2385; GFX11-NEXT:    s_waitcnt_vscnt null, 0x02386; GFX11-NEXT:    buffer_gl0_inv2387; GFX11-NEXT:    s_endpgm2388entry:2389  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index2390  %val = atomicrmw volatile max ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02391  ret void2392}2393 2394define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) {2395; SI-LABEL: atomic_max_i32_ret_addr64:2396; SI:       ; %bb.0: ; %entry2397; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x92398; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf2399; SI-NEXT:    s_load_dword s6, s[4:5], 0xd2400; SI-NEXT:    s_mov_b32 s7, 0xf0002401; SI-NEXT:    s_waitcnt lgkmcnt(0)2402; SI-NEXT:    s_mov_b32 s4, s22403; SI-NEXT:    s_mov_b32 s5, s32404; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 22405; SI-NEXT:    s_mov_b32 s2, 02406; SI-NEXT:    s_mov_b32 s3, s72407; SI-NEXT:    v_mov_b32_e32 v2, s62408; SI-NEXT:    v_mov_b32_e32 v0, s82409; SI-NEXT:    v_mov_b32_e32 v1, s92410; SI-NEXT:    buffer_atomic_smax v2, v[0:1], s[0:3], 0 addr64 glc2411; SI-NEXT:    s_mov_b32 s6, -12412; SI-NEXT:    s_waitcnt vmcnt(0)2413; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 02414; SI-NEXT:    s_endpgm2415;2416; VI-LABEL: atomic_max_i32_ret_addr64:2417; VI:       ; %bb.0: ; %entry2418; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c2419; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242420; VI-NEXT:    s_load_dword s8, s[4:5], 0x342421; VI-NEXT:    s_waitcnt lgkmcnt(0)2422; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 22423; VI-NEXT:    s_add_u32 s0, s0, s42424; VI-NEXT:    s_addc_u32 s1, s1, s52425; VI-NEXT:    v_mov_b32_e32 v0, s02426; VI-NEXT:    v_mov_b32_e32 v1, s12427; VI-NEXT:    v_mov_b32_e32 v2, s82428; VI-NEXT:    flat_atomic_smax v0, v[0:1], v2 glc2429; VI-NEXT:    s_mov_b32 s7, 0xf0002430; VI-NEXT:    s_mov_b32 s6, -12431; VI-NEXT:    s_mov_b32 s4, s22432; VI-NEXT:    s_mov_b32 s5, s32433; VI-NEXT:    s_waitcnt vmcnt(0)2434; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 02435; VI-NEXT:    s_endpgm2436;2437; GFX9-LABEL: atomic_max_i32_ret_addr64:2438; GFX9:       ; %bb.0: ; %entry2439; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c2440; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242441; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x342442; GFX9-NEXT:    v_mov_b32_e32 v0, 02443; GFX9-NEXT:    s_waitcnt lgkmcnt(0)2444; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 22445; GFX9-NEXT:    s_add_u32 s0, s0, s42446; GFX9-NEXT:    s_addc_u32 s1, s1, s52447; GFX9-NEXT:    v_mov_b32_e32 v1, s82448; GFX9-NEXT:    global_atomic_smax v1, v0, v1, s[0:1] glc2449; GFX9-NEXT:    s_waitcnt vmcnt(0)2450; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]2451; GFX9-NEXT:    s_endpgm2452;2453; GFX11-LABEL: atomic_max_i32_ret_addr64:2454; GFX11:       ; %bb.0: ; %entry2455; GFX11-NEXT:    s_clause 0x22456; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c2457; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x342458; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x242459; GFX11-NEXT:    v_mov_b32_e32 v0, 02460; GFX11-NEXT:    s_waitcnt lgkmcnt(0)2461; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 22462; GFX11-NEXT:    v_mov_b32_e32 v1, s82463; GFX11-NEXT:    s_add_u32 s0, s0, s42464; GFX11-NEXT:    s_addc_u32 s1, s1, s52465; GFX11-NEXT:    global_atomic_max_i32 v1, v0, v1, s[0:1] glc2466; GFX11-NEXT:    s_waitcnt vmcnt(0)2467; GFX11-NEXT:    buffer_gl0_inv2468; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]2469; GFX11-NEXT:    s_endpgm2470entry:2471  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index2472  %val = atomicrmw volatile max ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02473  store i32 %val, ptr addrspace(1) %out22474  ret void2475}2476 2477define amdgpu_kernel void @atomic_umax_i32_offset(ptr addrspace(1) %out, i32 %in) {2478; SI-LABEL: atomic_umax_i32_offset:2479; SI:       ; %bb.0: ; %entry2480; SI-NEXT:    s_load_dword s6, s[4:5], 0xb2481; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x92482; SI-NEXT:    s_mov_b32 s3, 0xf0002483; SI-NEXT:    s_mov_b32 s2, -12484; SI-NEXT:    s_waitcnt lgkmcnt(0)2485; SI-NEXT:    v_mov_b32_e32 v0, s62486; SI-NEXT:    buffer_atomic_umax v0, off, s[0:3], 0 offset:162487; SI-NEXT:    s_endpgm2488;2489; VI-LABEL: atomic_umax_i32_offset:2490; VI:       ; %bb.0: ; %entry2491; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c2492; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x242493; VI-NEXT:    s_mov_b32 s3, 0xf0002494; VI-NEXT:    s_mov_b32 s2, -12495; VI-NEXT:    s_waitcnt lgkmcnt(0)2496; VI-NEXT:    v_mov_b32_e32 v0, s62497; VI-NEXT:    buffer_atomic_umax v0, off, s[0:3], 0 offset:162498; VI-NEXT:    s_endpgm2499;2500; GFX9-LABEL: atomic_umax_i32_offset:2501; GFX9:       ; %bb.0: ; %entry2502; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c2503; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x242504; GFX9-NEXT:    v_mov_b32_e32 v0, 02505; GFX9-NEXT:    s_waitcnt lgkmcnt(0)2506; GFX9-NEXT:    v_mov_b32_e32 v1, s22507; GFX9-NEXT:    global_atomic_umax v0, v1, s[0:1] offset:162508; GFX9-NEXT:    s_endpgm2509;2510; GFX11-LABEL: atomic_umax_i32_offset:2511; GFX11:       ; %bb.0: ; %entry2512; GFX11-NEXT:    s_clause 0x12513; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c2514; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x242515; GFX11-NEXT:    s_waitcnt lgkmcnt(0)2516; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s22517; GFX11-NEXT:    global_atomic_max_u32 v0, v1, s[0:1] offset:162518; GFX11-NEXT:    s_waitcnt_vscnt null, 0x02519; GFX11-NEXT:    buffer_gl0_inv2520; GFX11-NEXT:    s_endpgm2521entry:2522  %gep = getelementptr i32, ptr addrspace(1) %out, i64 42523  %val = atomicrmw volatile umax ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02524  ret void2525}2526 2527define amdgpu_kernel void @atomic_umax_i32_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) {2528; SI-LABEL: atomic_umax_i32_ret_offset:2529; SI:       ; %bb.0: ; %entry2530; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x92531; SI-NEXT:    s_load_dword s8, s[4:5], 0xd2532; SI-NEXT:    s_mov_b32 s7, 0xf0002533; SI-NEXT:    s_mov_b32 s6, -12534; SI-NEXT:    s_waitcnt lgkmcnt(0)2535; SI-NEXT:    s_mov_b32 s4, s22536; SI-NEXT:    s_mov_b32 s5, s32537; SI-NEXT:    s_mov_b32 s2, s62538; SI-NEXT:    s_mov_b32 s3, s72539; SI-NEXT:    v_mov_b32_e32 v0, s82540; SI-NEXT:    buffer_atomic_umax v0, off, s[0:3], 0 offset:16 glc2541; SI-NEXT:    s_waitcnt vmcnt(0)2542; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 02543; SI-NEXT:    s_endpgm2544;2545; VI-LABEL: atomic_umax_i32_ret_offset:2546; VI:       ; %bb.0: ; %entry2547; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242548; VI-NEXT:    s_load_dword s8, s[4:5], 0x342549; VI-NEXT:    s_mov_b32 s7, 0xf0002550; VI-NEXT:    s_mov_b32 s6, -12551; VI-NEXT:    s_waitcnt lgkmcnt(0)2552; VI-NEXT:    s_mov_b32 s4, s22553; VI-NEXT:    s_mov_b32 s5, s32554; VI-NEXT:    s_mov_b32 s2, s62555; VI-NEXT:    s_mov_b32 s3, s72556; VI-NEXT:    v_mov_b32_e32 v0, s82557; VI-NEXT:    buffer_atomic_umax v0, off, s[0:3], 0 offset:16 glc2558; VI-NEXT:    s_waitcnt vmcnt(0)2559; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 02560; VI-NEXT:    s_endpgm2561;2562; GFX9-LABEL: atomic_umax_i32_ret_offset:2563; GFX9:       ; %bb.0: ; %entry2564; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x342565; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242566; GFX9-NEXT:    v_mov_b32_e32 v0, 02567; GFX9-NEXT:    s_waitcnt lgkmcnt(0)2568; GFX9-NEXT:    v_mov_b32_e32 v1, s62569; GFX9-NEXT:    global_atomic_umax v1, v0, v1, s[0:1] offset:16 glc2570; GFX9-NEXT:    s_waitcnt vmcnt(0)2571; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]2572; GFX9-NEXT:    s_endpgm2573;2574; GFX11-LABEL: atomic_umax_i32_ret_offset:2575; GFX11:       ; %bb.0: ; %entry2576; GFX11-NEXT:    s_clause 0x12577; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x342578; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x242579; GFX11-NEXT:    s_waitcnt lgkmcnt(0)2580; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s62581; GFX11-NEXT:    global_atomic_max_u32 v1, v0, v1, s[0:1] offset:16 glc2582; GFX11-NEXT:    s_waitcnt vmcnt(0)2583; GFX11-NEXT:    buffer_gl0_inv2584; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]2585; GFX11-NEXT:    s_endpgm2586entry:2587  %gep = getelementptr i32, ptr addrspace(1) %out, i64 42588  %val = atomicrmw volatile umax ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02589  store i32 %val, ptr addrspace(1) %out22590  ret void2591}2592 2593define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {2594; SI-LABEL: atomic_umax_i32_addr64_offset:2595; SI:       ; %bb.0: ; %entry2596; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd2597; SI-NEXT:    s_load_dword s8, s[4:5], 0xb2598; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x92599; SI-NEXT:    s_mov_b32 s3, 0xf0002600; SI-NEXT:    s_waitcnt lgkmcnt(0)2601; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 22602; SI-NEXT:    s_mov_b32 s2, 02603; SI-NEXT:    v_mov_b32_e32 v2, s82604; SI-NEXT:    v_mov_b32_e32 v0, s42605; SI-NEXT:    v_mov_b32_e32 v1, s52606; SI-NEXT:    buffer_atomic_umax v2, v[0:1], s[0:3], 0 addr64 offset:162607; SI-NEXT:    s_endpgm2608;2609; VI-LABEL: atomic_umax_i32_addr64_offset:2610; VI:       ; %bb.0: ; %entry2611; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x342612; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x242613; VI-NEXT:    s_load_dword s4, s[4:5], 0x2c2614; VI-NEXT:    s_waitcnt lgkmcnt(0)2615; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 22616; VI-NEXT:    s_add_u32 s0, s2, s02617; VI-NEXT:    s_addc_u32 s1, s3, s12618; VI-NEXT:    s_add_u32 s0, s0, 162619; VI-NEXT:    s_addc_u32 s1, s1, 02620; VI-NEXT:    v_mov_b32_e32 v0, s02621; VI-NEXT:    v_mov_b32_e32 v1, s12622; VI-NEXT:    v_mov_b32_e32 v2, s42623; VI-NEXT:    flat_atomic_umax v[0:1], v22624; VI-NEXT:    s_endpgm2625;2626; GFX9-LABEL: atomic_umax_i32_addr64_offset:2627; GFX9:       ; %bb.0: ; %entry2628; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x342629; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x242630; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c2631; GFX9-NEXT:    v_mov_b32_e32 v0, 02632; GFX9-NEXT:    s_waitcnt lgkmcnt(0)2633; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 22634; GFX9-NEXT:    s_add_u32 s0, s2, s02635; GFX9-NEXT:    s_addc_u32 s1, s3, s12636; GFX9-NEXT:    v_mov_b32_e32 v1, s62637; GFX9-NEXT:    global_atomic_umax v0, v1, s[0:1] offset:162638; GFX9-NEXT:    s_endpgm2639;2640; GFX11-LABEL: atomic_umax_i32_addr64_offset:2641; GFX11:       ; %bb.0: ; %entry2642; GFX11-NEXT:    s_clause 0x22643; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x342644; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c2645; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x242646; GFX11-NEXT:    v_mov_b32_e32 v0, 02647; GFX11-NEXT:    s_waitcnt lgkmcnt(0)2648; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 22649; GFX11-NEXT:    v_mov_b32_e32 v1, s62650; GFX11-NEXT:    s_add_u32 s0, s2, s02651; GFX11-NEXT:    s_addc_u32 s1, s3, s12652; GFX11-NEXT:    global_atomic_max_u32 v0, v1, s[0:1] offset:162653; GFX11-NEXT:    s_waitcnt_vscnt null, 0x02654; GFX11-NEXT:    buffer_gl0_inv2655; GFX11-NEXT:    s_endpgm2656entry:2657  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index2658  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 42659  %val = atomicrmw volatile umax ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02660  ret void2661}2662 2663define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) {2664; SI-LABEL: atomic_umax_i32_ret_addr64_offset:2665; SI:       ; %bb.0: ; %entry2666; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x92667; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf2668; SI-NEXT:    s_load_dword s6, s[4:5], 0xd2669; SI-NEXT:    s_mov_b32 s7, 0xf0002670; SI-NEXT:    s_waitcnt lgkmcnt(0)2671; SI-NEXT:    s_mov_b32 s4, s22672; SI-NEXT:    s_mov_b32 s5, s32673; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 22674; SI-NEXT:    s_mov_b32 s2, 02675; SI-NEXT:    s_mov_b32 s3, s72676; SI-NEXT:    v_mov_b32_e32 v2, s62677; SI-NEXT:    v_mov_b32_e32 v0, s82678; SI-NEXT:    v_mov_b32_e32 v1, s92679; SI-NEXT:    buffer_atomic_umax v2, v[0:1], s[0:3], 0 addr64 offset:16 glc2680; SI-NEXT:    s_mov_b32 s6, -12681; SI-NEXT:    s_waitcnt vmcnt(0)2682; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 02683; SI-NEXT:    s_endpgm2684;2685; VI-LABEL: atomic_umax_i32_ret_addr64_offset:2686; VI:       ; %bb.0: ; %entry2687; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c2688; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242689; VI-NEXT:    s_load_dword s8, s[4:5], 0x342690; VI-NEXT:    s_waitcnt lgkmcnt(0)2691; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 22692; VI-NEXT:    s_add_u32 s0, s0, s42693; VI-NEXT:    s_addc_u32 s1, s1, s52694; VI-NEXT:    s_add_u32 s0, s0, 162695; VI-NEXT:    s_addc_u32 s1, s1, 02696; VI-NEXT:    v_mov_b32_e32 v0, s02697; VI-NEXT:    v_mov_b32_e32 v1, s12698; VI-NEXT:    v_mov_b32_e32 v2, s82699; VI-NEXT:    flat_atomic_umax v0, v[0:1], v2 glc2700; VI-NEXT:    s_mov_b32 s7, 0xf0002701; VI-NEXT:    s_mov_b32 s6, -12702; VI-NEXT:    s_mov_b32 s4, s22703; VI-NEXT:    s_mov_b32 s5, s32704; VI-NEXT:    s_waitcnt vmcnt(0)2705; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 02706; VI-NEXT:    s_endpgm2707;2708; GFX9-LABEL: atomic_umax_i32_ret_addr64_offset:2709; GFX9:       ; %bb.0: ; %entry2710; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c2711; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242712; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x342713; GFX9-NEXT:    v_mov_b32_e32 v0, 02714; GFX9-NEXT:    s_waitcnt lgkmcnt(0)2715; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 22716; GFX9-NEXT:    s_add_u32 s0, s0, s42717; GFX9-NEXT:    s_addc_u32 s1, s1, s52718; GFX9-NEXT:    v_mov_b32_e32 v1, s82719; GFX9-NEXT:    global_atomic_umax v1, v0, v1, s[0:1] offset:16 glc2720; GFX9-NEXT:    s_waitcnt vmcnt(0)2721; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]2722; GFX9-NEXT:    s_endpgm2723;2724; GFX11-LABEL: atomic_umax_i32_ret_addr64_offset:2725; GFX11:       ; %bb.0: ; %entry2726; GFX11-NEXT:    s_clause 0x22727; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c2728; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x342729; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x242730; GFX11-NEXT:    v_mov_b32_e32 v0, 02731; GFX11-NEXT:    s_waitcnt lgkmcnt(0)2732; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 22733; GFX11-NEXT:    v_mov_b32_e32 v1, s82734; GFX11-NEXT:    s_add_u32 s0, s0, s42735; GFX11-NEXT:    s_addc_u32 s1, s1, s52736; GFX11-NEXT:    global_atomic_max_u32 v1, v0, v1, s[0:1] offset:16 glc2737; GFX11-NEXT:    s_waitcnt vmcnt(0)2738; GFX11-NEXT:    buffer_gl0_inv2739; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]2740; GFX11-NEXT:    s_endpgm2741entry:2742  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index2743  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 42744  %val = atomicrmw volatile umax ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02745  store i32 %val, ptr addrspace(1) %out22746  ret void2747}2748 2749define amdgpu_kernel void @atomic_umax_i32(ptr addrspace(1) %out, i32 %in) {2750; SI-LABEL: atomic_umax_i32:2751; SI:       ; %bb.0: ; %entry2752; SI-NEXT:    s_load_dword s6, s[4:5], 0xb2753; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x92754; SI-NEXT:    s_mov_b32 s3, 0xf0002755; SI-NEXT:    s_mov_b32 s2, -12756; SI-NEXT:    s_waitcnt lgkmcnt(0)2757; SI-NEXT:    v_mov_b32_e32 v0, s62758; SI-NEXT:    buffer_atomic_umax v0, off, s[0:3], 02759; SI-NEXT:    s_endpgm2760;2761; VI-LABEL: atomic_umax_i32:2762; VI:       ; %bb.0: ; %entry2763; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c2764; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x242765; VI-NEXT:    s_mov_b32 s3, 0xf0002766; VI-NEXT:    s_mov_b32 s2, -12767; VI-NEXT:    s_waitcnt lgkmcnt(0)2768; VI-NEXT:    v_mov_b32_e32 v0, s62769; VI-NEXT:    buffer_atomic_umax v0, off, s[0:3], 02770; VI-NEXT:    s_endpgm2771;2772; GFX9-LABEL: atomic_umax_i32:2773; GFX9:       ; %bb.0: ; %entry2774; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c2775; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x242776; GFX9-NEXT:    v_mov_b32_e32 v0, 02777; GFX9-NEXT:    s_waitcnt lgkmcnt(0)2778; GFX9-NEXT:    v_mov_b32_e32 v1, s22779; GFX9-NEXT:    global_atomic_umax v0, v1, s[0:1]2780; GFX9-NEXT:    s_endpgm2781;2782; GFX11-LABEL: atomic_umax_i32:2783; GFX11:       ; %bb.0: ; %entry2784; GFX11-NEXT:    s_clause 0x12785; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c2786; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x242787; GFX11-NEXT:    s_waitcnt lgkmcnt(0)2788; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s22789; GFX11-NEXT:    global_atomic_max_u32 v0, v1, s[0:1]2790; GFX11-NEXT:    s_waitcnt_vscnt null, 0x02791; GFX11-NEXT:    buffer_gl0_inv2792; GFX11-NEXT:    s_endpgm2793entry:2794  %val = atomicrmw volatile umax ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02795  ret void2796}2797 2798define amdgpu_kernel void @atomic_umax_i32_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) {2799; SI-LABEL: atomic_umax_i32_ret:2800; SI:       ; %bb.0: ; %entry2801; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x92802; SI-NEXT:    s_load_dword s8, s[4:5], 0xd2803; SI-NEXT:    s_mov_b32 s7, 0xf0002804; SI-NEXT:    s_mov_b32 s6, -12805; SI-NEXT:    s_waitcnt lgkmcnt(0)2806; SI-NEXT:    s_mov_b32 s4, s02807; SI-NEXT:    s_mov_b32 s5, s12808; SI-NEXT:    v_mov_b32_e32 v0, s82809; SI-NEXT:    buffer_atomic_umax v0, off, s[4:7], 0 glc2810; SI-NEXT:    s_mov_b32 s4, s22811; SI-NEXT:    s_mov_b32 s5, s32812; SI-NEXT:    s_waitcnt vmcnt(0)2813; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 02814; SI-NEXT:    s_endpgm2815;2816; VI-LABEL: atomic_umax_i32_ret:2817; VI:       ; %bb.0: ; %entry2818; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242819; VI-NEXT:    s_load_dword s8, s[4:5], 0x342820; VI-NEXT:    s_mov_b32 s7, 0xf0002821; VI-NEXT:    s_mov_b32 s6, -12822; VI-NEXT:    s_waitcnt lgkmcnt(0)2823; VI-NEXT:    s_mov_b32 s4, s02824; VI-NEXT:    s_mov_b32 s5, s12825; VI-NEXT:    v_mov_b32_e32 v0, s82826; VI-NEXT:    buffer_atomic_umax v0, off, s[4:7], 0 glc2827; VI-NEXT:    s_mov_b32 s4, s22828; VI-NEXT:    s_mov_b32 s5, s32829; VI-NEXT:    s_waitcnt vmcnt(0)2830; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 02831; VI-NEXT:    s_endpgm2832;2833; GFX9-LABEL: atomic_umax_i32_ret:2834; GFX9:       ; %bb.0: ; %entry2835; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x342836; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242837; GFX9-NEXT:    v_mov_b32_e32 v0, 02838; GFX9-NEXT:    s_waitcnt lgkmcnt(0)2839; GFX9-NEXT:    v_mov_b32_e32 v1, s62840; GFX9-NEXT:    global_atomic_umax v1, v0, v1, s[0:1] glc2841; GFX9-NEXT:    s_waitcnt vmcnt(0)2842; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]2843; GFX9-NEXT:    s_endpgm2844;2845; GFX11-LABEL: atomic_umax_i32_ret:2846; GFX11:       ; %bb.0: ; %entry2847; GFX11-NEXT:    s_clause 0x12848; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x342849; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x242850; GFX11-NEXT:    s_waitcnt lgkmcnt(0)2851; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s62852; GFX11-NEXT:    global_atomic_max_u32 v1, v0, v1, s[0:1] glc2853; GFX11-NEXT:    s_waitcnt vmcnt(0)2854; GFX11-NEXT:    buffer_gl0_inv2855; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]2856; GFX11-NEXT:    s_endpgm2857entry:2858  %val = atomicrmw volatile umax ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02859  store i32 %val, ptr addrspace(1) %out22860  ret void2861}2862 2863define amdgpu_kernel void @atomic_umax_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) {2864; SI-LABEL: atomic_umax_i32_addr64:2865; SI:       ; %bb.0: ; %entry2866; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd2867; SI-NEXT:    s_load_dword s8, s[4:5], 0xb2868; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x92869; SI-NEXT:    s_mov_b32 s3, 0xf0002870; SI-NEXT:    s_waitcnt lgkmcnt(0)2871; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 22872; SI-NEXT:    s_mov_b32 s2, 02873; SI-NEXT:    v_mov_b32_e32 v2, s82874; SI-NEXT:    v_mov_b32_e32 v0, s42875; SI-NEXT:    v_mov_b32_e32 v1, s52876; SI-NEXT:    buffer_atomic_umax v2, v[0:1], s[0:3], 0 addr642877; SI-NEXT:    s_endpgm2878;2879; VI-LABEL: atomic_umax_i32_addr64:2880; VI:       ; %bb.0: ; %entry2881; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x342882; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x242883; VI-NEXT:    s_load_dword s4, s[4:5], 0x2c2884; VI-NEXT:    s_waitcnt lgkmcnt(0)2885; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 22886; VI-NEXT:    s_add_u32 s0, s2, s02887; VI-NEXT:    s_addc_u32 s1, s3, s12888; VI-NEXT:    v_mov_b32_e32 v0, s02889; VI-NEXT:    v_mov_b32_e32 v1, s12890; VI-NEXT:    v_mov_b32_e32 v2, s42891; VI-NEXT:    flat_atomic_umax v[0:1], v22892; VI-NEXT:    s_endpgm2893;2894; GFX9-LABEL: atomic_umax_i32_addr64:2895; GFX9:       ; %bb.0: ; %entry2896; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x342897; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x242898; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c2899; GFX9-NEXT:    v_mov_b32_e32 v0, 02900; GFX9-NEXT:    s_waitcnt lgkmcnt(0)2901; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 22902; GFX9-NEXT:    s_add_u32 s0, s2, s02903; GFX9-NEXT:    s_addc_u32 s1, s3, s12904; GFX9-NEXT:    v_mov_b32_e32 v1, s62905; GFX9-NEXT:    global_atomic_umax v0, v1, s[0:1]2906; GFX9-NEXT:    s_endpgm2907;2908; GFX11-LABEL: atomic_umax_i32_addr64:2909; GFX11:       ; %bb.0: ; %entry2910; GFX11-NEXT:    s_clause 0x22911; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x342912; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c2913; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x242914; GFX11-NEXT:    v_mov_b32_e32 v0, 02915; GFX11-NEXT:    s_waitcnt lgkmcnt(0)2916; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 22917; GFX11-NEXT:    v_mov_b32_e32 v1, s62918; GFX11-NEXT:    s_add_u32 s0, s2, s02919; GFX11-NEXT:    s_addc_u32 s1, s3, s12920; GFX11-NEXT:    global_atomic_max_u32 v0, v1, s[0:1]2921; GFX11-NEXT:    s_waitcnt_vscnt null, 0x02922; GFX11-NEXT:    buffer_gl0_inv2923; GFX11-NEXT:    s_endpgm2924entry:2925  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index2926  %val = atomicrmw volatile umax ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02927  ret void2928}2929 2930define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) {2931; SI-LABEL: atomic_umax_i32_ret_addr64:2932; SI:       ; %bb.0: ; %entry2933; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x92934; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf2935; SI-NEXT:    s_load_dword s6, s[4:5], 0xd2936; SI-NEXT:    s_mov_b32 s7, 0xf0002937; SI-NEXT:    s_waitcnt lgkmcnt(0)2938; SI-NEXT:    s_mov_b32 s4, s22939; SI-NEXT:    s_mov_b32 s5, s32940; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 22941; SI-NEXT:    s_mov_b32 s2, 02942; SI-NEXT:    s_mov_b32 s3, s72943; SI-NEXT:    v_mov_b32_e32 v2, s62944; SI-NEXT:    v_mov_b32_e32 v0, s82945; SI-NEXT:    v_mov_b32_e32 v1, s92946; SI-NEXT:    buffer_atomic_umax v2, v[0:1], s[0:3], 0 addr64 glc2947; SI-NEXT:    s_mov_b32 s6, -12948; SI-NEXT:    s_waitcnt vmcnt(0)2949; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 02950; SI-NEXT:    s_endpgm2951;2952; VI-LABEL: atomic_umax_i32_ret_addr64:2953; VI:       ; %bb.0: ; %entry2954; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c2955; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242956; VI-NEXT:    s_load_dword s8, s[4:5], 0x342957; VI-NEXT:    s_waitcnt lgkmcnt(0)2958; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 22959; VI-NEXT:    s_add_u32 s0, s0, s42960; VI-NEXT:    s_addc_u32 s1, s1, s52961; VI-NEXT:    v_mov_b32_e32 v0, s02962; VI-NEXT:    v_mov_b32_e32 v1, s12963; VI-NEXT:    v_mov_b32_e32 v2, s82964; VI-NEXT:    flat_atomic_umax v0, v[0:1], v2 glc2965; VI-NEXT:    s_mov_b32 s7, 0xf0002966; VI-NEXT:    s_mov_b32 s6, -12967; VI-NEXT:    s_mov_b32 s4, s22968; VI-NEXT:    s_mov_b32 s5, s32969; VI-NEXT:    s_waitcnt vmcnt(0)2970; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 02971; VI-NEXT:    s_endpgm2972;2973; GFX9-LABEL: atomic_umax_i32_ret_addr64:2974; GFX9:       ; %bb.0: ; %entry2975; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c2976; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x242977; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x342978; GFX9-NEXT:    v_mov_b32_e32 v0, 02979; GFX9-NEXT:    s_waitcnt lgkmcnt(0)2980; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 22981; GFX9-NEXT:    s_add_u32 s0, s0, s42982; GFX9-NEXT:    s_addc_u32 s1, s1, s52983; GFX9-NEXT:    v_mov_b32_e32 v1, s82984; GFX9-NEXT:    global_atomic_umax v1, v0, v1, s[0:1] glc2985; GFX9-NEXT:    s_waitcnt vmcnt(0)2986; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]2987; GFX9-NEXT:    s_endpgm2988;2989; GFX11-LABEL: atomic_umax_i32_ret_addr64:2990; GFX11:       ; %bb.0: ; %entry2991; GFX11-NEXT:    s_clause 0x22992; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c2993; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x342994; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x242995; GFX11-NEXT:    v_mov_b32_e32 v0, 02996; GFX11-NEXT:    s_waitcnt lgkmcnt(0)2997; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 22998; GFX11-NEXT:    v_mov_b32_e32 v1, s82999; GFX11-NEXT:    s_add_u32 s0, s0, s43000; GFX11-NEXT:    s_addc_u32 s1, s1, s53001; GFX11-NEXT:    global_atomic_max_u32 v1, v0, v1, s[0:1] glc3002; GFX11-NEXT:    s_waitcnt vmcnt(0)3003; GFX11-NEXT:    buffer_gl0_inv3004; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]3005; GFX11-NEXT:    s_endpgm3006entry:3007  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index3008  %val = atomicrmw volatile umax ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03009  store i32 %val, ptr addrspace(1) %out23010  ret void3011}3012 3013define amdgpu_kernel void @atomic_min_i32_offset(ptr addrspace(1) %out, i32 %in) {3014; SI-LABEL: atomic_min_i32_offset:3015; SI:       ; %bb.0: ; %entry3016; SI-NEXT:    s_load_dword s6, s[4:5], 0xb3017; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x93018; SI-NEXT:    s_mov_b32 s3, 0xf0003019; SI-NEXT:    s_mov_b32 s2, -13020; SI-NEXT:    s_waitcnt lgkmcnt(0)3021; SI-NEXT:    v_mov_b32_e32 v0, s63022; SI-NEXT:    buffer_atomic_smin v0, off, s[0:3], 0 offset:163023; SI-NEXT:    s_endpgm3024;3025; VI-LABEL: atomic_min_i32_offset:3026; VI:       ; %bb.0: ; %entry3027; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c3028; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x243029; VI-NEXT:    s_mov_b32 s3, 0xf0003030; VI-NEXT:    s_mov_b32 s2, -13031; VI-NEXT:    s_waitcnt lgkmcnt(0)3032; VI-NEXT:    v_mov_b32_e32 v0, s63033; VI-NEXT:    buffer_atomic_smin v0, off, s[0:3], 0 offset:163034; VI-NEXT:    s_endpgm3035;3036; GFX9-LABEL: atomic_min_i32_offset:3037; GFX9:       ; %bb.0: ; %entry3038; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c3039; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x243040; GFX9-NEXT:    v_mov_b32_e32 v0, 03041; GFX9-NEXT:    s_waitcnt lgkmcnt(0)3042; GFX9-NEXT:    v_mov_b32_e32 v1, s23043; GFX9-NEXT:    global_atomic_smin v0, v1, s[0:1] offset:163044; GFX9-NEXT:    s_endpgm3045;3046; GFX11-LABEL: atomic_min_i32_offset:3047; GFX11:       ; %bb.0: ; %entry3048; GFX11-NEXT:    s_clause 0x13049; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c3050; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x243051; GFX11-NEXT:    s_waitcnt lgkmcnt(0)3052; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s23053; GFX11-NEXT:    global_atomic_min_i32 v0, v1, s[0:1] offset:163054; GFX11-NEXT:    s_waitcnt_vscnt null, 0x03055; GFX11-NEXT:    buffer_gl0_inv3056; GFX11-NEXT:    s_endpgm3057entry:3058  %gep = getelementptr i32, ptr addrspace(1) %out, i64 43059  %val = atomicrmw volatile min ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03060  ret void3061}3062 3063define amdgpu_kernel void @atomic_min_i32_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) {3064; SI-LABEL: atomic_min_i32_ret_offset:3065; SI:       ; %bb.0: ; %entry3066; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x93067; SI-NEXT:    s_load_dword s8, s[4:5], 0xd3068; SI-NEXT:    s_mov_b32 s7, 0xf0003069; SI-NEXT:    s_mov_b32 s6, -13070; SI-NEXT:    s_waitcnt lgkmcnt(0)3071; SI-NEXT:    s_mov_b32 s4, s23072; SI-NEXT:    s_mov_b32 s5, s33073; SI-NEXT:    s_mov_b32 s2, s63074; SI-NEXT:    s_mov_b32 s3, s73075; SI-NEXT:    v_mov_b32_e32 v0, s83076; SI-NEXT:    buffer_atomic_smin v0, off, s[0:3], 0 offset:16 glc3077; SI-NEXT:    s_waitcnt vmcnt(0)3078; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 03079; SI-NEXT:    s_endpgm3080;3081; VI-LABEL: atomic_min_i32_ret_offset:3082; VI:       ; %bb.0: ; %entry3083; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243084; VI-NEXT:    s_load_dword s8, s[4:5], 0x343085; VI-NEXT:    s_mov_b32 s7, 0xf0003086; VI-NEXT:    s_mov_b32 s6, -13087; VI-NEXT:    s_waitcnt lgkmcnt(0)3088; VI-NEXT:    s_mov_b32 s4, s23089; VI-NEXT:    s_mov_b32 s5, s33090; VI-NEXT:    s_mov_b32 s2, s63091; VI-NEXT:    s_mov_b32 s3, s73092; VI-NEXT:    v_mov_b32_e32 v0, s83093; VI-NEXT:    buffer_atomic_smin v0, off, s[0:3], 0 offset:16 glc3094; VI-NEXT:    s_waitcnt vmcnt(0)3095; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 03096; VI-NEXT:    s_endpgm3097;3098; GFX9-LABEL: atomic_min_i32_ret_offset:3099; GFX9:       ; %bb.0: ; %entry3100; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x343101; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243102; GFX9-NEXT:    v_mov_b32_e32 v0, 03103; GFX9-NEXT:    s_waitcnt lgkmcnt(0)3104; GFX9-NEXT:    v_mov_b32_e32 v1, s63105; GFX9-NEXT:    global_atomic_smin v1, v0, v1, s[0:1] offset:16 glc3106; GFX9-NEXT:    s_waitcnt vmcnt(0)3107; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]3108; GFX9-NEXT:    s_endpgm3109;3110; GFX11-LABEL: atomic_min_i32_ret_offset:3111; GFX11:       ; %bb.0: ; %entry3112; GFX11-NEXT:    s_clause 0x13113; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x343114; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x243115; GFX11-NEXT:    s_waitcnt lgkmcnt(0)3116; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s63117; GFX11-NEXT:    global_atomic_min_i32 v1, v0, v1, s[0:1] offset:16 glc3118; GFX11-NEXT:    s_waitcnt vmcnt(0)3119; GFX11-NEXT:    buffer_gl0_inv3120; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]3121; GFX11-NEXT:    s_endpgm3122entry:3123  %gep = getelementptr i32, ptr addrspace(1) %out, i64 43124  %val = atomicrmw volatile min ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03125  store i32 %val, ptr addrspace(1) %out23126  ret void3127}3128 3129define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {3130; SI-LABEL: atomic_min_i32_addr64_offset:3131; SI:       ; %bb.0: ; %entry3132; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd3133; SI-NEXT:    s_load_dword s8, s[4:5], 0xb3134; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x93135; SI-NEXT:    s_mov_b32 s3, 0xf0003136; SI-NEXT:    s_waitcnt lgkmcnt(0)3137; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 23138; SI-NEXT:    s_mov_b32 s2, 03139; SI-NEXT:    v_mov_b32_e32 v2, s83140; SI-NEXT:    v_mov_b32_e32 v0, s43141; SI-NEXT:    v_mov_b32_e32 v1, s53142; SI-NEXT:    buffer_atomic_smin v2, v[0:1], s[0:3], 0 addr64 offset:163143; SI-NEXT:    s_endpgm3144;3145; VI-LABEL: atomic_min_i32_addr64_offset:3146; VI:       ; %bb.0: ; %entry3147; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x343148; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x243149; VI-NEXT:    s_load_dword s4, s[4:5], 0x2c3150; VI-NEXT:    s_waitcnt lgkmcnt(0)3151; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 23152; VI-NEXT:    s_add_u32 s0, s2, s03153; VI-NEXT:    s_addc_u32 s1, s3, s13154; VI-NEXT:    s_add_u32 s0, s0, 163155; VI-NEXT:    s_addc_u32 s1, s1, 03156; VI-NEXT:    v_mov_b32_e32 v0, s03157; VI-NEXT:    v_mov_b32_e32 v1, s13158; VI-NEXT:    v_mov_b32_e32 v2, s43159; VI-NEXT:    flat_atomic_smin v[0:1], v23160; VI-NEXT:    s_endpgm3161;3162; GFX9-LABEL: atomic_min_i32_addr64_offset:3163; GFX9:       ; %bb.0: ; %entry3164; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x343165; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x243166; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c3167; GFX9-NEXT:    v_mov_b32_e32 v0, 03168; GFX9-NEXT:    s_waitcnt lgkmcnt(0)3169; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 23170; GFX9-NEXT:    s_add_u32 s0, s2, s03171; GFX9-NEXT:    s_addc_u32 s1, s3, s13172; GFX9-NEXT:    v_mov_b32_e32 v1, s63173; GFX9-NEXT:    global_atomic_smin v0, v1, s[0:1] offset:163174; GFX9-NEXT:    s_endpgm3175;3176; GFX11-LABEL: atomic_min_i32_addr64_offset:3177; GFX11:       ; %bb.0: ; %entry3178; GFX11-NEXT:    s_clause 0x23179; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x343180; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c3181; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x243182; GFX11-NEXT:    v_mov_b32_e32 v0, 03183; GFX11-NEXT:    s_waitcnt lgkmcnt(0)3184; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 23185; GFX11-NEXT:    v_mov_b32_e32 v1, s63186; GFX11-NEXT:    s_add_u32 s0, s2, s03187; GFX11-NEXT:    s_addc_u32 s1, s3, s13188; GFX11-NEXT:    global_atomic_min_i32 v0, v1, s[0:1] offset:163189; GFX11-NEXT:    s_waitcnt_vscnt null, 0x03190; GFX11-NEXT:    buffer_gl0_inv3191; GFX11-NEXT:    s_endpgm3192entry:3193  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index3194  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 43195  %val = atomicrmw volatile min ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03196  ret void3197}3198 3199define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) {3200; SI-LABEL: atomic_min_i32_ret_addr64_offset:3201; SI:       ; %bb.0: ; %entry3202; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x93203; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf3204; SI-NEXT:    s_load_dword s6, s[4:5], 0xd3205; SI-NEXT:    s_mov_b32 s7, 0xf0003206; SI-NEXT:    s_waitcnt lgkmcnt(0)3207; SI-NEXT:    s_mov_b32 s4, s23208; SI-NEXT:    s_mov_b32 s5, s33209; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 23210; SI-NEXT:    s_mov_b32 s2, 03211; SI-NEXT:    s_mov_b32 s3, s73212; SI-NEXT:    v_mov_b32_e32 v2, s63213; SI-NEXT:    v_mov_b32_e32 v0, s83214; SI-NEXT:    v_mov_b32_e32 v1, s93215; SI-NEXT:    buffer_atomic_smin v2, v[0:1], s[0:3], 0 addr64 offset:16 glc3216; SI-NEXT:    s_mov_b32 s6, -13217; SI-NEXT:    s_waitcnt vmcnt(0)3218; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 03219; SI-NEXT:    s_endpgm3220;3221; VI-LABEL: atomic_min_i32_ret_addr64_offset:3222; VI:       ; %bb.0: ; %entry3223; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c3224; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243225; VI-NEXT:    s_load_dword s8, s[4:5], 0x343226; VI-NEXT:    s_waitcnt lgkmcnt(0)3227; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 23228; VI-NEXT:    s_add_u32 s0, s0, s43229; VI-NEXT:    s_addc_u32 s1, s1, s53230; VI-NEXT:    s_add_u32 s0, s0, 163231; VI-NEXT:    s_addc_u32 s1, s1, 03232; VI-NEXT:    v_mov_b32_e32 v0, s03233; VI-NEXT:    v_mov_b32_e32 v1, s13234; VI-NEXT:    v_mov_b32_e32 v2, s83235; VI-NEXT:    flat_atomic_smin v0, v[0:1], v2 glc3236; VI-NEXT:    s_mov_b32 s7, 0xf0003237; VI-NEXT:    s_mov_b32 s6, -13238; VI-NEXT:    s_mov_b32 s4, s23239; VI-NEXT:    s_mov_b32 s5, s33240; VI-NEXT:    s_waitcnt vmcnt(0)3241; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 03242; VI-NEXT:    s_endpgm3243;3244; GFX9-LABEL: atomic_min_i32_ret_addr64_offset:3245; GFX9:       ; %bb.0: ; %entry3246; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c3247; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243248; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x343249; GFX9-NEXT:    v_mov_b32_e32 v0, 03250; GFX9-NEXT:    s_waitcnt lgkmcnt(0)3251; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 23252; GFX9-NEXT:    s_add_u32 s0, s0, s43253; GFX9-NEXT:    s_addc_u32 s1, s1, s53254; GFX9-NEXT:    v_mov_b32_e32 v1, s83255; GFX9-NEXT:    global_atomic_smin v1, v0, v1, s[0:1] offset:16 glc3256; GFX9-NEXT:    s_waitcnt vmcnt(0)3257; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]3258; GFX9-NEXT:    s_endpgm3259;3260; GFX11-LABEL: atomic_min_i32_ret_addr64_offset:3261; GFX11:       ; %bb.0: ; %entry3262; GFX11-NEXT:    s_clause 0x23263; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c3264; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x343265; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x243266; GFX11-NEXT:    v_mov_b32_e32 v0, 03267; GFX11-NEXT:    s_waitcnt lgkmcnt(0)3268; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 23269; GFX11-NEXT:    v_mov_b32_e32 v1, s83270; GFX11-NEXT:    s_add_u32 s0, s0, s43271; GFX11-NEXT:    s_addc_u32 s1, s1, s53272; GFX11-NEXT:    global_atomic_min_i32 v1, v0, v1, s[0:1] offset:16 glc3273; GFX11-NEXT:    s_waitcnt vmcnt(0)3274; GFX11-NEXT:    buffer_gl0_inv3275; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]3276; GFX11-NEXT:    s_endpgm3277entry:3278  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index3279  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 43280  %val = atomicrmw volatile min ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03281  store i32 %val, ptr addrspace(1) %out23282  ret void3283}3284 3285define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {3286; SI-LABEL: atomic_min_i32:3287; SI:       ; %bb.0: ; %entry3288; SI-NEXT:    s_load_dword s6, s[4:5], 0xb3289; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x93290; SI-NEXT:    s_mov_b32 s3, 0xf0003291; SI-NEXT:    s_mov_b32 s2, -13292; SI-NEXT:    s_waitcnt lgkmcnt(0)3293; SI-NEXT:    v_mov_b32_e32 v0, s63294; SI-NEXT:    buffer_atomic_smin v0, off, s[0:3], 03295; SI-NEXT:    s_endpgm3296;3297; VI-LABEL: atomic_min_i32:3298; VI:       ; %bb.0: ; %entry3299; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c3300; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x243301; VI-NEXT:    s_mov_b32 s3, 0xf0003302; VI-NEXT:    s_mov_b32 s2, -13303; VI-NEXT:    s_waitcnt lgkmcnt(0)3304; VI-NEXT:    v_mov_b32_e32 v0, s63305; VI-NEXT:    buffer_atomic_smin v0, off, s[0:3], 03306; VI-NEXT:    s_endpgm3307;3308; GFX9-LABEL: atomic_min_i32:3309; GFX9:       ; %bb.0: ; %entry3310; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c3311; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x243312; GFX9-NEXT:    v_mov_b32_e32 v0, 03313; GFX9-NEXT:    s_waitcnt lgkmcnt(0)3314; GFX9-NEXT:    v_mov_b32_e32 v1, s23315; GFX9-NEXT:    global_atomic_smin v0, v1, s[0:1]3316; GFX9-NEXT:    s_endpgm3317;3318; GFX11-LABEL: atomic_min_i32:3319; GFX11:       ; %bb.0: ; %entry3320; GFX11-NEXT:    s_clause 0x13321; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c3322; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x243323; GFX11-NEXT:    s_waitcnt lgkmcnt(0)3324; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s23325; GFX11-NEXT:    global_atomic_min_i32 v0, v1, s[0:1]3326; GFX11-NEXT:    s_waitcnt_vscnt null, 0x03327; GFX11-NEXT:    buffer_gl0_inv3328; GFX11-NEXT:    s_endpgm3329entry:3330  %val = atomicrmw volatile min ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03331  ret void3332}3333 3334define amdgpu_kernel void @atomic_min_i32_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) {3335; SI-LABEL: atomic_min_i32_ret:3336; SI:       ; %bb.0: ; %entry3337; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x93338; SI-NEXT:    s_load_dword s8, s[4:5], 0xd3339; SI-NEXT:    s_mov_b32 s7, 0xf0003340; SI-NEXT:    s_mov_b32 s6, -13341; SI-NEXT:    s_waitcnt lgkmcnt(0)3342; SI-NEXT:    s_mov_b32 s4, s03343; SI-NEXT:    s_mov_b32 s5, s13344; SI-NEXT:    v_mov_b32_e32 v0, s83345; SI-NEXT:    buffer_atomic_smin v0, off, s[4:7], 0 glc3346; SI-NEXT:    s_mov_b32 s4, s23347; SI-NEXT:    s_mov_b32 s5, s33348; SI-NEXT:    s_waitcnt vmcnt(0)3349; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 03350; SI-NEXT:    s_endpgm3351;3352; VI-LABEL: atomic_min_i32_ret:3353; VI:       ; %bb.0: ; %entry3354; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243355; VI-NEXT:    s_load_dword s8, s[4:5], 0x343356; VI-NEXT:    s_mov_b32 s7, 0xf0003357; VI-NEXT:    s_mov_b32 s6, -13358; VI-NEXT:    s_waitcnt lgkmcnt(0)3359; VI-NEXT:    s_mov_b32 s4, s03360; VI-NEXT:    s_mov_b32 s5, s13361; VI-NEXT:    v_mov_b32_e32 v0, s83362; VI-NEXT:    buffer_atomic_smin v0, off, s[4:7], 0 glc3363; VI-NEXT:    s_mov_b32 s4, s23364; VI-NEXT:    s_mov_b32 s5, s33365; VI-NEXT:    s_waitcnt vmcnt(0)3366; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 03367; VI-NEXT:    s_endpgm3368;3369; GFX9-LABEL: atomic_min_i32_ret:3370; GFX9:       ; %bb.0: ; %entry3371; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x343372; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243373; GFX9-NEXT:    v_mov_b32_e32 v0, 03374; GFX9-NEXT:    s_waitcnt lgkmcnt(0)3375; GFX9-NEXT:    v_mov_b32_e32 v1, s63376; GFX9-NEXT:    global_atomic_smin v1, v0, v1, s[0:1] glc3377; GFX9-NEXT:    s_waitcnt vmcnt(0)3378; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]3379; GFX9-NEXT:    s_endpgm3380;3381; GFX11-LABEL: atomic_min_i32_ret:3382; GFX11:       ; %bb.0: ; %entry3383; GFX11-NEXT:    s_clause 0x13384; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x343385; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x243386; GFX11-NEXT:    s_waitcnt lgkmcnt(0)3387; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s63388; GFX11-NEXT:    global_atomic_min_i32 v1, v0, v1, s[0:1] glc3389; GFX11-NEXT:    s_waitcnt vmcnt(0)3390; GFX11-NEXT:    buffer_gl0_inv3391; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]3392; GFX11-NEXT:    s_endpgm3393entry:3394  %val = atomicrmw volatile min ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03395  store i32 %val, ptr addrspace(1) %out23396  ret void3397}3398 3399define amdgpu_kernel void @atomic_min_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) {3400; SI-LABEL: atomic_min_i32_addr64:3401; SI:       ; %bb.0: ; %entry3402; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd3403; SI-NEXT:    s_load_dword s8, s[4:5], 0xb3404; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x93405; SI-NEXT:    s_mov_b32 s3, 0xf0003406; SI-NEXT:    s_waitcnt lgkmcnt(0)3407; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 23408; SI-NEXT:    s_mov_b32 s2, 03409; SI-NEXT:    v_mov_b32_e32 v2, s83410; SI-NEXT:    v_mov_b32_e32 v0, s43411; SI-NEXT:    v_mov_b32_e32 v1, s53412; SI-NEXT:    buffer_atomic_smin v2, v[0:1], s[0:3], 0 addr643413; SI-NEXT:    s_endpgm3414;3415; VI-LABEL: atomic_min_i32_addr64:3416; VI:       ; %bb.0: ; %entry3417; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x343418; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x243419; VI-NEXT:    s_load_dword s4, s[4:5], 0x2c3420; VI-NEXT:    s_waitcnt lgkmcnt(0)3421; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 23422; VI-NEXT:    s_add_u32 s0, s2, s03423; VI-NEXT:    s_addc_u32 s1, s3, s13424; VI-NEXT:    v_mov_b32_e32 v0, s03425; VI-NEXT:    v_mov_b32_e32 v1, s13426; VI-NEXT:    v_mov_b32_e32 v2, s43427; VI-NEXT:    flat_atomic_smin v[0:1], v23428; VI-NEXT:    s_endpgm3429;3430; GFX9-LABEL: atomic_min_i32_addr64:3431; GFX9:       ; %bb.0: ; %entry3432; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x343433; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x243434; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c3435; GFX9-NEXT:    v_mov_b32_e32 v0, 03436; GFX9-NEXT:    s_waitcnt lgkmcnt(0)3437; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 23438; GFX9-NEXT:    s_add_u32 s0, s2, s03439; GFX9-NEXT:    s_addc_u32 s1, s3, s13440; GFX9-NEXT:    v_mov_b32_e32 v1, s63441; GFX9-NEXT:    global_atomic_smin v0, v1, s[0:1]3442; GFX9-NEXT:    s_endpgm3443;3444; GFX11-LABEL: atomic_min_i32_addr64:3445; GFX11:       ; %bb.0: ; %entry3446; GFX11-NEXT:    s_clause 0x23447; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x343448; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c3449; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x243450; GFX11-NEXT:    v_mov_b32_e32 v0, 03451; GFX11-NEXT:    s_waitcnt lgkmcnt(0)3452; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 23453; GFX11-NEXT:    v_mov_b32_e32 v1, s63454; GFX11-NEXT:    s_add_u32 s0, s2, s03455; GFX11-NEXT:    s_addc_u32 s1, s3, s13456; GFX11-NEXT:    global_atomic_min_i32 v0, v1, s[0:1]3457; GFX11-NEXT:    s_waitcnt_vscnt null, 0x03458; GFX11-NEXT:    buffer_gl0_inv3459; GFX11-NEXT:    s_endpgm3460entry:3461  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index3462  %val = atomicrmw volatile min ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03463  ret void3464}3465 3466define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) {3467; SI-LABEL: atomic_min_i32_ret_addr64:3468; SI:       ; %bb.0: ; %entry3469; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x93470; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf3471; SI-NEXT:    s_load_dword s6, s[4:5], 0xd3472; SI-NEXT:    s_mov_b32 s7, 0xf0003473; SI-NEXT:    s_waitcnt lgkmcnt(0)3474; SI-NEXT:    s_mov_b32 s4, s23475; SI-NEXT:    s_mov_b32 s5, s33476; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 23477; SI-NEXT:    s_mov_b32 s2, 03478; SI-NEXT:    s_mov_b32 s3, s73479; SI-NEXT:    v_mov_b32_e32 v2, s63480; SI-NEXT:    v_mov_b32_e32 v0, s83481; SI-NEXT:    v_mov_b32_e32 v1, s93482; SI-NEXT:    buffer_atomic_smin v2, v[0:1], s[0:3], 0 addr64 glc3483; SI-NEXT:    s_mov_b32 s6, -13484; SI-NEXT:    s_waitcnt vmcnt(0)3485; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 03486; SI-NEXT:    s_endpgm3487;3488; VI-LABEL: atomic_min_i32_ret_addr64:3489; VI:       ; %bb.0: ; %entry3490; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c3491; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243492; VI-NEXT:    s_load_dword s8, s[4:5], 0x343493; VI-NEXT:    s_waitcnt lgkmcnt(0)3494; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 23495; VI-NEXT:    s_add_u32 s0, s0, s43496; VI-NEXT:    s_addc_u32 s1, s1, s53497; VI-NEXT:    v_mov_b32_e32 v0, s03498; VI-NEXT:    v_mov_b32_e32 v1, s13499; VI-NEXT:    v_mov_b32_e32 v2, s83500; VI-NEXT:    flat_atomic_smin v0, v[0:1], v2 glc3501; VI-NEXT:    s_mov_b32 s7, 0xf0003502; VI-NEXT:    s_mov_b32 s6, -13503; VI-NEXT:    s_mov_b32 s4, s23504; VI-NEXT:    s_mov_b32 s5, s33505; VI-NEXT:    s_waitcnt vmcnt(0)3506; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 03507; VI-NEXT:    s_endpgm3508;3509; GFX9-LABEL: atomic_min_i32_ret_addr64:3510; GFX9:       ; %bb.0: ; %entry3511; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c3512; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243513; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x343514; GFX9-NEXT:    v_mov_b32_e32 v0, 03515; GFX9-NEXT:    s_waitcnt lgkmcnt(0)3516; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 23517; GFX9-NEXT:    s_add_u32 s0, s0, s43518; GFX9-NEXT:    s_addc_u32 s1, s1, s53519; GFX9-NEXT:    v_mov_b32_e32 v1, s83520; GFX9-NEXT:    global_atomic_smin v1, v0, v1, s[0:1] glc3521; GFX9-NEXT:    s_waitcnt vmcnt(0)3522; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]3523; GFX9-NEXT:    s_endpgm3524;3525; GFX11-LABEL: atomic_min_i32_ret_addr64:3526; GFX11:       ; %bb.0: ; %entry3527; GFX11-NEXT:    s_clause 0x23528; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c3529; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x343530; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x243531; GFX11-NEXT:    v_mov_b32_e32 v0, 03532; GFX11-NEXT:    s_waitcnt lgkmcnt(0)3533; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 23534; GFX11-NEXT:    v_mov_b32_e32 v1, s83535; GFX11-NEXT:    s_add_u32 s0, s0, s43536; GFX11-NEXT:    s_addc_u32 s1, s1, s53537; GFX11-NEXT:    global_atomic_min_i32 v1, v0, v1, s[0:1] glc3538; GFX11-NEXT:    s_waitcnt vmcnt(0)3539; GFX11-NEXT:    buffer_gl0_inv3540; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]3541; GFX11-NEXT:    s_endpgm3542entry:3543  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index3544  %val = atomicrmw volatile min ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03545  store i32 %val, ptr addrspace(1) %out23546  ret void3547}3548 3549define amdgpu_kernel void @atomic_umin_i32_offset(ptr addrspace(1) %out, i32 %in) {3550; SI-LABEL: atomic_umin_i32_offset:3551; SI:       ; %bb.0: ; %entry3552; SI-NEXT:    s_load_dword s6, s[4:5], 0xb3553; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x93554; SI-NEXT:    s_mov_b32 s3, 0xf0003555; SI-NEXT:    s_mov_b32 s2, -13556; SI-NEXT:    s_waitcnt lgkmcnt(0)3557; SI-NEXT:    v_mov_b32_e32 v0, s63558; SI-NEXT:    buffer_atomic_umin v0, off, s[0:3], 0 offset:163559; SI-NEXT:    s_endpgm3560;3561; VI-LABEL: atomic_umin_i32_offset:3562; VI:       ; %bb.0: ; %entry3563; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c3564; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x243565; VI-NEXT:    s_mov_b32 s3, 0xf0003566; VI-NEXT:    s_mov_b32 s2, -13567; VI-NEXT:    s_waitcnt lgkmcnt(0)3568; VI-NEXT:    v_mov_b32_e32 v0, s63569; VI-NEXT:    buffer_atomic_umin v0, off, s[0:3], 0 offset:163570; VI-NEXT:    s_endpgm3571;3572; GFX9-LABEL: atomic_umin_i32_offset:3573; GFX9:       ; %bb.0: ; %entry3574; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c3575; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x243576; GFX9-NEXT:    v_mov_b32_e32 v0, 03577; GFX9-NEXT:    s_waitcnt lgkmcnt(0)3578; GFX9-NEXT:    v_mov_b32_e32 v1, s23579; GFX9-NEXT:    global_atomic_umin v0, v1, s[0:1] offset:163580; GFX9-NEXT:    s_endpgm3581;3582; GFX11-LABEL: atomic_umin_i32_offset:3583; GFX11:       ; %bb.0: ; %entry3584; GFX11-NEXT:    s_clause 0x13585; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c3586; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x243587; GFX11-NEXT:    s_waitcnt lgkmcnt(0)3588; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s23589; GFX11-NEXT:    global_atomic_min_u32 v0, v1, s[0:1] offset:163590; GFX11-NEXT:    s_waitcnt_vscnt null, 0x03591; GFX11-NEXT:    buffer_gl0_inv3592; GFX11-NEXT:    s_endpgm3593entry:3594  %gep = getelementptr i32, ptr addrspace(1) %out, i64 43595  %val = atomicrmw volatile umin ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03596  ret void3597}3598 3599define amdgpu_kernel void @atomic_umin_i32_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) {3600; SI-LABEL: atomic_umin_i32_ret_offset:3601; SI:       ; %bb.0: ; %entry3602; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x93603; SI-NEXT:    s_load_dword s8, s[4:5], 0xd3604; SI-NEXT:    s_mov_b32 s7, 0xf0003605; SI-NEXT:    s_mov_b32 s6, -13606; SI-NEXT:    s_waitcnt lgkmcnt(0)3607; SI-NEXT:    s_mov_b32 s4, s23608; SI-NEXT:    s_mov_b32 s5, s33609; SI-NEXT:    s_mov_b32 s2, s63610; SI-NEXT:    s_mov_b32 s3, s73611; SI-NEXT:    v_mov_b32_e32 v0, s83612; SI-NEXT:    buffer_atomic_umin v0, off, s[0:3], 0 offset:16 glc3613; SI-NEXT:    s_waitcnt vmcnt(0)3614; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 03615; SI-NEXT:    s_endpgm3616;3617; VI-LABEL: atomic_umin_i32_ret_offset:3618; VI:       ; %bb.0: ; %entry3619; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243620; VI-NEXT:    s_load_dword s8, s[4:5], 0x343621; VI-NEXT:    s_mov_b32 s7, 0xf0003622; VI-NEXT:    s_mov_b32 s6, -13623; VI-NEXT:    s_waitcnt lgkmcnt(0)3624; VI-NEXT:    s_mov_b32 s4, s23625; VI-NEXT:    s_mov_b32 s5, s33626; VI-NEXT:    s_mov_b32 s2, s63627; VI-NEXT:    s_mov_b32 s3, s73628; VI-NEXT:    v_mov_b32_e32 v0, s83629; VI-NEXT:    buffer_atomic_umin v0, off, s[0:3], 0 offset:16 glc3630; VI-NEXT:    s_waitcnt vmcnt(0)3631; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 03632; VI-NEXT:    s_endpgm3633;3634; GFX9-LABEL: atomic_umin_i32_ret_offset:3635; GFX9:       ; %bb.0: ; %entry3636; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x343637; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243638; GFX9-NEXT:    v_mov_b32_e32 v0, 03639; GFX9-NEXT:    s_waitcnt lgkmcnt(0)3640; GFX9-NEXT:    v_mov_b32_e32 v1, s63641; GFX9-NEXT:    global_atomic_umin v1, v0, v1, s[0:1] offset:16 glc3642; GFX9-NEXT:    s_waitcnt vmcnt(0)3643; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]3644; GFX9-NEXT:    s_endpgm3645;3646; GFX11-LABEL: atomic_umin_i32_ret_offset:3647; GFX11:       ; %bb.0: ; %entry3648; GFX11-NEXT:    s_clause 0x13649; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x343650; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x243651; GFX11-NEXT:    s_waitcnt lgkmcnt(0)3652; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s63653; GFX11-NEXT:    global_atomic_min_u32 v1, v0, v1, s[0:1] offset:16 glc3654; GFX11-NEXT:    s_waitcnt vmcnt(0)3655; GFX11-NEXT:    buffer_gl0_inv3656; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]3657; GFX11-NEXT:    s_endpgm3658entry:3659  %gep = getelementptr i32, ptr addrspace(1) %out, i64 43660  %val = atomicrmw volatile umin ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03661  store i32 %val, ptr addrspace(1) %out23662  ret void3663}3664 3665define amdgpu_kernel void @atomic_umin_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {3666; SI-LABEL: atomic_umin_i32_addr64_offset:3667; SI:       ; %bb.0: ; %entry3668; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd3669; SI-NEXT:    s_load_dword s8, s[4:5], 0xb3670; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x93671; SI-NEXT:    s_mov_b32 s3, 0xf0003672; SI-NEXT:    s_waitcnt lgkmcnt(0)3673; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 23674; SI-NEXT:    s_mov_b32 s2, 03675; SI-NEXT:    v_mov_b32_e32 v2, s83676; SI-NEXT:    v_mov_b32_e32 v0, s43677; SI-NEXT:    v_mov_b32_e32 v1, s53678; SI-NEXT:    buffer_atomic_umin v2, v[0:1], s[0:3], 0 addr64 offset:163679; SI-NEXT:    s_endpgm3680;3681; VI-LABEL: atomic_umin_i32_addr64_offset:3682; VI:       ; %bb.0: ; %entry3683; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x343684; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x243685; VI-NEXT:    s_load_dword s4, s[4:5], 0x2c3686; VI-NEXT:    s_waitcnt lgkmcnt(0)3687; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 23688; VI-NEXT:    s_add_u32 s0, s2, s03689; VI-NEXT:    s_addc_u32 s1, s3, s13690; VI-NEXT:    s_add_u32 s0, s0, 163691; VI-NEXT:    s_addc_u32 s1, s1, 03692; VI-NEXT:    v_mov_b32_e32 v0, s03693; VI-NEXT:    v_mov_b32_e32 v1, s13694; VI-NEXT:    v_mov_b32_e32 v2, s43695; VI-NEXT:    flat_atomic_umin v[0:1], v23696; VI-NEXT:    s_endpgm3697;3698; GFX9-LABEL: atomic_umin_i32_addr64_offset:3699; GFX9:       ; %bb.0: ; %entry3700; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x343701; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x243702; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c3703; GFX9-NEXT:    v_mov_b32_e32 v0, 03704; GFX9-NEXT:    s_waitcnt lgkmcnt(0)3705; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 23706; GFX9-NEXT:    s_add_u32 s0, s2, s03707; GFX9-NEXT:    s_addc_u32 s1, s3, s13708; GFX9-NEXT:    v_mov_b32_e32 v1, s63709; GFX9-NEXT:    global_atomic_umin v0, v1, s[0:1] offset:163710; GFX9-NEXT:    s_endpgm3711;3712; GFX11-LABEL: atomic_umin_i32_addr64_offset:3713; GFX11:       ; %bb.0: ; %entry3714; GFX11-NEXT:    s_clause 0x23715; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x343716; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c3717; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x243718; GFX11-NEXT:    v_mov_b32_e32 v0, 03719; GFX11-NEXT:    s_waitcnt lgkmcnt(0)3720; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 23721; GFX11-NEXT:    v_mov_b32_e32 v1, s63722; GFX11-NEXT:    s_add_u32 s0, s2, s03723; GFX11-NEXT:    s_addc_u32 s1, s3, s13724; GFX11-NEXT:    global_atomic_min_u32 v0, v1, s[0:1] offset:163725; GFX11-NEXT:    s_waitcnt_vscnt null, 0x03726; GFX11-NEXT:    buffer_gl0_inv3727; GFX11-NEXT:    s_endpgm3728entry:3729  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index3730  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 43731  %val = atomicrmw volatile umin ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03732  ret void3733}3734 3735define amdgpu_kernel void @atomic_umin_i32_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) {3736; SI-LABEL: atomic_umin_i32_ret_addr64_offset:3737; SI:       ; %bb.0: ; %entry3738; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x93739; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf3740; SI-NEXT:    s_load_dword s6, s[4:5], 0xd3741; SI-NEXT:    s_mov_b32 s7, 0xf0003742; SI-NEXT:    s_waitcnt lgkmcnt(0)3743; SI-NEXT:    s_mov_b32 s4, s23744; SI-NEXT:    s_mov_b32 s5, s33745; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 23746; SI-NEXT:    s_mov_b32 s2, 03747; SI-NEXT:    s_mov_b32 s3, s73748; SI-NEXT:    v_mov_b32_e32 v2, s63749; SI-NEXT:    v_mov_b32_e32 v0, s83750; SI-NEXT:    v_mov_b32_e32 v1, s93751; SI-NEXT:    buffer_atomic_umin v2, v[0:1], s[0:3], 0 addr64 offset:16 glc3752; SI-NEXT:    s_mov_b32 s6, -13753; SI-NEXT:    s_waitcnt vmcnt(0)3754; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 03755; SI-NEXT:    s_endpgm3756;3757; VI-LABEL: atomic_umin_i32_ret_addr64_offset:3758; VI:       ; %bb.0: ; %entry3759; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c3760; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243761; VI-NEXT:    s_load_dword s8, s[4:5], 0x343762; VI-NEXT:    s_waitcnt lgkmcnt(0)3763; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 23764; VI-NEXT:    s_add_u32 s0, s0, s43765; VI-NEXT:    s_addc_u32 s1, s1, s53766; VI-NEXT:    s_add_u32 s0, s0, 163767; VI-NEXT:    s_addc_u32 s1, s1, 03768; VI-NEXT:    v_mov_b32_e32 v0, s03769; VI-NEXT:    v_mov_b32_e32 v1, s13770; VI-NEXT:    v_mov_b32_e32 v2, s83771; VI-NEXT:    flat_atomic_umin v0, v[0:1], v2 glc3772; VI-NEXT:    s_mov_b32 s7, 0xf0003773; VI-NEXT:    s_mov_b32 s6, -13774; VI-NEXT:    s_mov_b32 s4, s23775; VI-NEXT:    s_mov_b32 s5, s33776; VI-NEXT:    s_waitcnt vmcnt(0)3777; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 03778; VI-NEXT:    s_endpgm3779;3780; GFX9-LABEL: atomic_umin_i32_ret_addr64_offset:3781; GFX9:       ; %bb.0: ; %entry3782; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c3783; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243784; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x343785; GFX9-NEXT:    v_mov_b32_e32 v0, 03786; GFX9-NEXT:    s_waitcnt lgkmcnt(0)3787; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 23788; GFX9-NEXT:    s_add_u32 s0, s0, s43789; GFX9-NEXT:    s_addc_u32 s1, s1, s53790; GFX9-NEXT:    v_mov_b32_e32 v1, s83791; GFX9-NEXT:    global_atomic_umin v1, v0, v1, s[0:1] offset:16 glc3792; GFX9-NEXT:    s_waitcnt vmcnt(0)3793; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]3794; GFX9-NEXT:    s_endpgm3795;3796; GFX11-LABEL: atomic_umin_i32_ret_addr64_offset:3797; GFX11:       ; %bb.0: ; %entry3798; GFX11-NEXT:    s_clause 0x23799; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c3800; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x343801; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x243802; GFX11-NEXT:    v_mov_b32_e32 v0, 03803; GFX11-NEXT:    s_waitcnt lgkmcnt(0)3804; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 23805; GFX11-NEXT:    v_mov_b32_e32 v1, s83806; GFX11-NEXT:    s_add_u32 s0, s0, s43807; GFX11-NEXT:    s_addc_u32 s1, s1, s53808; GFX11-NEXT:    global_atomic_min_u32 v1, v0, v1, s[0:1] offset:16 glc3809; GFX11-NEXT:    s_waitcnt vmcnt(0)3810; GFX11-NEXT:    buffer_gl0_inv3811; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]3812; GFX11-NEXT:    s_endpgm3813entry:3814  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index3815  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 43816  %val = atomicrmw volatile umin ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03817  store i32 %val, ptr addrspace(1) %out23818  ret void3819}3820 3821define amdgpu_kernel void @atomic_umin_i32(ptr addrspace(1) %out, i32 %in) {3822; SI-LABEL: atomic_umin_i32:3823; SI:       ; %bb.0: ; %entry3824; SI-NEXT:    s_load_dword s6, s[4:5], 0xb3825; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x93826; SI-NEXT:    s_mov_b32 s3, 0xf0003827; SI-NEXT:    s_mov_b32 s2, -13828; SI-NEXT:    s_waitcnt lgkmcnt(0)3829; SI-NEXT:    v_mov_b32_e32 v0, s63830; SI-NEXT:    buffer_atomic_umin v0, off, s[0:3], 03831; SI-NEXT:    s_endpgm3832;3833; VI-LABEL: atomic_umin_i32:3834; VI:       ; %bb.0: ; %entry3835; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c3836; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x243837; VI-NEXT:    s_mov_b32 s3, 0xf0003838; VI-NEXT:    s_mov_b32 s2, -13839; VI-NEXT:    s_waitcnt lgkmcnt(0)3840; VI-NEXT:    v_mov_b32_e32 v0, s63841; VI-NEXT:    buffer_atomic_umin v0, off, s[0:3], 03842; VI-NEXT:    s_endpgm3843;3844; GFX9-LABEL: atomic_umin_i32:3845; GFX9:       ; %bb.0: ; %entry3846; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c3847; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x243848; GFX9-NEXT:    v_mov_b32_e32 v0, 03849; GFX9-NEXT:    s_waitcnt lgkmcnt(0)3850; GFX9-NEXT:    v_mov_b32_e32 v1, s23851; GFX9-NEXT:    global_atomic_umin v0, v1, s[0:1]3852; GFX9-NEXT:    s_endpgm3853;3854; GFX11-LABEL: atomic_umin_i32:3855; GFX11:       ; %bb.0: ; %entry3856; GFX11-NEXT:    s_clause 0x13857; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c3858; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x243859; GFX11-NEXT:    s_waitcnt lgkmcnt(0)3860; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s23861; GFX11-NEXT:    global_atomic_min_u32 v0, v1, s[0:1]3862; GFX11-NEXT:    s_waitcnt_vscnt null, 0x03863; GFX11-NEXT:    buffer_gl0_inv3864; GFX11-NEXT:    s_endpgm3865entry:3866  %val = atomicrmw volatile umin ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03867  ret void3868}3869 3870define amdgpu_kernel void @atomic_umin_i32_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) {3871; SI-LABEL: atomic_umin_i32_ret:3872; SI:       ; %bb.0: ; %entry3873; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x93874; SI-NEXT:    s_load_dword s8, s[4:5], 0xd3875; SI-NEXT:    s_mov_b32 s7, 0xf0003876; SI-NEXT:    s_mov_b32 s6, -13877; SI-NEXT:    s_waitcnt lgkmcnt(0)3878; SI-NEXT:    s_mov_b32 s4, s03879; SI-NEXT:    s_mov_b32 s5, s13880; SI-NEXT:    v_mov_b32_e32 v0, s83881; SI-NEXT:    buffer_atomic_umin v0, off, s[4:7], 0 glc3882; SI-NEXT:    s_mov_b32 s4, s23883; SI-NEXT:    s_mov_b32 s5, s33884; SI-NEXT:    s_waitcnt vmcnt(0)3885; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 03886; SI-NEXT:    s_endpgm3887;3888; VI-LABEL: atomic_umin_i32_ret:3889; VI:       ; %bb.0: ; %entry3890; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243891; VI-NEXT:    s_load_dword s8, s[4:5], 0x343892; VI-NEXT:    s_mov_b32 s7, 0xf0003893; VI-NEXT:    s_mov_b32 s6, -13894; VI-NEXT:    s_waitcnt lgkmcnt(0)3895; VI-NEXT:    s_mov_b32 s4, s03896; VI-NEXT:    s_mov_b32 s5, s13897; VI-NEXT:    v_mov_b32_e32 v0, s83898; VI-NEXT:    buffer_atomic_umin v0, off, s[4:7], 0 glc3899; VI-NEXT:    s_mov_b32 s4, s23900; VI-NEXT:    s_mov_b32 s5, s33901; VI-NEXT:    s_waitcnt vmcnt(0)3902; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 03903; VI-NEXT:    s_endpgm3904;3905; GFX9-LABEL: atomic_umin_i32_ret:3906; GFX9:       ; %bb.0: ; %entry3907; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x343908; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x243909; GFX9-NEXT:    v_mov_b32_e32 v0, 03910; GFX9-NEXT:    s_waitcnt lgkmcnt(0)3911; GFX9-NEXT:    v_mov_b32_e32 v1, s63912; GFX9-NEXT:    global_atomic_umin v1, v0, v1, s[0:1] glc3913; GFX9-NEXT:    s_waitcnt vmcnt(0)3914; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]3915; GFX9-NEXT:    s_endpgm3916;3917; GFX11-LABEL: atomic_umin_i32_ret:3918; GFX11:       ; %bb.0: ; %entry3919; GFX11-NEXT:    s_clause 0x13920; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x343921; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x243922; GFX11-NEXT:    s_waitcnt lgkmcnt(0)3923; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s63924; GFX11-NEXT:    global_atomic_min_u32 v1, v0, v1, s[0:1] glc3925; GFX11-NEXT:    s_waitcnt vmcnt(0)3926; GFX11-NEXT:    buffer_gl0_inv3927; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]3928; GFX11-NEXT:    s_endpgm3929entry:3930  %val = atomicrmw volatile umin ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03931  store i32 %val, ptr addrspace(1) %out23932  ret void3933}3934 3935define amdgpu_kernel void @atomic_umin_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) {3936; SI-LABEL: atomic_umin_i32_addr64:3937; SI:       ; %bb.0: ; %entry3938; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd3939; SI-NEXT:    s_load_dword s8, s[4:5], 0xb3940; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x93941; SI-NEXT:    s_mov_b32 s3, 0xf0003942; SI-NEXT:    s_waitcnt lgkmcnt(0)3943; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 23944; SI-NEXT:    s_mov_b32 s2, 03945; SI-NEXT:    v_mov_b32_e32 v2, s83946; SI-NEXT:    v_mov_b32_e32 v0, s43947; SI-NEXT:    v_mov_b32_e32 v1, s53948; SI-NEXT:    buffer_atomic_umin v2, v[0:1], s[0:3], 0 addr643949; SI-NEXT:    s_endpgm3950;3951; VI-LABEL: atomic_umin_i32_addr64:3952; VI:       ; %bb.0: ; %entry3953; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x343954; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x243955; VI-NEXT:    s_load_dword s4, s[4:5], 0x2c3956; VI-NEXT:    s_waitcnt lgkmcnt(0)3957; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 23958; VI-NEXT:    s_add_u32 s0, s2, s03959; VI-NEXT:    s_addc_u32 s1, s3, s13960; VI-NEXT:    v_mov_b32_e32 v0, s03961; VI-NEXT:    v_mov_b32_e32 v1, s13962; VI-NEXT:    v_mov_b32_e32 v2, s43963; VI-NEXT:    flat_atomic_umin v[0:1], v23964; VI-NEXT:    s_endpgm3965;3966; GFX9-LABEL: atomic_umin_i32_addr64:3967; GFX9:       ; %bb.0: ; %entry3968; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x343969; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x243970; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c3971; GFX9-NEXT:    v_mov_b32_e32 v0, 03972; GFX9-NEXT:    s_waitcnt lgkmcnt(0)3973; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 23974; GFX9-NEXT:    s_add_u32 s0, s2, s03975; GFX9-NEXT:    s_addc_u32 s1, s3, s13976; GFX9-NEXT:    v_mov_b32_e32 v1, s63977; GFX9-NEXT:    global_atomic_umin v0, v1, s[0:1]3978; GFX9-NEXT:    s_endpgm3979;3980; GFX11-LABEL: atomic_umin_i32_addr64:3981; GFX11:       ; %bb.0: ; %entry3982; GFX11-NEXT:    s_clause 0x23983; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x343984; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c3985; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x243986; GFX11-NEXT:    v_mov_b32_e32 v0, 03987; GFX11-NEXT:    s_waitcnt lgkmcnt(0)3988; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 23989; GFX11-NEXT:    v_mov_b32_e32 v1, s63990; GFX11-NEXT:    s_add_u32 s0, s2, s03991; GFX11-NEXT:    s_addc_u32 s1, s3, s13992; GFX11-NEXT:    global_atomic_min_u32 v0, v1, s[0:1]3993; GFX11-NEXT:    s_waitcnt_vscnt null, 0x03994; GFX11-NEXT:    buffer_gl0_inv3995; GFX11-NEXT:    s_endpgm3996entry:3997  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index3998  %val = atomicrmw volatile umin ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03999  ret void4000}4001 4002define amdgpu_kernel void @atomic_umin_i32_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) {4003; SI-LABEL: atomic_umin_i32_ret_addr64:4004; SI:       ; %bb.0: ; %entry4005; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x94006; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf4007; SI-NEXT:    s_load_dword s6, s[4:5], 0xd4008; SI-NEXT:    s_mov_b32 s7, 0xf0004009; SI-NEXT:    s_waitcnt lgkmcnt(0)4010; SI-NEXT:    s_mov_b32 s4, s24011; SI-NEXT:    s_mov_b32 s5, s34012; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 24013; SI-NEXT:    s_mov_b32 s2, 04014; SI-NEXT:    s_mov_b32 s3, s74015; SI-NEXT:    v_mov_b32_e32 v2, s64016; SI-NEXT:    v_mov_b32_e32 v0, s84017; SI-NEXT:    v_mov_b32_e32 v1, s94018; SI-NEXT:    buffer_atomic_umin v2, v[0:1], s[0:3], 0 addr64 glc4019; SI-NEXT:    s_mov_b32 s6, -14020; SI-NEXT:    s_waitcnt vmcnt(0)4021; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 04022; SI-NEXT:    s_endpgm4023;4024; VI-LABEL: atomic_umin_i32_ret_addr64:4025; VI:       ; %bb.0: ; %entry4026; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c4027; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244028; VI-NEXT:    s_load_dword s8, s[4:5], 0x344029; VI-NEXT:    s_waitcnt lgkmcnt(0)4030; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 24031; VI-NEXT:    s_add_u32 s0, s0, s44032; VI-NEXT:    s_addc_u32 s1, s1, s54033; VI-NEXT:    v_mov_b32_e32 v0, s04034; VI-NEXT:    v_mov_b32_e32 v1, s14035; VI-NEXT:    v_mov_b32_e32 v2, s84036; VI-NEXT:    flat_atomic_umin v0, v[0:1], v2 glc4037; VI-NEXT:    s_mov_b32 s7, 0xf0004038; VI-NEXT:    s_mov_b32 s6, -14039; VI-NEXT:    s_mov_b32 s4, s24040; VI-NEXT:    s_mov_b32 s5, s34041; VI-NEXT:    s_waitcnt vmcnt(0)4042; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 04043; VI-NEXT:    s_endpgm4044;4045; GFX9-LABEL: atomic_umin_i32_ret_addr64:4046; GFX9:       ; %bb.0: ; %entry4047; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c4048; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244049; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x344050; GFX9-NEXT:    v_mov_b32_e32 v0, 04051; GFX9-NEXT:    s_waitcnt lgkmcnt(0)4052; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 24053; GFX9-NEXT:    s_add_u32 s0, s0, s44054; GFX9-NEXT:    s_addc_u32 s1, s1, s54055; GFX9-NEXT:    v_mov_b32_e32 v1, s84056; GFX9-NEXT:    global_atomic_umin v1, v0, v1, s[0:1] glc4057; GFX9-NEXT:    s_waitcnt vmcnt(0)4058; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]4059; GFX9-NEXT:    s_endpgm4060;4061; GFX11-LABEL: atomic_umin_i32_ret_addr64:4062; GFX11:       ; %bb.0: ; %entry4063; GFX11-NEXT:    s_clause 0x24064; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c4065; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x344066; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x244067; GFX11-NEXT:    v_mov_b32_e32 v0, 04068; GFX11-NEXT:    s_waitcnt lgkmcnt(0)4069; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 24070; GFX11-NEXT:    v_mov_b32_e32 v1, s84071; GFX11-NEXT:    s_add_u32 s0, s0, s44072; GFX11-NEXT:    s_addc_u32 s1, s1, s54073; GFX11-NEXT:    global_atomic_min_u32 v1, v0, v1, s[0:1] glc4074; GFX11-NEXT:    s_waitcnt vmcnt(0)4075; GFX11-NEXT:    buffer_gl0_inv4076; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]4077; GFX11-NEXT:    s_endpgm4078entry:4079  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index4080  %val = atomicrmw volatile umin ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !04081  store i32 %val, ptr addrspace(1) %out24082  ret void4083}4084 4085define amdgpu_kernel void @atomic_or_i32_offset(ptr addrspace(1) %out, i32 %in) {4086; SI-LABEL: atomic_or_i32_offset:4087; SI:       ; %bb.0: ; %entry4088; SI-NEXT:    s_load_dword s6, s[4:5], 0xb4089; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x94090; SI-NEXT:    s_mov_b32 s3, 0xf0004091; SI-NEXT:    s_mov_b32 s2, -14092; SI-NEXT:    s_waitcnt lgkmcnt(0)4093; SI-NEXT:    v_mov_b32_e32 v0, s64094; SI-NEXT:    buffer_atomic_or v0, off, s[0:3], 0 offset:164095; SI-NEXT:    s_waitcnt vmcnt(0)4096; SI-NEXT:    buffer_wbinvl14097; SI-NEXT:    s_endpgm4098;4099; VI-LABEL: atomic_or_i32_offset:4100; VI:       ; %bb.0: ; %entry4101; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c4102; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x244103; VI-NEXT:    s_mov_b32 s3, 0xf0004104; VI-NEXT:    s_mov_b32 s2, -14105; VI-NEXT:    s_waitcnt lgkmcnt(0)4106; VI-NEXT:    v_mov_b32_e32 v0, s64107; VI-NEXT:    buffer_atomic_or v0, off, s[0:3], 0 offset:164108; VI-NEXT:    s_waitcnt vmcnt(0)4109; VI-NEXT:    buffer_wbinvl1_vol4110; VI-NEXT:    s_endpgm4111;4112; GFX9-LABEL: atomic_or_i32_offset:4113; GFX9:       ; %bb.0: ; %entry4114; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c4115; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x244116; GFX9-NEXT:    v_mov_b32_e32 v0, 04117; GFX9-NEXT:    s_waitcnt lgkmcnt(0)4118; GFX9-NEXT:    v_mov_b32_e32 v1, s24119; GFX9-NEXT:    global_atomic_or v0, v1, s[0:1] offset:164120; GFX9-NEXT:    s_waitcnt vmcnt(0)4121; GFX9-NEXT:    buffer_wbinvl1_vol4122; GFX9-NEXT:    s_endpgm4123;4124; GFX11-LABEL: atomic_or_i32_offset:4125; GFX11:       ; %bb.0: ; %entry4126; GFX11-NEXT:    s_clause 0x14127; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c4128; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x244129; GFX11-NEXT:    s_waitcnt lgkmcnt(0)4130; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s24131; GFX11-NEXT:    global_atomic_or_b32 v0, v1, s[0:1] offset:164132; GFX11-NEXT:    s_waitcnt_vscnt null, 0x04133; GFX11-NEXT:    buffer_gl1_inv4134; GFX11-NEXT:    buffer_gl0_inv4135; GFX11-NEXT:    s_endpgm4136entry:4137  %gep = getelementptr i32, ptr addrspace(1) %out, i64 44138  %val = atomicrmw volatile or ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04139  ret void4140}4141 4142define amdgpu_kernel void @atomic_or_i32_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) {4143; SI-LABEL: atomic_or_i32_ret_offset:4144; SI:       ; %bb.0: ; %entry4145; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x94146; SI-NEXT:    s_load_dword s8, s[4:5], 0xd4147; SI-NEXT:    s_mov_b32 s7, 0xf0004148; SI-NEXT:    s_mov_b32 s6, -14149; SI-NEXT:    s_waitcnt lgkmcnt(0)4150; SI-NEXT:    s_mov_b32 s4, s24151; SI-NEXT:    s_mov_b32 s5, s34152; SI-NEXT:    s_mov_b32 s2, s64153; SI-NEXT:    s_mov_b32 s3, s74154; SI-NEXT:    v_mov_b32_e32 v0, s84155; SI-NEXT:    buffer_atomic_or v0, off, s[0:3], 0 offset:16 glc4156; SI-NEXT:    s_waitcnt vmcnt(0)4157; SI-NEXT:    buffer_wbinvl14158; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 04159; SI-NEXT:    s_endpgm4160;4161; VI-LABEL: atomic_or_i32_ret_offset:4162; VI:       ; %bb.0: ; %entry4163; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244164; VI-NEXT:    s_load_dword s8, s[4:5], 0x344165; VI-NEXT:    s_mov_b32 s7, 0xf0004166; VI-NEXT:    s_mov_b32 s6, -14167; VI-NEXT:    s_waitcnt lgkmcnt(0)4168; VI-NEXT:    s_mov_b32 s4, s24169; VI-NEXT:    s_mov_b32 s5, s34170; VI-NEXT:    s_mov_b32 s2, s64171; VI-NEXT:    s_mov_b32 s3, s74172; VI-NEXT:    v_mov_b32_e32 v0, s84173; VI-NEXT:    buffer_atomic_or v0, off, s[0:3], 0 offset:16 glc4174; VI-NEXT:    s_waitcnt vmcnt(0)4175; VI-NEXT:    buffer_wbinvl1_vol4176; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 04177; VI-NEXT:    s_endpgm4178;4179; GFX9-LABEL: atomic_or_i32_ret_offset:4180; GFX9:       ; %bb.0: ; %entry4181; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x344182; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244183; GFX9-NEXT:    v_mov_b32_e32 v0, 04184; GFX9-NEXT:    s_waitcnt lgkmcnt(0)4185; GFX9-NEXT:    v_mov_b32_e32 v1, s64186; GFX9-NEXT:    global_atomic_or v1, v0, v1, s[0:1] offset:16 glc4187; GFX9-NEXT:    s_waitcnt vmcnt(0)4188; GFX9-NEXT:    buffer_wbinvl1_vol4189; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]4190; GFX9-NEXT:    s_endpgm4191;4192; GFX11-LABEL: atomic_or_i32_ret_offset:4193; GFX11:       ; %bb.0: ; %entry4194; GFX11-NEXT:    s_clause 0x14195; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x344196; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x244197; GFX11-NEXT:    s_waitcnt lgkmcnt(0)4198; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s64199; GFX11-NEXT:    global_atomic_or_b32 v1, v0, v1, s[0:1] offset:16 glc4200; GFX11-NEXT:    s_waitcnt vmcnt(0)4201; GFX11-NEXT:    buffer_gl1_inv4202; GFX11-NEXT:    buffer_gl0_inv4203; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]4204; GFX11-NEXT:    s_endpgm4205entry:4206  %gep = getelementptr i32, ptr addrspace(1) %out, i64 44207  %val = atomicrmw volatile or ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04208  store i32 %val, ptr addrspace(1) %out24209  ret void4210}4211 4212define amdgpu_kernel void @atomic_or_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {4213; SI-LABEL: atomic_or_i32_addr64_offset:4214; SI:       ; %bb.0: ; %entry4215; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd4216; SI-NEXT:    s_load_dword s8, s[4:5], 0xb4217; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x94218; SI-NEXT:    s_mov_b32 s3, 0xf0004219; SI-NEXT:    s_waitcnt lgkmcnt(0)4220; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 24221; SI-NEXT:    s_mov_b32 s2, 04222; SI-NEXT:    v_mov_b32_e32 v2, s84223; SI-NEXT:    v_mov_b32_e32 v0, s44224; SI-NEXT:    v_mov_b32_e32 v1, s54225; SI-NEXT:    buffer_atomic_or v2, v[0:1], s[0:3], 0 addr64 offset:164226; SI-NEXT:    s_waitcnt vmcnt(0)4227; SI-NEXT:    buffer_wbinvl14228; SI-NEXT:    s_endpgm4229;4230; VI-LABEL: atomic_or_i32_addr64_offset:4231; VI:       ; %bb.0: ; %entry4232; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x344233; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x244234; VI-NEXT:    s_load_dword s4, s[4:5], 0x2c4235; VI-NEXT:    s_waitcnt lgkmcnt(0)4236; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 24237; VI-NEXT:    s_add_u32 s0, s2, s04238; VI-NEXT:    s_addc_u32 s1, s3, s14239; VI-NEXT:    s_add_u32 s0, s0, 164240; VI-NEXT:    s_addc_u32 s1, s1, 04241; VI-NEXT:    v_mov_b32_e32 v0, s04242; VI-NEXT:    v_mov_b32_e32 v1, s14243; VI-NEXT:    v_mov_b32_e32 v2, s44244; VI-NEXT:    flat_atomic_or v[0:1], v24245; VI-NEXT:    s_waitcnt vmcnt(0)4246; VI-NEXT:    buffer_wbinvl1_vol4247; VI-NEXT:    s_endpgm4248;4249; GFX9-LABEL: atomic_or_i32_addr64_offset:4250; GFX9:       ; %bb.0: ; %entry4251; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x344252; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x244253; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c4254; GFX9-NEXT:    v_mov_b32_e32 v0, 04255; GFX9-NEXT:    s_waitcnt lgkmcnt(0)4256; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 24257; GFX9-NEXT:    s_add_u32 s0, s2, s04258; GFX9-NEXT:    s_addc_u32 s1, s3, s14259; GFX9-NEXT:    v_mov_b32_e32 v1, s64260; GFX9-NEXT:    global_atomic_or v0, v1, s[0:1] offset:164261; GFX9-NEXT:    s_waitcnt vmcnt(0)4262; GFX9-NEXT:    buffer_wbinvl1_vol4263; GFX9-NEXT:    s_endpgm4264;4265; GFX11-LABEL: atomic_or_i32_addr64_offset:4266; GFX11:       ; %bb.0: ; %entry4267; GFX11-NEXT:    s_clause 0x24268; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x344269; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c4270; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x244271; GFX11-NEXT:    v_mov_b32_e32 v0, 04272; GFX11-NEXT:    s_waitcnt lgkmcnt(0)4273; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 24274; GFX11-NEXT:    v_mov_b32_e32 v1, s64275; GFX11-NEXT:    s_add_u32 s0, s2, s04276; GFX11-NEXT:    s_addc_u32 s1, s3, s14277; GFX11-NEXT:    global_atomic_or_b32 v0, v1, s[0:1] offset:164278; GFX11-NEXT:    s_waitcnt_vscnt null, 0x04279; GFX11-NEXT:    buffer_gl1_inv4280; GFX11-NEXT:    buffer_gl0_inv4281; GFX11-NEXT:    s_endpgm4282entry:4283  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index4284  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 44285  %val = atomicrmw volatile or ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04286  ret void4287}4288 4289define amdgpu_kernel void @atomic_or_i32_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) {4290; SI-LABEL: atomic_or_i32_ret_addr64_offset:4291; SI:       ; %bb.0: ; %entry4292; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x94293; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf4294; SI-NEXT:    s_load_dword s6, s[4:5], 0xd4295; SI-NEXT:    s_mov_b32 s7, 0xf0004296; SI-NEXT:    s_waitcnt lgkmcnt(0)4297; SI-NEXT:    s_mov_b32 s4, s24298; SI-NEXT:    s_mov_b32 s5, s34299; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 24300; SI-NEXT:    s_mov_b32 s2, 04301; SI-NEXT:    s_mov_b32 s3, s74302; SI-NEXT:    v_mov_b32_e32 v2, s64303; SI-NEXT:    v_mov_b32_e32 v0, s84304; SI-NEXT:    v_mov_b32_e32 v1, s94305; SI-NEXT:    buffer_atomic_or v2, v[0:1], s[0:3], 0 addr64 offset:16 glc4306; SI-NEXT:    s_waitcnt vmcnt(0)4307; SI-NEXT:    buffer_wbinvl14308; SI-NEXT:    s_mov_b32 s6, -14309; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 04310; SI-NEXT:    s_endpgm4311;4312; VI-LABEL: atomic_or_i32_ret_addr64_offset:4313; VI:       ; %bb.0: ; %entry4314; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c4315; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244316; VI-NEXT:    s_load_dword s8, s[4:5], 0x344317; VI-NEXT:    s_waitcnt lgkmcnt(0)4318; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 24319; VI-NEXT:    s_add_u32 s0, s0, s44320; VI-NEXT:    s_addc_u32 s1, s1, s54321; VI-NEXT:    s_add_u32 s0, s0, 164322; VI-NEXT:    s_addc_u32 s1, s1, 04323; VI-NEXT:    v_mov_b32_e32 v0, s04324; VI-NEXT:    v_mov_b32_e32 v1, s14325; VI-NEXT:    v_mov_b32_e32 v2, s84326; VI-NEXT:    flat_atomic_or v0, v[0:1], v2 glc4327; VI-NEXT:    s_waitcnt vmcnt(0)4328; VI-NEXT:    buffer_wbinvl1_vol4329; VI-NEXT:    s_mov_b32 s7, 0xf0004330; VI-NEXT:    s_mov_b32 s6, -14331; VI-NEXT:    s_mov_b32 s4, s24332; VI-NEXT:    s_mov_b32 s5, s34333; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 04334; VI-NEXT:    s_endpgm4335;4336; GFX9-LABEL: atomic_or_i32_ret_addr64_offset:4337; GFX9:       ; %bb.0: ; %entry4338; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c4339; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244340; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x344341; GFX9-NEXT:    v_mov_b32_e32 v0, 04342; GFX9-NEXT:    s_waitcnt lgkmcnt(0)4343; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 24344; GFX9-NEXT:    s_add_u32 s0, s0, s44345; GFX9-NEXT:    s_addc_u32 s1, s1, s54346; GFX9-NEXT:    v_mov_b32_e32 v1, s84347; GFX9-NEXT:    global_atomic_or v1, v0, v1, s[0:1] offset:16 glc4348; GFX9-NEXT:    s_waitcnt vmcnt(0)4349; GFX9-NEXT:    buffer_wbinvl1_vol4350; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]4351; GFX9-NEXT:    s_endpgm4352;4353; GFX11-LABEL: atomic_or_i32_ret_addr64_offset:4354; GFX11:       ; %bb.0: ; %entry4355; GFX11-NEXT:    s_clause 0x24356; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c4357; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x344358; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x244359; GFX11-NEXT:    v_mov_b32_e32 v0, 04360; GFX11-NEXT:    s_waitcnt lgkmcnt(0)4361; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 24362; GFX11-NEXT:    v_mov_b32_e32 v1, s84363; GFX11-NEXT:    s_add_u32 s0, s0, s44364; GFX11-NEXT:    s_addc_u32 s1, s1, s54365; GFX11-NEXT:    global_atomic_or_b32 v1, v0, v1, s[0:1] offset:16 glc4366; GFX11-NEXT:    s_waitcnt vmcnt(0)4367; GFX11-NEXT:    buffer_gl1_inv4368; GFX11-NEXT:    buffer_gl0_inv4369; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]4370; GFX11-NEXT:    s_endpgm4371entry:4372  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index4373  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 44374  %val = atomicrmw volatile or ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04375  store i32 %val, ptr addrspace(1) %out24376  ret void4377}4378 4379define amdgpu_kernel void @atomic_or_i32(ptr addrspace(1) %out, i32 %in) {4380; SI-LABEL: atomic_or_i32:4381; SI:       ; %bb.0: ; %entry4382; SI-NEXT:    s_load_dword s6, s[4:5], 0xb4383; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x94384; SI-NEXT:    s_mov_b32 s3, 0xf0004385; SI-NEXT:    s_mov_b32 s2, -14386; SI-NEXT:    s_waitcnt lgkmcnt(0)4387; SI-NEXT:    v_mov_b32_e32 v0, s64388; SI-NEXT:    buffer_atomic_or v0, off, s[0:3], 04389; SI-NEXT:    s_waitcnt vmcnt(0)4390; SI-NEXT:    buffer_wbinvl14391; SI-NEXT:    s_endpgm4392;4393; VI-LABEL: atomic_or_i32:4394; VI:       ; %bb.0: ; %entry4395; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c4396; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x244397; VI-NEXT:    s_mov_b32 s3, 0xf0004398; VI-NEXT:    s_mov_b32 s2, -14399; VI-NEXT:    s_waitcnt lgkmcnt(0)4400; VI-NEXT:    v_mov_b32_e32 v0, s64401; VI-NEXT:    buffer_atomic_or v0, off, s[0:3], 04402; VI-NEXT:    s_waitcnt vmcnt(0)4403; VI-NEXT:    buffer_wbinvl1_vol4404; VI-NEXT:    s_endpgm4405;4406; GFX9-LABEL: atomic_or_i32:4407; GFX9:       ; %bb.0: ; %entry4408; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c4409; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x244410; GFX9-NEXT:    v_mov_b32_e32 v0, 04411; GFX9-NEXT:    s_waitcnt lgkmcnt(0)4412; GFX9-NEXT:    v_mov_b32_e32 v1, s24413; GFX9-NEXT:    global_atomic_or v0, v1, s[0:1]4414; GFX9-NEXT:    s_waitcnt vmcnt(0)4415; GFX9-NEXT:    buffer_wbinvl1_vol4416; GFX9-NEXT:    s_endpgm4417;4418; GFX11-LABEL: atomic_or_i32:4419; GFX11:       ; %bb.0: ; %entry4420; GFX11-NEXT:    s_clause 0x14421; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c4422; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x244423; GFX11-NEXT:    s_waitcnt lgkmcnt(0)4424; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s24425; GFX11-NEXT:    global_atomic_or_b32 v0, v1, s[0:1]4426; GFX11-NEXT:    s_waitcnt_vscnt null, 0x04427; GFX11-NEXT:    buffer_gl1_inv4428; GFX11-NEXT:    buffer_gl0_inv4429; GFX11-NEXT:    s_endpgm4430entry:4431  %val = atomicrmw volatile or ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04432  ret void4433}4434 4435define amdgpu_kernel void @atomic_or_i32_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) {4436; SI-LABEL: atomic_or_i32_ret:4437; SI:       ; %bb.0: ; %entry4438; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x94439; SI-NEXT:    s_load_dword s8, s[4:5], 0xd4440; SI-NEXT:    s_mov_b32 s7, 0xf0004441; SI-NEXT:    s_mov_b32 s6, -14442; SI-NEXT:    s_waitcnt lgkmcnt(0)4443; SI-NEXT:    s_mov_b32 s4, s04444; SI-NEXT:    s_mov_b32 s5, s14445; SI-NEXT:    v_mov_b32_e32 v0, s84446; SI-NEXT:    buffer_atomic_or v0, off, s[4:7], 0 glc4447; SI-NEXT:    s_waitcnt vmcnt(0)4448; SI-NEXT:    buffer_wbinvl14449; SI-NEXT:    s_mov_b32 s4, s24450; SI-NEXT:    s_mov_b32 s5, s34451; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 04452; SI-NEXT:    s_endpgm4453;4454; VI-LABEL: atomic_or_i32_ret:4455; VI:       ; %bb.0: ; %entry4456; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244457; VI-NEXT:    s_load_dword s8, s[4:5], 0x344458; VI-NEXT:    s_mov_b32 s7, 0xf0004459; VI-NEXT:    s_mov_b32 s6, -14460; VI-NEXT:    s_waitcnt lgkmcnt(0)4461; VI-NEXT:    s_mov_b32 s4, s04462; VI-NEXT:    s_mov_b32 s5, s14463; VI-NEXT:    v_mov_b32_e32 v0, s84464; VI-NEXT:    buffer_atomic_or v0, off, s[4:7], 0 glc4465; VI-NEXT:    s_waitcnt vmcnt(0)4466; VI-NEXT:    buffer_wbinvl1_vol4467; VI-NEXT:    s_mov_b32 s4, s24468; VI-NEXT:    s_mov_b32 s5, s34469; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 04470; VI-NEXT:    s_endpgm4471;4472; GFX9-LABEL: atomic_or_i32_ret:4473; GFX9:       ; %bb.0: ; %entry4474; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x344475; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244476; GFX9-NEXT:    v_mov_b32_e32 v0, 04477; GFX9-NEXT:    s_waitcnt lgkmcnt(0)4478; GFX9-NEXT:    v_mov_b32_e32 v1, s64479; GFX9-NEXT:    global_atomic_or v1, v0, v1, s[0:1] glc4480; GFX9-NEXT:    s_waitcnt vmcnt(0)4481; GFX9-NEXT:    buffer_wbinvl1_vol4482; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]4483; GFX9-NEXT:    s_endpgm4484;4485; GFX11-LABEL: atomic_or_i32_ret:4486; GFX11:       ; %bb.0: ; %entry4487; GFX11-NEXT:    s_clause 0x14488; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x344489; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x244490; GFX11-NEXT:    s_waitcnt lgkmcnt(0)4491; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s64492; GFX11-NEXT:    global_atomic_or_b32 v1, v0, v1, s[0:1] glc4493; GFX11-NEXT:    s_waitcnt vmcnt(0)4494; GFX11-NEXT:    buffer_gl1_inv4495; GFX11-NEXT:    buffer_gl0_inv4496; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]4497; GFX11-NEXT:    s_endpgm4498entry:4499  %val = atomicrmw volatile or ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04500  store i32 %val, ptr addrspace(1) %out24501  ret void4502}4503 4504define amdgpu_kernel void @atomic_or_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) {4505; SI-LABEL: atomic_or_i32_addr64:4506; SI:       ; %bb.0: ; %entry4507; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd4508; SI-NEXT:    s_load_dword s8, s[4:5], 0xb4509; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x94510; SI-NEXT:    s_mov_b32 s3, 0xf0004511; SI-NEXT:    s_waitcnt lgkmcnt(0)4512; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 24513; SI-NEXT:    s_mov_b32 s2, 04514; SI-NEXT:    v_mov_b32_e32 v2, s84515; SI-NEXT:    v_mov_b32_e32 v0, s44516; SI-NEXT:    v_mov_b32_e32 v1, s54517; SI-NEXT:    buffer_atomic_or v2, v[0:1], s[0:3], 0 addr644518; SI-NEXT:    s_waitcnt vmcnt(0)4519; SI-NEXT:    buffer_wbinvl14520; SI-NEXT:    s_endpgm4521;4522; VI-LABEL: atomic_or_i32_addr64:4523; VI:       ; %bb.0: ; %entry4524; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x344525; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x244526; VI-NEXT:    s_load_dword s4, s[4:5], 0x2c4527; VI-NEXT:    s_waitcnt lgkmcnt(0)4528; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 24529; VI-NEXT:    s_add_u32 s0, s2, s04530; VI-NEXT:    s_addc_u32 s1, s3, s14531; VI-NEXT:    v_mov_b32_e32 v0, s04532; VI-NEXT:    v_mov_b32_e32 v1, s14533; VI-NEXT:    v_mov_b32_e32 v2, s44534; VI-NEXT:    flat_atomic_or v[0:1], v24535; VI-NEXT:    s_waitcnt vmcnt(0)4536; VI-NEXT:    buffer_wbinvl1_vol4537; VI-NEXT:    s_endpgm4538;4539; GFX9-LABEL: atomic_or_i32_addr64:4540; GFX9:       ; %bb.0: ; %entry4541; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x344542; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x244543; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c4544; GFX9-NEXT:    v_mov_b32_e32 v0, 04545; GFX9-NEXT:    s_waitcnt lgkmcnt(0)4546; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 24547; GFX9-NEXT:    s_add_u32 s0, s2, s04548; GFX9-NEXT:    s_addc_u32 s1, s3, s14549; GFX9-NEXT:    v_mov_b32_e32 v1, s64550; GFX9-NEXT:    global_atomic_or v0, v1, s[0:1]4551; GFX9-NEXT:    s_waitcnt vmcnt(0)4552; GFX9-NEXT:    buffer_wbinvl1_vol4553; GFX9-NEXT:    s_endpgm4554;4555; GFX11-LABEL: atomic_or_i32_addr64:4556; GFX11:       ; %bb.0: ; %entry4557; GFX11-NEXT:    s_clause 0x24558; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x344559; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c4560; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x244561; GFX11-NEXT:    v_mov_b32_e32 v0, 04562; GFX11-NEXT:    s_waitcnt lgkmcnt(0)4563; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 24564; GFX11-NEXT:    v_mov_b32_e32 v1, s64565; GFX11-NEXT:    s_add_u32 s0, s2, s04566; GFX11-NEXT:    s_addc_u32 s1, s3, s14567; GFX11-NEXT:    global_atomic_or_b32 v0, v1, s[0:1]4568; GFX11-NEXT:    s_waitcnt_vscnt null, 0x04569; GFX11-NEXT:    buffer_gl1_inv4570; GFX11-NEXT:    buffer_gl0_inv4571; GFX11-NEXT:    s_endpgm4572entry:4573  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index4574  %val = atomicrmw volatile or ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04575  ret void4576}4577 4578define amdgpu_kernel void @atomic_or_i32_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) {4579; SI-LABEL: atomic_or_i32_ret_addr64:4580; SI:       ; %bb.0: ; %entry4581; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x94582; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf4583; SI-NEXT:    s_load_dword s6, s[4:5], 0xd4584; SI-NEXT:    s_mov_b32 s7, 0xf0004585; SI-NEXT:    s_waitcnt lgkmcnt(0)4586; SI-NEXT:    s_mov_b32 s4, s24587; SI-NEXT:    s_mov_b32 s5, s34588; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 24589; SI-NEXT:    s_mov_b32 s2, 04590; SI-NEXT:    s_mov_b32 s3, s74591; SI-NEXT:    v_mov_b32_e32 v2, s64592; SI-NEXT:    v_mov_b32_e32 v0, s84593; SI-NEXT:    v_mov_b32_e32 v1, s94594; SI-NEXT:    buffer_atomic_or v2, v[0:1], s[0:3], 0 addr64 glc4595; SI-NEXT:    s_waitcnt vmcnt(0)4596; SI-NEXT:    buffer_wbinvl14597; SI-NEXT:    s_mov_b32 s6, -14598; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 04599; SI-NEXT:    s_endpgm4600;4601; VI-LABEL: atomic_or_i32_ret_addr64:4602; VI:       ; %bb.0: ; %entry4603; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c4604; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244605; VI-NEXT:    s_load_dword s8, s[4:5], 0x344606; VI-NEXT:    s_waitcnt lgkmcnt(0)4607; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 24608; VI-NEXT:    s_add_u32 s0, s0, s44609; VI-NEXT:    s_addc_u32 s1, s1, s54610; VI-NEXT:    v_mov_b32_e32 v0, s04611; VI-NEXT:    v_mov_b32_e32 v1, s14612; VI-NEXT:    v_mov_b32_e32 v2, s84613; VI-NEXT:    flat_atomic_or v0, v[0:1], v2 glc4614; VI-NEXT:    s_waitcnt vmcnt(0)4615; VI-NEXT:    buffer_wbinvl1_vol4616; VI-NEXT:    s_mov_b32 s7, 0xf0004617; VI-NEXT:    s_mov_b32 s6, -14618; VI-NEXT:    s_mov_b32 s4, s24619; VI-NEXT:    s_mov_b32 s5, s34620; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 04621; VI-NEXT:    s_endpgm4622;4623; GFX9-LABEL: atomic_or_i32_ret_addr64:4624; GFX9:       ; %bb.0: ; %entry4625; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c4626; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244627; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x344628; GFX9-NEXT:    v_mov_b32_e32 v0, 04629; GFX9-NEXT:    s_waitcnt lgkmcnt(0)4630; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 24631; GFX9-NEXT:    s_add_u32 s0, s0, s44632; GFX9-NEXT:    s_addc_u32 s1, s1, s54633; GFX9-NEXT:    v_mov_b32_e32 v1, s84634; GFX9-NEXT:    global_atomic_or v1, v0, v1, s[0:1] glc4635; GFX9-NEXT:    s_waitcnt vmcnt(0)4636; GFX9-NEXT:    buffer_wbinvl1_vol4637; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]4638; GFX9-NEXT:    s_endpgm4639;4640; GFX11-LABEL: atomic_or_i32_ret_addr64:4641; GFX11:       ; %bb.0: ; %entry4642; GFX11-NEXT:    s_clause 0x24643; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c4644; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x344645; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x244646; GFX11-NEXT:    v_mov_b32_e32 v0, 04647; GFX11-NEXT:    s_waitcnt lgkmcnt(0)4648; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 24649; GFX11-NEXT:    v_mov_b32_e32 v1, s84650; GFX11-NEXT:    s_add_u32 s0, s0, s44651; GFX11-NEXT:    s_addc_u32 s1, s1, s54652; GFX11-NEXT:    global_atomic_or_b32 v1, v0, v1, s[0:1] glc4653; GFX11-NEXT:    s_waitcnt vmcnt(0)4654; GFX11-NEXT:    buffer_gl1_inv4655; GFX11-NEXT:    buffer_gl0_inv4656; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]4657; GFX11-NEXT:    s_endpgm4658entry:4659  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index4660  %val = atomicrmw volatile or ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04661  store i32 %val, ptr addrspace(1) %out24662  ret void4663}4664 4665define amdgpu_kernel void @atomic_xchg_i32_offset(ptr addrspace(1) %out, i32 %in) {4666; SI-LABEL: atomic_xchg_i32_offset:4667; SI:       ; %bb.0: ; %entry4668; SI-NEXT:    s_load_dword s6, s[4:5], 0xb4669; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x94670; SI-NEXT:    s_mov_b32 s3, 0xf0004671; SI-NEXT:    s_mov_b32 s2, -14672; SI-NEXT:    s_waitcnt lgkmcnt(0)4673; SI-NEXT:    v_mov_b32_e32 v0, s64674; SI-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 offset:164675; SI-NEXT:    s_waitcnt vmcnt(0)4676; SI-NEXT:    buffer_wbinvl14677; SI-NEXT:    s_endpgm4678;4679; VI-LABEL: atomic_xchg_i32_offset:4680; VI:       ; %bb.0: ; %entry4681; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c4682; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x244683; VI-NEXT:    s_mov_b32 s3, 0xf0004684; VI-NEXT:    s_mov_b32 s2, -14685; VI-NEXT:    s_waitcnt lgkmcnt(0)4686; VI-NEXT:    v_mov_b32_e32 v0, s64687; VI-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 offset:164688; VI-NEXT:    s_waitcnt vmcnt(0)4689; VI-NEXT:    buffer_wbinvl1_vol4690; VI-NEXT:    s_endpgm4691;4692; GFX9-LABEL: atomic_xchg_i32_offset:4693; GFX9:       ; %bb.0: ; %entry4694; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c4695; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x244696; GFX9-NEXT:    v_mov_b32_e32 v0, 04697; GFX9-NEXT:    s_waitcnt lgkmcnt(0)4698; GFX9-NEXT:    v_mov_b32_e32 v1, s24699; GFX9-NEXT:    global_atomic_swap v0, v1, s[0:1] offset:164700; GFX9-NEXT:    s_waitcnt vmcnt(0)4701; GFX9-NEXT:    buffer_wbinvl1_vol4702; GFX9-NEXT:    s_endpgm4703;4704; GFX11-LABEL: atomic_xchg_i32_offset:4705; GFX11:       ; %bb.0: ; %entry4706; GFX11-NEXT:    s_clause 0x14707; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c4708; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x244709; GFX11-NEXT:    s_waitcnt lgkmcnt(0)4710; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s24711; GFX11-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] offset:164712; GFX11-NEXT:    s_waitcnt_vscnt null, 0x04713; GFX11-NEXT:    buffer_gl1_inv4714; GFX11-NEXT:    buffer_gl0_inv4715; GFX11-NEXT:    s_endpgm4716entry:4717  %gep = getelementptr i32, ptr addrspace(1) %out, i64 44718  %val = atomicrmw volatile xchg ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04719  ret void4720}4721 4722define amdgpu_kernel void @atomic_xchg_f32_offset(ptr addrspace(1) %out, float %in) {4723; SI-LABEL: atomic_xchg_f32_offset:4724; SI:       ; %bb.0: ; %entry4725; SI-NEXT:    s_load_dword s6, s[4:5], 0xb4726; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x94727; SI-NEXT:    s_mov_b32 s3, 0xf0004728; SI-NEXT:    s_mov_b32 s2, -14729; SI-NEXT:    s_waitcnt lgkmcnt(0)4730; SI-NEXT:    v_mov_b32_e32 v0, s64731; SI-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 offset:164732; SI-NEXT:    s_waitcnt vmcnt(0)4733; SI-NEXT:    buffer_wbinvl14734; SI-NEXT:    s_endpgm4735;4736; VI-LABEL: atomic_xchg_f32_offset:4737; VI:       ; %bb.0: ; %entry4738; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c4739; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x244740; VI-NEXT:    s_mov_b32 s3, 0xf0004741; VI-NEXT:    s_mov_b32 s2, -14742; VI-NEXT:    s_waitcnt lgkmcnt(0)4743; VI-NEXT:    v_mov_b32_e32 v0, s64744; VI-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 offset:164745; VI-NEXT:    s_waitcnt vmcnt(0)4746; VI-NEXT:    buffer_wbinvl1_vol4747; VI-NEXT:    s_endpgm4748;4749; GFX9-LABEL: atomic_xchg_f32_offset:4750; GFX9:       ; %bb.0: ; %entry4751; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c4752; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x244753; GFX9-NEXT:    v_mov_b32_e32 v0, 04754; GFX9-NEXT:    s_waitcnt lgkmcnt(0)4755; GFX9-NEXT:    v_mov_b32_e32 v1, s24756; GFX9-NEXT:    global_atomic_swap v0, v1, s[0:1] offset:164757; GFX9-NEXT:    s_waitcnt vmcnt(0)4758; GFX9-NEXT:    buffer_wbinvl1_vol4759; GFX9-NEXT:    s_endpgm4760;4761; GFX11-LABEL: atomic_xchg_f32_offset:4762; GFX11:       ; %bb.0: ; %entry4763; GFX11-NEXT:    s_clause 0x14764; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c4765; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x244766; GFX11-NEXT:    s_waitcnt lgkmcnt(0)4767; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s24768; GFX11-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] offset:164769; GFX11-NEXT:    s_waitcnt_vscnt null, 0x04770; GFX11-NEXT:    buffer_gl1_inv4771; GFX11-NEXT:    buffer_gl0_inv4772; GFX11-NEXT:    s_endpgm4773entry:4774  %gep = getelementptr float, ptr addrspace(1) %out, i64 44775  %val = atomicrmw volatile xchg ptr addrspace(1) %gep, float %in syncscope("agent") seq_cst4776  ret void4777}4778 4779define amdgpu_kernel void @atomic_xchg_i32_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) {4780; SI-LABEL: atomic_xchg_i32_ret_offset:4781; SI:       ; %bb.0: ; %entry4782; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x94783; SI-NEXT:    s_load_dword s8, s[4:5], 0xd4784; SI-NEXT:    s_mov_b32 s7, 0xf0004785; SI-NEXT:    s_mov_b32 s6, -14786; SI-NEXT:    s_waitcnt lgkmcnt(0)4787; SI-NEXT:    s_mov_b32 s4, s24788; SI-NEXT:    s_mov_b32 s5, s34789; SI-NEXT:    s_mov_b32 s2, s64790; SI-NEXT:    s_mov_b32 s3, s74791; SI-NEXT:    v_mov_b32_e32 v0, s84792; SI-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 offset:16 glc4793; SI-NEXT:    s_waitcnt vmcnt(0)4794; SI-NEXT:    buffer_wbinvl14795; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 04796; SI-NEXT:    s_endpgm4797;4798; VI-LABEL: atomic_xchg_i32_ret_offset:4799; VI:       ; %bb.0: ; %entry4800; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244801; VI-NEXT:    s_load_dword s8, s[4:5], 0x344802; VI-NEXT:    s_mov_b32 s7, 0xf0004803; VI-NEXT:    s_mov_b32 s6, -14804; VI-NEXT:    s_waitcnt lgkmcnt(0)4805; VI-NEXT:    s_mov_b32 s4, s24806; VI-NEXT:    s_mov_b32 s5, s34807; VI-NEXT:    s_mov_b32 s2, s64808; VI-NEXT:    s_mov_b32 s3, s74809; VI-NEXT:    v_mov_b32_e32 v0, s84810; VI-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 offset:16 glc4811; VI-NEXT:    s_waitcnt vmcnt(0)4812; VI-NEXT:    buffer_wbinvl1_vol4813; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 04814; VI-NEXT:    s_endpgm4815;4816; GFX9-LABEL: atomic_xchg_i32_ret_offset:4817; GFX9:       ; %bb.0: ; %entry4818; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x344819; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244820; GFX9-NEXT:    v_mov_b32_e32 v0, 04821; GFX9-NEXT:    s_waitcnt lgkmcnt(0)4822; GFX9-NEXT:    v_mov_b32_e32 v1, s64823; GFX9-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] offset:16 glc4824; GFX9-NEXT:    s_waitcnt vmcnt(0)4825; GFX9-NEXT:    buffer_wbinvl1_vol4826; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]4827; GFX9-NEXT:    s_endpgm4828;4829; GFX11-LABEL: atomic_xchg_i32_ret_offset:4830; GFX11:       ; %bb.0: ; %entry4831; GFX11-NEXT:    s_clause 0x14832; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x344833; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x244834; GFX11-NEXT:    s_waitcnt lgkmcnt(0)4835; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s64836; GFX11-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] offset:16 glc4837; GFX11-NEXT:    s_waitcnt vmcnt(0)4838; GFX11-NEXT:    buffer_gl1_inv4839; GFX11-NEXT:    buffer_gl0_inv4840; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]4841; GFX11-NEXT:    s_endpgm4842entry:4843  %gep = getelementptr i32, ptr addrspace(1) %out, i64 44844  %val = atomicrmw volatile xchg ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst4845  store i32 %val, ptr addrspace(1) %out24846  ret void4847}4848 4849define amdgpu_kernel void @atomic_xchg_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {4850; SI-LABEL: atomic_xchg_i32_addr64_offset:4851; SI:       ; %bb.0: ; %entry4852; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd4853; SI-NEXT:    s_load_dword s8, s[4:5], 0xb4854; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x94855; SI-NEXT:    s_mov_b32 s3, 0xf0004856; SI-NEXT:    s_waitcnt lgkmcnt(0)4857; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 24858; SI-NEXT:    s_mov_b32 s2, 04859; SI-NEXT:    v_mov_b32_e32 v2, s84860; SI-NEXT:    v_mov_b32_e32 v0, s44861; SI-NEXT:    v_mov_b32_e32 v1, s54862; SI-NEXT:    buffer_atomic_swap v2, v[0:1], s[0:3], 0 addr64 offset:164863; SI-NEXT:    s_waitcnt vmcnt(0)4864; SI-NEXT:    buffer_wbinvl14865; SI-NEXT:    s_endpgm4866;4867; VI-LABEL: atomic_xchg_i32_addr64_offset:4868; VI:       ; %bb.0: ; %entry4869; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x344870; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x244871; VI-NEXT:    s_load_dword s4, s[4:5], 0x2c4872; VI-NEXT:    s_waitcnt lgkmcnt(0)4873; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 24874; VI-NEXT:    s_add_u32 s0, s2, s04875; VI-NEXT:    s_addc_u32 s1, s3, s14876; VI-NEXT:    s_add_u32 s0, s0, 164877; VI-NEXT:    s_addc_u32 s1, s1, 04878; VI-NEXT:    v_mov_b32_e32 v0, s04879; VI-NEXT:    v_mov_b32_e32 v1, s14880; VI-NEXT:    v_mov_b32_e32 v2, s44881; VI-NEXT:    flat_atomic_swap v[0:1], v24882; VI-NEXT:    s_waitcnt vmcnt(0)4883; VI-NEXT:    buffer_wbinvl1_vol4884; VI-NEXT:    s_endpgm4885;4886; GFX9-LABEL: atomic_xchg_i32_addr64_offset:4887; GFX9:       ; %bb.0: ; %entry4888; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x344889; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x244890; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c4891; GFX9-NEXT:    v_mov_b32_e32 v0, 04892; GFX9-NEXT:    s_waitcnt lgkmcnt(0)4893; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 24894; GFX9-NEXT:    s_add_u32 s0, s2, s04895; GFX9-NEXT:    s_addc_u32 s1, s3, s14896; GFX9-NEXT:    v_mov_b32_e32 v1, s64897; GFX9-NEXT:    global_atomic_swap v0, v1, s[0:1] offset:164898; GFX9-NEXT:    s_waitcnt vmcnt(0)4899; GFX9-NEXT:    buffer_wbinvl1_vol4900; GFX9-NEXT:    s_endpgm4901;4902; GFX11-LABEL: atomic_xchg_i32_addr64_offset:4903; GFX11:       ; %bb.0: ; %entry4904; GFX11-NEXT:    s_clause 0x24905; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x344906; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c4907; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x244908; GFX11-NEXT:    v_mov_b32_e32 v0, 04909; GFX11-NEXT:    s_waitcnt lgkmcnt(0)4910; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 24911; GFX11-NEXT:    v_mov_b32_e32 v1, s64912; GFX11-NEXT:    s_add_u32 s0, s2, s04913; GFX11-NEXT:    s_addc_u32 s1, s3, s14914; GFX11-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] offset:164915; GFX11-NEXT:    s_waitcnt_vscnt null, 0x04916; GFX11-NEXT:    buffer_gl1_inv4917; GFX11-NEXT:    buffer_gl0_inv4918; GFX11-NEXT:    s_endpgm4919entry:4920  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index4921  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 44922  %val = atomicrmw volatile xchg ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst4923  ret void4924}4925 4926define amdgpu_kernel void @atomic_xchg_i32_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) {4927; SI-LABEL: atomic_xchg_i32_ret_addr64_offset:4928; SI:       ; %bb.0: ; %entry4929; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x94930; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf4931; SI-NEXT:    s_load_dword s6, s[4:5], 0xd4932; SI-NEXT:    s_mov_b32 s7, 0xf0004933; SI-NEXT:    s_waitcnt lgkmcnt(0)4934; SI-NEXT:    s_mov_b32 s4, s24935; SI-NEXT:    s_mov_b32 s5, s34936; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 24937; SI-NEXT:    s_mov_b32 s2, 04938; SI-NEXT:    s_mov_b32 s3, s74939; SI-NEXT:    v_mov_b32_e32 v2, s64940; SI-NEXT:    v_mov_b32_e32 v0, s84941; SI-NEXT:    v_mov_b32_e32 v1, s94942; SI-NEXT:    buffer_atomic_swap v2, v[0:1], s[0:3], 0 addr64 offset:16 glc4943; SI-NEXT:    s_waitcnt vmcnt(0)4944; SI-NEXT:    buffer_wbinvl14945; SI-NEXT:    s_mov_b32 s6, -14946; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 04947; SI-NEXT:    s_endpgm4948;4949; VI-LABEL: atomic_xchg_i32_ret_addr64_offset:4950; VI:       ; %bb.0: ; %entry4951; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c4952; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244953; VI-NEXT:    s_load_dword s8, s[4:5], 0x344954; VI-NEXT:    s_waitcnt lgkmcnt(0)4955; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 24956; VI-NEXT:    s_add_u32 s0, s0, s44957; VI-NEXT:    s_addc_u32 s1, s1, s54958; VI-NEXT:    s_add_u32 s0, s0, 164959; VI-NEXT:    s_addc_u32 s1, s1, 04960; VI-NEXT:    v_mov_b32_e32 v0, s04961; VI-NEXT:    v_mov_b32_e32 v1, s14962; VI-NEXT:    v_mov_b32_e32 v2, s84963; VI-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc4964; VI-NEXT:    s_waitcnt vmcnt(0)4965; VI-NEXT:    buffer_wbinvl1_vol4966; VI-NEXT:    s_mov_b32 s7, 0xf0004967; VI-NEXT:    s_mov_b32 s6, -14968; VI-NEXT:    s_mov_b32 s4, s24969; VI-NEXT:    s_mov_b32 s5, s34970; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 04971; VI-NEXT:    s_endpgm4972;4973; GFX9-LABEL: atomic_xchg_i32_ret_addr64_offset:4974; GFX9:       ; %bb.0: ; %entry4975; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c4976; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x244977; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x344978; GFX9-NEXT:    v_mov_b32_e32 v0, 04979; GFX9-NEXT:    s_waitcnt lgkmcnt(0)4980; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 24981; GFX9-NEXT:    s_add_u32 s0, s0, s44982; GFX9-NEXT:    s_addc_u32 s1, s1, s54983; GFX9-NEXT:    v_mov_b32_e32 v1, s84984; GFX9-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] offset:16 glc4985; GFX9-NEXT:    s_waitcnt vmcnt(0)4986; GFX9-NEXT:    buffer_wbinvl1_vol4987; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]4988; GFX9-NEXT:    s_endpgm4989;4990; GFX11-LABEL: atomic_xchg_i32_ret_addr64_offset:4991; GFX11:       ; %bb.0: ; %entry4992; GFX11-NEXT:    s_clause 0x24993; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c4994; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x344995; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x244996; GFX11-NEXT:    v_mov_b32_e32 v0, 04997; GFX11-NEXT:    s_waitcnt lgkmcnt(0)4998; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 24999; GFX11-NEXT:    v_mov_b32_e32 v1, s85000; GFX11-NEXT:    s_add_u32 s0, s0, s45001; GFX11-NEXT:    s_addc_u32 s1, s1, s55002; GFX11-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] offset:16 glc5003; GFX11-NEXT:    s_waitcnt vmcnt(0)5004; GFX11-NEXT:    buffer_gl1_inv5005; GFX11-NEXT:    buffer_gl0_inv5006; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]5007; GFX11-NEXT:    s_endpgm5008entry:5009  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index5010  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 45011  %val = atomicrmw volatile xchg ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst5012  store i32 %val, ptr addrspace(1) %out25013  ret void5014}5015 5016define amdgpu_kernel void @atomic_xchg_i32(ptr addrspace(1) %out, i32 %in) {5017; SI-LABEL: atomic_xchg_i32:5018; SI:       ; %bb.0: ; %entry5019; SI-NEXT:    s_load_dword s6, s[4:5], 0xb5020; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x95021; SI-NEXT:    s_mov_b32 s3, 0xf0005022; SI-NEXT:    s_mov_b32 s2, -15023; SI-NEXT:    s_waitcnt lgkmcnt(0)5024; SI-NEXT:    v_mov_b32_e32 v0, s65025; SI-NEXT:    buffer_atomic_swap v0, off, s[0:3], 05026; SI-NEXT:    s_waitcnt vmcnt(0)5027; SI-NEXT:    buffer_wbinvl15028; SI-NEXT:    s_endpgm5029;5030; VI-LABEL: atomic_xchg_i32:5031; VI:       ; %bb.0: ; %entry5032; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c5033; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x245034; VI-NEXT:    s_mov_b32 s3, 0xf0005035; VI-NEXT:    s_mov_b32 s2, -15036; VI-NEXT:    s_waitcnt lgkmcnt(0)5037; VI-NEXT:    v_mov_b32_e32 v0, s65038; VI-NEXT:    buffer_atomic_swap v0, off, s[0:3], 05039; VI-NEXT:    s_waitcnt vmcnt(0)5040; VI-NEXT:    buffer_wbinvl1_vol5041; VI-NEXT:    s_endpgm5042;5043; GFX9-LABEL: atomic_xchg_i32:5044; GFX9:       ; %bb.0: ; %entry5045; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c5046; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x245047; GFX9-NEXT:    v_mov_b32_e32 v0, 05048; GFX9-NEXT:    s_waitcnt lgkmcnt(0)5049; GFX9-NEXT:    v_mov_b32_e32 v1, s25050; GFX9-NEXT:    global_atomic_swap v0, v1, s[0:1]5051; GFX9-NEXT:    s_waitcnt vmcnt(0)5052; GFX9-NEXT:    buffer_wbinvl1_vol5053; GFX9-NEXT:    s_endpgm5054;5055; GFX11-LABEL: atomic_xchg_i32:5056; GFX11:       ; %bb.0: ; %entry5057; GFX11-NEXT:    s_clause 0x15058; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c5059; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x245060; GFX11-NEXT:    s_waitcnt lgkmcnt(0)5061; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s25062; GFX11-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]5063; GFX11-NEXT:    s_waitcnt_vscnt null, 0x05064; GFX11-NEXT:    buffer_gl1_inv5065; GFX11-NEXT:    buffer_gl0_inv5066; GFX11-NEXT:    s_endpgm5067entry:5068  %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst5069  ret void5070}5071 5072define amdgpu_kernel void @atomic_xchg_i32_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) {5073; SI-LABEL: atomic_xchg_i32_ret:5074; SI:       ; %bb.0: ; %entry5075; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x95076; SI-NEXT:    s_load_dword s8, s[4:5], 0xd5077; SI-NEXT:    s_mov_b32 s7, 0xf0005078; SI-NEXT:    s_mov_b32 s6, -15079; SI-NEXT:    s_waitcnt lgkmcnt(0)5080; SI-NEXT:    s_mov_b32 s4, s05081; SI-NEXT:    s_mov_b32 s5, s15082; SI-NEXT:    v_mov_b32_e32 v0, s85083; SI-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc5084; SI-NEXT:    s_waitcnt vmcnt(0)5085; SI-NEXT:    buffer_wbinvl15086; SI-NEXT:    s_mov_b32 s4, s25087; SI-NEXT:    s_mov_b32 s5, s35088; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 05089; SI-NEXT:    s_endpgm5090;5091; VI-LABEL: atomic_xchg_i32_ret:5092; VI:       ; %bb.0: ; %entry5093; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245094; VI-NEXT:    s_load_dword s8, s[4:5], 0x345095; VI-NEXT:    s_mov_b32 s7, 0xf0005096; VI-NEXT:    s_mov_b32 s6, -15097; VI-NEXT:    s_waitcnt lgkmcnt(0)5098; VI-NEXT:    s_mov_b32 s4, s05099; VI-NEXT:    s_mov_b32 s5, s15100; VI-NEXT:    v_mov_b32_e32 v0, s85101; VI-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc5102; VI-NEXT:    s_waitcnt vmcnt(0)5103; VI-NEXT:    buffer_wbinvl1_vol5104; VI-NEXT:    s_mov_b32 s4, s25105; VI-NEXT:    s_mov_b32 s5, s35106; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 05107; VI-NEXT:    s_endpgm5108;5109; GFX9-LABEL: atomic_xchg_i32_ret:5110; GFX9:       ; %bb.0: ; %entry5111; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x345112; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245113; GFX9-NEXT:    v_mov_b32_e32 v0, 05114; GFX9-NEXT:    s_waitcnt lgkmcnt(0)5115; GFX9-NEXT:    v_mov_b32_e32 v1, s65116; GFX9-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc5117; GFX9-NEXT:    s_waitcnt vmcnt(0)5118; GFX9-NEXT:    buffer_wbinvl1_vol5119; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]5120; GFX9-NEXT:    s_endpgm5121;5122; GFX11-LABEL: atomic_xchg_i32_ret:5123; GFX11:       ; %bb.0: ; %entry5124; GFX11-NEXT:    s_clause 0x15125; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x345126; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x245127; GFX11-NEXT:    s_waitcnt lgkmcnt(0)5128; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s65129; GFX11-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc5130; GFX11-NEXT:    s_waitcnt vmcnt(0)5131; GFX11-NEXT:    buffer_gl1_inv5132; GFX11-NEXT:    buffer_gl0_inv5133; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]5134; GFX11-NEXT:    s_endpgm5135entry:5136  %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst5137  store i32 %val, ptr addrspace(1) %out25138  ret void5139}5140 5141define amdgpu_kernel void @atomic_xchg_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) {5142; SI-LABEL: atomic_xchg_i32_addr64:5143; SI:       ; %bb.0: ; %entry5144; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd5145; SI-NEXT:    s_load_dword s8, s[4:5], 0xb5146; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x95147; SI-NEXT:    s_mov_b32 s3, 0xf0005148; SI-NEXT:    s_waitcnt lgkmcnt(0)5149; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 25150; SI-NEXT:    s_mov_b32 s2, 05151; SI-NEXT:    v_mov_b32_e32 v2, s85152; SI-NEXT:    v_mov_b32_e32 v0, s45153; SI-NEXT:    v_mov_b32_e32 v1, s55154; SI-NEXT:    buffer_atomic_swap v2, v[0:1], s[0:3], 0 addr645155; SI-NEXT:    s_waitcnt vmcnt(0)5156; SI-NEXT:    buffer_wbinvl15157; SI-NEXT:    s_endpgm5158;5159; VI-LABEL: atomic_xchg_i32_addr64:5160; VI:       ; %bb.0: ; %entry5161; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x345162; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x245163; VI-NEXT:    s_load_dword s4, s[4:5], 0x2c5164; VI-NEXT:    s_waitcnt lgkmcnt(0)5165; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 25166; VI-NEXT:    s_add_u32 s0, s2, s05167; VI-NEXT:    s_addc_u32 s1, s3, s15168; VI-NEXT:    v_mov_b32_e32 v0, s05169; VI-NEXT:    v_mov_b32_e32 v1, s15170; VI-NEXT:    v_mov_b32_e32 v2, s45171; VI-NEXT:    flat_atomic_swap v[0:1], v25172; VI-NEXT:    s_waitcnt vmcnt(0)5173; VI-NEXT:    buffer_wbinvl1_vol5174; VI-NEXT:    s_endpgm5175;5176; GFX9-LABEL: atomic_xchg_i32_addr64:5177; GFX9:       ; %bb.0: ; %entry5178; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x345179; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x245180; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c5181; GFX9-NEXT:    v_mov_b32_e32 v0, 05182; GFX9-NEXT:    s_waitcnt lgkmcnt(0)5183; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 25184; GFX9-NEXT:    s_add_u32 s0, s2, s05185; GFX9-NEXT:    s_addc_u32 s1, s3, s15186; GFX9-NEXT:    v_mov_b32_e32 v1, s65187; GFX9-NEXT:    global_atomic_swap v0, v1, s[0:1]5188; GFX9-NEXT:    s_waitcnt vmcnt(0)5189; GFX9-NEXT:    buffer_wbinvl1_vol5190; GFX9-NEXT:    s_endpgm5191;5192; GFX11-LABEL: atomic_xchg_i32_addr64:5193; GFX11:       ; %bb.0: ; %entry5194; GFX11-NEXT:    s_clause 0x25195; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x345196; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c5197; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x245198; GFX11-NEXT:    v_mov_b32_e32 v0, 05199; GFX11-NEXT:    s_waitcnt lgkmcnt(0)5200; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 25201; GFX11-NEXT:    v_mov_b32_e32 v1, s65202; GFX11-NEXT:    s_add_u32 s0, s2, s05203; GFX11-NEXT:    s_addc_u32 s1, s3, s15204; GFX11-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]5205; GFX11-NEXT:    s_waitcnt_vscnt null, 0x05206; GFX11-NEXT:    buffer_gl1_inv5207; GFX11-NEXT:    buffer_gl0_inv5208; GFX11-NEXT:    s_endpgm5209entry:5210  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index5211  %val = atomicrmw volatile xchg ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst5212  ret void5213}5214 5215define amdgpu_kernel void @atomic_xchg_i32_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) {5216; SI-LABEL: atomic_xchg_i32_ret_addr64:5217; SI:       ; %bb.0: ; %entry5218; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x95219; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf5220; SI-NEXT:    s_load_dword s6, s[4:5], 0xd5221; SI-NEXT:    s_mov_b32 s7, 0xf0005222; SI-NEXT:    s_waitcnt lgkmcnt(0)5223; SI-NEXT:    s_mov_b32 s4, s25224; SI-NEXT:    s_mov_b32 s5, s35225; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 25226; SI-NEXT:    s_mov_b32 s2, 05227; SI-NEXT:    s_mov_b32 s3, s75228; SI-NEXT:    v_mov_b32_e32 v2, s65229; SI-NEXT:    v_mov_b32_e32 v0, s85230; SI-NEXT:    v_mov_b32_e32 v1, s95231; SI-NEXT:    buffer_atomic_swap v2, v[0:1], s[0:3], 0 addr64 glc5232; SI-NEXT:    s_waitcnt vmcnt(0)5233; SI-NEXT:    buffer_wbinvl15234; SI-NEXT:    s_mov_b32 s6, -15235; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 05236; SI-NEXT:    s_endpgm5237;5238; VI-LABEL: atomic_xchg_i32_ret_addr64:5239; VI:       ; %bb.0: ; %entry5240; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c5241; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245242; VI-NEXT:    s_load_dword s8, s[4:5], 0x345243; VI-NEXT:    s_waitcnt lgkmcnt(0)5244; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 25245; VI-NEXT:    s_add_u32 s0, s0, s45246; VI-NEXT:    s_addc_u32 s1, s1, s55247; VI-NEXT:    v_mov_b32_e32 v0, s05248; VI-NEXT:    v_mov_b32_e32 v1, s15249; VI-NEXT:    v_mov_b32_e32 v2, s85250; VI-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc5251; VI-NEXT:    s_waitcnt vmcnt(0)5252; VI-NEXT:    buffer_wbinvl1_vol5253; VI-NEXT:    s_mov_b32 s7, 0xf0005254; VI-NEXT:    s_mov_b32 s6, -15255; VI-NEXT:    s_mov_b32 s4, s25256; VI-NEXT:    s_mov_b32 s5, s35257; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 05258; VI-NEXT:    s_endpgm5259;5260; GFX9-LABEL: atomic_xchg_i32_ret_addr64:5261; GFX9:       ; %bb.0: ; %entry5262; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c5263; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245264; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x345265; GFX9-NEXT:    v_mov_b32_e32 v0, 05266; GFX9-NEXT:    s_waitcnt lgkmcnt(0)5267; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 25268; GFX9-NEXT:    s_add_u32 s0, s0, s45269; GFX9-NEXT:    s_addc_u32 s1, s1, s55270; GFX9-NEXT:    v_mov_b32_e32 v1, s85271; GFX9-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc5272; GFX9-NEXT:    s_waitcnt vmcnt(0)5273; GFX9-NEXT:    buffer_wbinvl1_vol5274; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]5275; GFX9-NEXT:    s_endpgm5276;5277; GFX11-LABEL: atomic_xchg_i32_ret_addr64:5278; GFX11:       ; %bb.0: ; %entry5279; GFX11-NEXT:    s_clause 0x25280; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c5281; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x345282; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x245283; GFX11-NEXT:    v_mov_b32_e32 v0, 05284; GFX11-NEXT:    s_waitcnt lgkmcnt(0)5285; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 25286; GFX11-NEXT:    v_mov_b32_e32 v1, s85287; GFX11-NEXT:    s_add_u32 s0, s0, s45288; GFX11-NEXT:    s_addc_u32 s1, s1, s55289; GFX11-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc5290; GFX11-NEXT:    s_waitcnt vmcnt(0)5291; GFX11-NEXT:    buffer_gl1_inv5292; GFX11-NEXT:    buffer_gl0_inv5293; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]5294; GFX11-NEXT:    s_endpgm5295entry:5296  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index5297  %val = atomicrmw volatile xchg ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst5298  store i32 %val, ptr addrspace(1) %out25299  ret void5300}5301 5302define amdgpu_kernel void @atomic_cmpxchg_i32_offset(ptr addrspace(1) %out, i32 %in, i32 %old) {5303; SI-LABEL: atomic_cmpxchg_i32_offset:5304; SI:       ; %bb.0: ; %entry5305; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x95306; SI-NEXT:    s_mov_b32 s7, 0xf0005307; SI-NEXT:    s_mov_b32 s6, -15308; SI-NEXT:    s_waitcnt lgkmcnt(0)5309; SI-NEXT:    s_mov_b32 s4, s05310; SI-NEXT:    s_mov_b32 s5, s15311; SI-NEXT:    v_mov_b32_e32 v0, s25312; SI-NEXT:    v_mov_b32_e32 v1, s35313; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:165314; SI-NEXT:    s_waitcnt vmcnt(0)5315; SI-NEXT:    buffer_wbinvl15316; SI-NEXT:    s_endpgm5317;5318; VI-LABEL: atomic_cmpxchg_i32_offset:5319; VI:       ; %bb.0: ; %entry5320; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245321; VI-NEXT:    s_mov_b32 s7, 0xf0005322; VI-NEXT:    s_mov_b32 s6, -15323; VI-NEXT:    s_waitcnt lgkmcnt(0)5324; VI-NEXT:    v_mov_b32_e32 v0, s25325; VI-NEXT:    s_mov_b32 s4, s05326; VI-NEXT:    s_mov_b32 s5, s15327; VI-NEXT:    v_mov_b32_e32 v1, s35328; VI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:165329; VI-NEXT:    s_waitcnt vmcnt(0)5330; VI-NEXT:    buffer_wbinvl1_vol5331; VI-NEXT:    s_endpgm5332;5333; GFX9-LABEL: atomic_cmpxchg_i32_offset:5334; GFX9:       ; %bb.0: ; %entry5335; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245336; GFX9-NEXT:    v_mov_b32_e32 v2, 05337; GFX9-NEXT:    s_waitcnt lgkmcnt(0)5338; GFX9-NEXT:    v_mov_b32_e32 v0, s25339; GFX9-NEXT:    v_mov_b32_e32 v1, s35340; GFX9-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:165341; GFX9-NEXT:    s_waitcnt vmcnt(0)5342; GFX9-NEXT:    buffer_wbinvl1_vol5343; GFX9-NEXT:    s_endpgm5344;5345; GFX11-LABEL: atomic_cmpxchg_i32_offset:5346; GFX11:       ; %bb.0: ; %entry5347; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x245348; GFX11-NEXT:    v_mov_b32_e32 v2, 05349; GFX11-NEXT:    s_waitcnt lgkmcnt(0)5350; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s35351; GFX11-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:165352; GFX11-NEXT:    s_waitcnt_vscnt null, 0x05353; GFX11-NEXT:    buffer_gl1_inv5354; GFX11-NEXT:    buffer_gl0_inv5355; GFX11-NEXT:    s_endpgm5356entry:5357  %gep = getelementptr i32, ptr addrspace(1) %out, i64 45358  %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst5359  ret void5360}5361 5362define amdgpu_kernel void @atomic_cmpxchg_i32_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i32 %old) {5363; SI-LABEL: atomic_cmpxchg_i32_ret_offset:5364; SI:       ; %bb.0: ; %entry5365; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x95366; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd5367; SI-NEXT:    s_mov_b32 s7, 0xf0005368; SI-NEXT:    s_mov_b32 s6, -15369; SI-NEXT:    s_waitcnt lgkmcnt(0)5370; SI-NEXT:    s_mov_b32 s4, s25371; SI-NEXT:    s_mov_b32 s5, s35372; SI-NEXT:    s_mov_b32 s2, s65373; SI-NEXT:    s_mov_b32 s3, s75374; SI-NEXT:    v_mov_b32_e32 v0, s85375; SI-NEXT:    v_mov_b32_e32 v1, s95376; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc5377; SI-NEXT:    s_waitcnt vmcnt(0)5378; SI-NEXT:    buffer_wbinvl15379; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 05380; SI-NEXT:    s_endpgm5381;5382; VI-LABEL: atomic_cmpxchg_i32_ret_offset:5383; VI:       ; %bb.0: ; %entry5384; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245385; VI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x345386; VI-NEXT:    s_mov_b32 s7, 0xf0005387; VI-NEXT:    s_mov_b32 s6, -15388; VI-NEXT:    s_waitcnt lgkmcnt(0)5389; VI-NEXT:    s_mov_b32 s4, s25390; VI-NEXT:    v_mov_b32_e32 v0, s85391; VI-NEXT:    s_mov_b32 s5, s35392; VI-NEXT:    s_mov_b32 s2, s65393; VI-NEXT:    s_mov_b32 s3, s75394; VI-NEXT:    v_mov_b32_e32 v1, s95395; VI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc5396; VI-NEXT:    s_waitcnt vmcnt(0)5397; VI-NEXT:    buffer_wbinvl1_vol5398; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 05399; VI-NEXT:    s_endpgm5400;5401; GFX9-LABEL: atomic_cmpxchg_i32_ret_offset:5402; GFX9:       ; %bb.0: ; %entry5403; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x345404; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245405; GFX9-NEXT:    v_mov_b32_e32 v2, 05406; GFX9-NEXT:    s_waitcnt lgkmcnt(0)5407; GFX9-NEXT:    v_mov_b32_e32 v0, s65408; GFX9-NEXT:    v_mov_b32_e32 v1, s75409; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc5410; GFX9-NEXT:    s_waitcnt vmcnt(0)5411; GFX9-NEXT:    buffer_wbinvl1_vol5412; GFX9-NEXT:    global_store_dword v2, v0, s[2:3]5413; GFX9-NEXT:    s_endpgm5414;5415; GFX11-LABEL: atomic_cmpxchg_i32_ret_offset:5416; GFX11:       ; %bb.0: ; %entry5417; GFX11-NEXT:    s_clause 0x15418; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x345419; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x245420; GFX11-NEXT:    v_mov_b32_e32 v2, 05421; GFX11-NEXT:    s_waitcnt lgkmcnt(0)5422; GFX11-NEXT:    v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s75423; GFX11-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc5424; GFX11-NEXT:    s_waitcnt vmcnt(0)5425; GFX11-NEXT:    buffer_gl1_inv5426; GFX11-NEXT:    buffer_gl0_inv5427; GFX11-NEXT:    global_store_b32 v2, v0, s[2:3]5428; GFX11-NEXT:    s_endpgm5429entry:5430  %gep = getelementptr i32, ptr addrspace(1) %out, i64 45431  %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst5432  %extract0 = extractvalue { i32, i1 } %val, 05433  store i32 %extract0, ptr addrspace(1) %out25434  ret void5435}5436 5437define amdgpu_kernel void @atomic_cmpxchg_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index, i32 %old) {5438; SI-LABEL: atomic_cmpxchg_i32_addr64_offset:5439; SI:       ; %bb.0: ; %entry5440; SI-NEXT:    s_load_dword s8, s[4:5], 0xb5441; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd5442; SI-NEXT:    s_load_dword s9, s[4:5], 0xf5443; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x95444; SI-NEXT:    s_mov_b32 s3, 0xf0005445; SI-NEXT:    s_waitcnt lgkmcnt(0)5446; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 25447; SI-NEXT:    s_mov_b32 s2, 05448; SI-NEXT:    v_mov_b32_e32 v0, s85449; SI-NEXT:    v_mov_b32_e32 v1, s95450; SI-NEXT:    v_mov_b32_e32 v2, s45451; SI-NEXT:    v_mov_b32_e32 v3, s55452; SI-NEXT:    buffer_atomic_cmpswap v[0:1], v[2:3], s[0:3], 0 addr64 offset:165453; SI-NEXT:    s_waitcnt vmcnt(0)5454; SI-NEXT:    buffer_wbinvl15455; SI-NEXT:    s_endpgm5456;5457; VI-LABEL: atomic_cmpxchg_i32_addr64_offset:5458; VI:       ; %bb.0: ; %entry5459; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x345460; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c5461; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x245462; VI-NEXT:    s_load_dword s4, s[4:5], 0x3c5463; VI-NEXT:    s_waitcnt lgkmcnt(0)5464; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 25465; VI-NEXT:    v_mov_b32_e32 v0, s65466; VI-NEXT:    s_add_u32 s0, s2, s05467; VI-NEXT:    s_addc_u32 s1, s3, s15468; VI-NEXT:    s_add_u32 s0, s0, 165469; VI-NEXT:    s_addc_u32 s1, s1, 05470; VI-NEXT:    v_mov_b32_e32 v3, s15471; VI-NEXT:    v_mov_b32_e32 v1, s45472; VI-NEXT:    v_mov_b32_e32 v2, s05473; VI-NEXT:    flat_atomic_cmpswap v[2:3], v[0:1]5474; VI-NEXT:    s_waitcnt vmcnt(0)5475; VI-NEXT:    buffer_wbinvl1_vol5476; VI-NEXT:    s_endpgm5477;5478; GFX9-LABEL: atomic_cmpxchg_i32_addr64_offset:5479; GFX9:       ; %bb.0: ; %entry5480; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x345481; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x245482; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c5483; GFX9-NEXT:    s_load_dword s7, s[4:5], 0x3c5484; GFX9-NEXT:    v_mov_b32_e32 v2, 05485; GFX9-NEXT:    s_waitcnt lgkmcnt(0)5486; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 25487; GFX9-NEXT:    s_add_u32 s0, s2, s05488; GFX9-NEXT:    s_addc_u32 s1, s3, s15489; GFX9-NEXT:    v_mov_b32_e32 v0, s65490; GFX9-NEXT:    v_mov_b32_e32 v1, s75491; GFX9-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:165492; GFX9-NEXT:    s_waitcnt vmcnt(0)5493; GFX9-NEXT:    buffer_wbinvl1_vol5494; GFX9-NEXT:    s_endpgm5495;5496; GFX11-LABEL: atomic_cmpxchg_i32_addr64_offset:5497; GFX11:       ; %bb.0: ; %entry5498; GFX11-NEXT:    s_clause 0x35499; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x345500; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c5501; GFX11-NEXT:    s_load_b32 s7, s[4:5], 0x3c5502; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x245503; GFX11-NEXT:    v_mov_b32_e32 v2, 05504; GFX11-NEXT:    s_waitcnt lgkmcnt(0)5505; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 25506; GFX11-NEXT:    v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s75507; GFX11-NEXT:    s_add_u32 s0, s2, s05508; GFX11-NEXT:    s_addc_u32 s1, s3, s15509; GFX11-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:165510; GFX11-NEXT:    s_waitcnt_vscnt null, 0x05511; GFX11-NEXT:    buffer_gl1_inv5512; GFX11-NEXT:    buffer_gl0_inv5513; GFX11-NEXT:    s_endpgm5514entry:5515  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index5516  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 45517  %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst5518  ret void5519}5520 5521define amdgpu_kernel void @atomic_cmpxchg_i32_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index, i32 %old) {5522; SI-LABEL: atomic_cmpxchg_i32_ret_addr64_offset:5523; SI:       ; %bb.0: ; %entry5524; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x95525; SI-NEXT:    s_load_dword s6, s[4:5], 0xd5526; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf5527; SI-NEXT:    s_load_dword s10, s[4:5], 0x115528; SI-NEXT:    s_mov_b32 s7, 0xf0005529; SI-NEXT:    s_waitcnt lgkmcnt(0)5530; SI-NEXT:    s_mov_b32 s4, s25531; SI-NEXT:    s_mov_b32 s5, s35532; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 25533; SI-NEXT:    s_mov_b32 s2, 05534; SI-NEXT:    s_mov_b32 s3, s75535; SI-NEXT:    v_mov_b32_e32 v0, s65536; SI-NEXT:    v_mov_b32_e32 v1, s105537; SI-NEXT:    v_mov_b32_e32 v2, s85538; SI-NEXT:    v_mov_b32_e32 v3, s95539; SI-NEXT:    buffer_atomic_cmpswap v[0:1], v[2:3], s[0:3], 0 addr64 offset:16 glc5540; SI-NEXT:    s_waitcnt vmcnt(0)5541; SI-NEXT:    buffer_wbinvl15542; SI-NEXT:    s_mov_b32 s6, -15543; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 05544; SI-NEXT:    s_endpgm5545;5546; VI-LABEL: atomic_cmpxchg_i32_ret_addr64_offset:5547; VI:       ; %bb.0: ; %entry5548; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c5549; VI-NEXT:    s_load_dword s8, s[4:5], 0x345550; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245551; VI-NEXT:    s_load_dword s9, s[4:5], 0x445552; VI-NEXT:    s_waitcnt lgkmcnt(0)5553; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 25554; VI-NEXT:    v_mov_b32_e32 v0, s85555; VI-NEXT:    s_add_u32 s0, s0, s45556; VI-NEXT:    s_addc_u32 s1, s1, s55557; VI-NEXT:    s_add_u32 s0, s0, 165558; VI-NEXT:    s_addc_u32 s1, s1, 05559; VI-NEXT:    v_mov_b32_e32 v3, s15560; VI-NEXT:    v_mov_b32_e32 v1, s95561; VI-NEXT:    v_mov_b32_e32 v2, s05562; VI-NEXT:    flat_atomic_cmpswap v0, v[2:3], v[0:1] glc5563; VI-NEXT:    s_waitcnt vmcnt(0)5564; VI-NEXT:    buffer_wbinvl1_vol5565; VI-NEXT:    s_mov_b32 s7, 0xf0005566; VI-NEXT:    s_mov_b32 s6, -15567; VI-NEXT:    s_mov_b32 s4, s25568; VI-NEXT:    s_mov_b32 s5, s35569; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 05570; VI-NEXT:    s_endpgm5571;5572; GFX9-LABEL: atomic_cmpxchg_i32_ret_addr64_offset:5573; GFX9:       ; %bb.0: ; %entry5574; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c5575; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245576; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x345577; GFX9-NEXT:    s_load_dword s9, s[4:5], 0x445578; GFX9-NEXT:    v_mov_b32_e32 v2, 05579; GFX9-NEXT:    s_waitcnt lgkmcnt(0)5580; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 25581; GFX9-NEXT:    s_add_u32 s0, s0, s45582; GFX9-NEXT:    s_addc_u32 s1, s1, s55583; GFX9-NEXT:    v_mov_b32_e32 v0, s85584; GFX9-NEXT:    v_mov_b32_e32 v1, s95585; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc5586; GFX9-NEXT:    s_waitcnt vmcnt(0)5587; GFX9-NEXT:    buffer_wbinvl1_vol5588; GFX9-NEXT:    global_store_dword v2, v0, s[2:3]5589; GFX9-NEXT:    s_endpgm5590;5591; GFX11-LABEL: atomic_cmpxchg_i32_ret_addr64_offset:5592; GFX11:       ; %bb.0: ; %entry5593; GFX11-NEXT:    s_clause 0x35594; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c5595; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x345596; GFX11-NEXT:    s_load_b32 s9, s[4:5], 0x445597; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x245598; GFX11-NEXT:    v_mov_b32_e32 v2, 05599; GFX11-NEXT:    s_waitcnt lgkmcnt(0)5600; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 25601; GFX11-NEXT:    v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s95602; GFX11-NEXT:    s_add_u32 s0, s0, s45603; GFX11-NEXT:    s_addc_u32 s1, s1, s55604; GFX11-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc5605; GFX11-NEXT:    s_waitcnt vmcnt(0)5606; GFX11-NEXT:    buffer_gl1_inv5607; GFX11-NEXT:    buffer_gl0_inv5608; GFX11-NEXT:    global_store_b32 v2, v0, s[2:3]5609; GFX11-NEXT:    s_endpgm5610entry:5611  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index5612  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 45613  %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst5614  %extract0 = extractvalue { i32, i1 } %val, 05615  store i32 %extract0, ptr addrspace(1) %out25616  ret void5617}5618 5619define amdgpu_kernel void @atomic_cmpxchg_i32(ptr addrspace(1) %out, i32 %in, i32 %old) {5620; SI-LABEL: atomic_cmpxchg_i32:5621; SI:       ; %bb.0: ; %entry5622; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x95623; SI-NEXT:    s_mov_b32 s7, 0xf0005624; SI-NEXT:    s_mov_b32 s6, -15625; SI-NEXT:    s_waitcnt lgkmcnt(0)5626; SI-NEXT:    s_mov_b32 s4, s05627; SI-NEXT:    s_mov_b32 s5, s15628; SI-NEXT:    v_mov_b32_e32 v0, s25629; SI-NEXT:    v_mov_b32_e32 v1, s35630; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 05631; SI-NEXT:    s_waitcnt vmcnt(0)5632; SI-NEXT:    buffer_wbinvl15633; SI-NEXT:    s_endpgm5634;5635; VI-LABEL: atomic_cmpxchg_i32:5636; VI:       ; %bb.0: ; %entry5637; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245638; VI-NEXT:    s_mov_b32 s7, 0xf0005639; VI-NEXT:    s_mov_b32 s6, -15640; VI-NEXT:    s_waitcnt lgkmcnt(0)5641; VI-NEXT:    v_mov_b32_e32 v0, s25642; VI-NEXT:    s_mov_b32 s4, s05643; VI-NEXT:    s_mov_b32 s5, s15644; VI-NEXT:    v_mov_b32_e32 v1, s35645; VI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 05646; VI-NEXT:    s_waitcnt vmcnt(0)5647; VI-NEXT:    buffer_wbinvl1_vol5648; VI-NEXT:    s_endpgm5649;5650; GFX9-LABEL: atomic_cmpxchg_i32:5651; GFX9:       ; %bb.0: ; %entry5652; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245653; GFX9-NEXT:    v_mov_b32_e32 v2, 05654; GFX9-NEXT:    s_waitcnt lgkmcnt(0)5655; GFX9-NEXT:    v_mov_b32_e32 v0, s25656; GFX9-NEXT:    v_mov_b32_e32 v1, s35657; GFX9-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1]5658; GFX9-NEXT:    s_waitcnt vmcnt(0)5659; GFX9-NEXT:    buffer_wbinvl1_vol5660; GFX9-NEXT:    s_endpgm5661;5662; GFX11-LABEL: atomic_cmpxchg_i32:5663; GFX11:       ; %bb.0: ; %entry5664; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x245665; GFX11-NEXT:    v_mov_b32_e32 v2, 05666; GFX11-NEXT:    s_waitcnt lgkmcnt(0)5667; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s35668; GFX11-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1]5669; GFX11-NEXT:    s_waitcnt_vscnt null, 0x05670; GFX11-NEXT:    buffer_gl1_inv5671; GFX11-NEXT:    buffer_gl0_inv5672; GFX11-NEXT:    s_endpgm5673entry:5674  %val = cmpxchg volatile ptr addrspace(1) %out, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst5675  ret void5676}5677 5678define amdgpu_kernel void @atomic_cmpxchg_i32_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i32 %old) {5679; SI-LABEL: atomic_cmpxchg_i32_ret:5680; SI:       ; %bb.0: ; %entry5681; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x95682; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd5683; SI-NEXT:    s_mov_b32 s7, 0xf0005684; SI-NEXT:    s_mov_b32 s6, -15685; SI-NEXT:    s_waitcnt lgkmcnt(0)5686; SI-NEXT:    s_mov_b32 s4, s05687; SI-NEXT:    s_mov_b32 s5, s15688; SI-NEXT:    v_mov_b32_e32 v0, s85689; SI-NEXT:    v_mov_b32_e32 v1, s95690; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc5691; SI-NEXT:    s_waitcnt vmcnt(0)5692; SI-NEXT:    buffer_wbinvl15693; SI-NEXT:    s_mov_b32 s4, s25694; SI-NEXT:    s_mov_b32 s5, s35695; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 05696; SI-NEXT:    s_endpgm5697;5698; VI-LABEL: atomic_cmpxchg_i32_ret:5699; VI:       ; %bb.0: ; %entry5700; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245701; VI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x345702; VI-NEXT:    s_mov_b32 s7, 0xf0005703; VI-NEXT:    s_mov_b32 s6, -15704; VI-NEXT:    s_waitcnt lgkmcnt(0)5705; VI-NEXT:    s_mov_b32 s4, s05706; VI-NEXT:    v_mov_b32_e32 v0, s85707; VI-NEXT:    s_mov_b32 s5, s15708; VI-NEXT:    v_mov_b32_e32 v1, s95709; VI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc5710; VI-NEXT:    s_waitcnt vmcnt(0)5711; VI-NEXT:    buffer_wbinvl1_vol5712; VI-NEXT:    s_mov_b32 s4, s25713; VI-NEXT:    s_mov_b32 s5, s35714; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 05715; VI-NEXT:    s_endpgm5716;5717; GFX9-LABEL: atomic_cmpxchg_i32_ret:5718; GFX9:       ; %bb.0: ; %entry5719; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x345720; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245721; GFX9-NEXT:    v_mov_b32_e32 v2, 05722; GFX9-NEXT:    s_waitcnt lgkmcnt(0)5723; GFX9-NEXT:    v_mov_b32_e32 v0, s65724; GFX9-NEXT:    v_mov_b32_e32 v1, s75725; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc5726; GFX9-NEXT:    s_waitcnt vmcnt(0)5727; GFX9-NEXT:    buffer_wbinvl1_vol5728; GFX9-NEXT:    global_store_dword v2, v0, s[2:3]5729; GFX9-NEXT:    s_endpgm5730;5731; GFX11-LABEL: atomic_cmpxchg_i32_ret:5732; GFX11:       ; %bb.0: ; %entry5733; GFX11-NEXT:    s_clause 0x15734; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x345735; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x245736; GFX11-NEXT:    v_mov_b32_e32 v2, 05737; GFX11-NEXT:    s_waitcnt lgkmcnt(0)5738; GFX11-NEXT:    v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s75739; GFX11-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc5740; GFX11-NEXT:    s_waitcnt vmcnt(0)5741; GFX11-NEXT:    buffer_gl1_inv5742; GFX11-NEXT:    buffer_gl0_inv5743; GFX11-NEXT:    global_store_b32 v2, v0, s[2:3]5744; GFX11-NEXT:    s_endpgm5745entry:5746  %val = cmpxchg volatile ptr addrspace(1) %out, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst5747  %extract0 = extractvalue { i32, i1 } %val, 05748  store i32 %extract0, ptr addrspace(1) %out25749  ret void5750}5751 5752define amdgpu_kernel void @atomic_cmpxchg_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index, i32 %old) {5753; SI-LABEL: atomic_cmpxchg_i32_addr64:5754; SI:       ; %bb.0: ; %entry5755; SI-NEXT:    s_load_dword s8, s[4:5], 0xb5756; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd5757; SI-NEXT:    s_load_dword s9, s[4:5], 0xf5758; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x95759; SI-NEXT:    s_mov_b32 s3, 0xf0005760; SI-NEXT:    s_waitcnt lgkmcnt(0)5761; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 25762; SI-NEXT:    s_mov_b32 s2, 05763; SI-NEXT:    v_mov_b32_e32 v0, s85764; SI-NEXT:    v_mov_b32_e32 v1, s95765; SI-NEXT:    v_mov_b32_e32 v2, s45766; SI-NEXT:    v_mov_b32_e32 v3, s55767; SI-NEXT:    buffer_atomic_cmpswap v[0:1], v[2:3], s[0:3], 0 addr645768; SI-NEXT:    s_waitcnt vmcnt(0)5769; SI-NEXT:    buffer_wbinvl15770; SI-NEXT:    s_endpgm5771;5772; VI-LABEL: atomic_cmpxchg_i32_addr64:5773; VI:       ; %bb.0: ; %entry5774; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x345775; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c5776; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x245777; VI-NEXT:    s_load_dword s4, s[4:5], 0x3c5778; VI-NEXT:    s_waitcnt lgkmcnt(0)5779; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 25780; VI-NEXT:    v_mov_b32_e32 v0, s65781; VI-NEXT:    s_add_u32 s0, s2, s05782; VI-NEXT:    s_addc_u32 s1, s3, s15783; VI-NEXT:    v_mov_b32_e32 v3, s15784; VI-NEXT:    v_mov_b32_e32 v1, s45785; VI-NEXT:    v_mov_b32_e32 v2, s05786; VI-NEXT:    flat_atomic_cmpswap v[2:3], v[0:1]5787; VI-NEXT:    s_waitcnt vmcnt(0)5788; VI-NEXT:    buffer_wbinvl1_vol5789; VI-NEXT:    s_endpgm5790;5791; GFX9-LABEL: atomic_cmpxchg_i32_addr64:5792; GFX9:       ; %bb.0: ; %entry5793; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x345794; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x245795; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c5796; GFX9-NEXT:    s_load_dword s7, s[4:5], 0x3c5797; GFX9-NEXT:    v_mov_b32_e32 v2, 05798; GFX9-NEXT:    s_waitcnt lgkmcnt(0)5799; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 25800; GFX9-NEXT:    s_add_u32 s0, s2, s05801; GFX9-NEXT:    s_addc_u32 s1, s3, s15802; GFX9-NEXT:    v_mov_b32_e32 v0, s65803; GFX9-NEXT:    v_mov_b32_e32 v1, s75804; GFX9-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1]5805; GFX9-NEXT:    s_waitcnt vmcnt(0)5806; GFX9-NEXT:    buffer_wbinvl1_vol5807; GFX9-NEXT:    s_endpgm5808;5809; GFX11-LABEL: atomic_cmpxchg_i32_addr64:5810; GFX11:       ; %bb.0: ; %entry5811; GFX11-NEXT:    s_clause 0x35812; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x345813; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c5814; GFX11-NEXT:    s_load_b32 s7, s[4:5], 0x3c5815; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x245816; GFX11-NEXT:    v_mov_b32_e32 v2, 05817; GFX11-NEXT:    s_waitcnt lgkmcnt(0)5818; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 25819; GFX11-NEXT:    v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s75820; GFX11-NEXT:    s_add_u32 s0, s2, s05821; GFX11-NEXT:    s_addc_u32 s1, s3, s15822; GFX11-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1]5823; GFX11-NEXT:    s_waitcnt_vscnt null, 0x05824; GFX11-NEXT:    buffer_gl1_inv5825; GFX11-NEXT:    buffer_gl0_inv5826; GFX11-NEXT:    s_endpgm5827entry:5828  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index5829  %val = cmpxchg volatile ptr addrspace(1) %ptr, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst5830  ret void5831}5832 5833define amdgpu_kernel void @atomic_cmpxchg_i32_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index, i32 %old) {5834; SI-LABEL: atomic_cmpxchg_i32_ret_addr64:5835; SI:       ; %bb.0: ; %entry5836; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x95837; SI-NEXT:    s_load_dword s6, s[4:5], 0xd5838; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf5839; SI-NEXT:    s_load_dword s10, s[4:5], 0x115840; SI-NEXT:    s_mov_b32 s7, 0xf0005841; SI-NEXT:    s_waitcnt lgkmcnt(0)5842; SI-NEXT:    s_mov_b32 s4, s25843; SI-NEXT:    s_mov_b32 s5, s35844; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 25845; SI-NEXT:    s_mov_b32 s2, 05846; SI-NEXT:    s_mov_b32 s3, s75847; SI-NEXT:    v_mov_b32_e32 v0, s65848; SI-NEXT:    v_mov_b32_e32 v1, s105849; SI-NEXT:    v_mov_b32_e32 v2, s85850; SI-NEXT:    v_mov_b32_e32 v3, s95851; SI-NEXT:    buffer_atomic_cmpswap v[0:1], v[2:3], s[0:3], 0 addr64 glc5852; SI-NEXT:    s_waitcnt vmcnt(0)5853; SI-NEXT:    buffer_wbinvl15854; SI-NEXT:    s_mov_b32 s6, -15855; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 05856; SI-NEXT:    s_endpgm5857;5858; VI-LABEL: atomic_cmpxchg_i32_ret_addr64:5859; VI:       ; %bb.0: ; %entry5860; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c5861; VI-NEXT:    s_load_dword s8, s[4:5], 0x345862; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245863; VI-NEXT:    s_load_dword s9, s[4:5], 0x445864; VI-NEXT:    s_waitcnt lgkmcnt(0)5865; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 25866; VI-NEXT:    v_mov_b32_e32 v0, s85867; VI-NEXT:    s_add_u32 s0, s0, s45868; VI-NEXT:    s_addc_u32 s1, s1, s55869; VI-NEXT:    v_mov_b32_e32 v3, s15870; VI-NEXT:    v_mov_b32_e32 v1, s95871; VI-NEXT:    v_mov_b32_e32 v2, s05872; VI-NEXT:    flat_atomic_cmpswap v0, v[2:3], v[0:1] glc5873; VI-NEXT:    s_waitcnt vmcnt(0)5874; VI-NEXT:    buffer_wbinvl1_vol5875; VI-NEXT:    s_mov_b32 s7, 0xf0005876; VI-NEXT:    s_mov_b32 s6, -15877; VI-NEXT:    s_mov_b32 s4, s25878; VI-NEXT:    s_mov_b32 s5, s35879; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 05880; VI-NEXT:    s_endpgm5881;5882; GFX9-LABEL: atomic_cmpxchg_i32_ret_addr64:5883; GFX9:       ; %bb.0: ; %entry5884; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c5885; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245886; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x345887; GFX9-NEXT:    s_load_dword s9, s[4:5], 0x445888; GFX9-NEXT:    v_mov_b32_e32 v2, 05889; GFX9-NEXT:    s_waitcnt lgkmcnt(0)5890; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 25891; GFX9-NEXT:    s_add_u32 s0, s0, s45892; GFX9-NEXT:    s_addc_u32 s1, s1, s55893; GFX9-NEXT:    v_mov_b32_e32 v0, s85894; GFX9-NEXT:    v_mov_b32_e32 v1, s95895; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc5896; GFX9-NEXT:    s_waitcnt vmcnt(0)5897; GFX9-NEXT:    buffer_wbinvl1_vol5898; GFX9-NEXT:    global_store_dword v2, v0, s[2:3]5899; GFX9-NEXT:    s_endpgm5900;5901; GFX11-LABEL: atomic_cmpxchg_i32_ret_addr64:5902; GFX11:       ; %bb.0: ; %entry5903; GFX11-NEXT:    s_clause 0x35904; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c5905; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x345906; GFX11-NEXT:    s_load_b32 s9, s[4:5], 0x445907; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x245908; GFX11-NEXT:    v_mov_b32_e32 v2, 05909; GFX11-NEXT:    s_waitcnt lgkmcnt(0)5910; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 25911; GFX11-NEXT:    v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s95912; GFX11-NEXT:    s_add_u32 s0, s0, s45913; GFX11-NEXT:    s_addc_u32 s1, s1, s55914; GFX11-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc5915; GFX11-NEXT:    s_waitcnt vmcnt(0)5916; GFX11-NEXT:    buffer_gl1_inv5917; GFX11-NEXT:    buffer_gl0_inv5918; GFX11-NEXT:    global_store_b32 v2, v0, s[2:3]5919; GFX11-NEXT:    s_endpgm5920entry:5921  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index5922  %val = cmpxchg volatile ptr addrspace(1) %ptr, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst5923  %extract0 = extractvalue { i32, i1 } %val, 05924  store i32 %extract0, ptr addrspace(1) %out25925  ret void5926}5927 5928define amdgpu_kernel void @atomic_xor_i32_offset(ptr addrspace(1) %out, i32 %in) {5929; SI-LABEL: atomic_xor_i32_offset:5930; SI:       ; %bb.0: ; %entry5931; SI-NEXT:    s_load_dword s6, s[4:5], 0xb5932; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x95933; SI-NEXT:    s_mov_b32 s3, 0xf0005934; SI-NEXT:    s_mov_b32 s2, -15935; SI-NEXT:    s_waitcnt lgkmcnt(0)5936; SI-NEXT:    v_mov_b32_e32 v0, s65937; SI-NEXT:    buffer_atomic_xor v0, off, s[0:3], 0 offset:165938; SI-NEXT:    s_waitcnt vmcnt(0)5939; SI-NEXT:    buffer_wbinvl15940; SI-NEXT:    s_endpgm5941;5942; VI-LABEL: atomic_xor_i32_offset:5943; VI:       ; %bb.0: ; %entry5944; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c5945; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x245946; VI-NEXT:    s_mov_b32 s3, 0xf0005947; VI-NEXT:    s_mov_b32 s2, -15948; VI-NEXT:    s_waitcnt lgkmcnt(0)5949; VI-NEXT:    v_mov_b32_e32 v0, s65950; VI-NEXT:    buffer_atomic_xor v0, off, s[0:3], 0 offset:165951; VI-NEXT:    s_waitcnt vmcnt(0)5952; VI-NEXT:    buffer_wbinvl1_vol5953; VI-NEXT:    s_endpgm5954;5955; GFX9-LABEL: atomic_xor_i32_offset:5956; GFX9:       ; %bb.0: ; %entry5957; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c5958; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x245959; GFX9-NEXT:    v_mov_b32_e32 v0, 05960; GFX9-NEXT:    s_waitcnt lgkmcnt(0)5961; GFX9-NEXT:    v_mov_b32_e32 v1, s25962; GFX9-NEXT:    global_atomic_xor v0, v1, s[0:1] offset:165963; GFX9-NEXT:    s_waitcnt vmcnt(0)5964; GFX9-NEXT:    buffer_wbinvl1_vol5965; GFX9-NEXT:    s_endpgm5966;5967; GFX11-LABEL: atomic_xor_i32_offset:5968; GFX11:       ; %bb.0: ; %entry5969; GFX11-NEXT:    s_clause 0x15970; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c5971; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x245972; GFX11-NEXT:    s_waitcnt lgkmcnt(0)5973; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s25974; GFX11-NEXT:    global_atomic_xor_b32 v0, v1, s[0:1] offset:165975; GFX11-NEXT:    s_waitcnt_vscnt null, 0x05976; GFX11-NEXT:    buffer_gl1_inv5977; GFX11-NEXT:    buffer_gl0_inv5978; GFX11-NEXT:    s_endpgm5979entry:5980  %gep = getelementptr i32, ptr addrspace(1) %out, i64 45981  %val = atomicrmw volatile xor ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !05982  ret void5983}5984 5985define amdgpu_kernel void @atomic_xor_i32_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) {5986; SI-LABEL: atomic_xor_i32_ret_offset:5987; SI:       ; %bb.0: ; %entry5988; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x95989; SI-NEXT:    s_load_dword s8, s[4:5], 0xd5990; SI-NEXT:    s_mov_b32 s7, 0xf0005991; SI-NEXT:    s_mov_b32 s6, -15992; SI-NEXT:    s_waitcnt lgkmcnt(0)5993; SI-NEXT:    s_mov_b32 s4, s25994; SI-NEXT:    s_mov_b32 s5, s35995; SI-NEXT:    s_mov_b32 s2, s65996; SI-NEXT:    s_mov_b32 s3, s75997; SI-NEXT:    v_mov_b32_e32 v0, s85998; SI-NEXT:    buffer_atomic_xor v0, off, s[0:3], 0 offset:16 glc5999; SI-NEXT:    s_waitcnt vmcnt(0)6000; SI-NEXT:    buffer_wbinvl16001; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 06002; SI-NEXT:    s_endpgm6003;6004; VI-LABEL: atomic_xor_i32_ret_offset:6005; VI:       ; %bb.0: ; %entry6006; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246007; VI-NEXT:    s_load_dword s8, s[4:5], 0x346008; VI-NEXT:    s_mov_b32 s7, 0xf0006009; VI-NEXT:    s_mov_b32 s6, -16010; VI-NEXT:    s_waitcnt lgkmcnt(0)6011; VI-NEXT:    s_mov_b32 s4, s26012; VI-NEXT:    s_mov_b32 s5, s36013; VI-NEXT:    s_mov_b32 s2, s66014; VI-NEXT:    s_mov_b32 s3, s76015; VI-NEXT:    v_mov_b32_e32 v0, s86016; VI-NEXT:    buffer_atomic_xor v0, off, s[0:3], 0 offset:16 glc6017; VI-NEXT:    s_waitcnt vmcnt(0)6018; VI-NEXT:    buffer_wbinvl1_vol6019; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 06020; VI-NEXT:    s_endpgm6021;6022; GFX9-LABEL: atomic_xor_i32_ret_offset:6023; GFX9:       ; %bb.0: ; %entry6024; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x346025; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246026; GFX9-NEXT:    v_mov_b32_e32 v0, 06027; GFX9-NEXT:    s_waitcnt lgkmcnt(0)6028; GFX9-NEXT:    v_mov_b32_e32 v1, s66029; GFX9-NEXT:    global_atomic_xor v1, v0, v1, s[0:1] offset:16 glc6030; GFX9-NEXT:    s_waitcnt vmcnt(0)6031; GFX9-NEXT:    buffer_wbinvl1_vol6032; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]6033; GFX9-NEXT:    s_endpgm6034;6035; GFX11-LABEL: atomic_xor_i32_ret_offset:6036; GFX11:       ; %bb.0: ; %entry6037; GFX11-NEXT:    s_clause 0x16038; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x346039; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246040; GFX11-NEXT:    s_waitcnt lgkmcnt(0)6041; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s66042; GFX11-NEXT:    global_atomic_xor_b32 v1, v0, v1, s[0:1] offset:16 glc6043; GFX11-NEXT:    s_waitcnt vmcnt(0)6044; GFX11-NEXT:    buffer_gl1_inv6045; GFX11-NEXT:    buffer_gl0_inv6046; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]6047; GFX11-NEXT:    s_endpgm6048entry:6049  %gep = getelementptr i32, ptr addrspace(1) %out, i64 46050  %val = atomicrmw volatile xor ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !06051  store i32 %val, ptr addrspace(1) %out26052  ret void6053}6054 6055define amdgpu_kernel void @atomic_xor_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {6056; SI-LABEL: atomic_xor_i32_addr64_offset:6057; SI:       ; %bb.0: ; %entry6058; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd6059; SI-NEXT:    s_load_dword s8, s[4:5], 0xb6060; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x96061; SI-NEXT:    s_mov_b32 s3, 0xf0006062; SI-NEXT:    s_waitcnt lgkmcnt(0)6063; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 26064; SI-NEXT:    s_mov_b32 s2, 06065; SI-NEXT:    v_mov_b32_e32 v2, s86066; SI-NEXT:    v_mov_b32_e32 v0, s46067; SI-NEXT:    v_mov_b32_e32 v1, s56068; SI-NEXT:    buffer_atomic_xor v2, v[0:1], s[0:3], 0 addr64 offset:166069; SI-NEXT:    s_waitcnt vmcnt(0)6070; SI-NEXT:    buffer_wbinvl16071; SI-NEXT:    s_endpgm6072;6073; VI-LABEL: atomic_xor_i32_addr64_offset:6074; VI:       ; %bb.0: ; %entry6075; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x346076; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x246077; VI-NEXT:    s_load_dword s4, s[4:5], 0x2c6078; VI-NEXT:    s_waitcnt lgkmcnt(0)6079; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 26080; VI-NEXT:    s_add_u32 s0, s2, s06081; VI-NEXT:    s_addc_u32 s1, s3, s16082; VI-NEXT:    s_add_u32 s0, s0, 166083; VI-NEXT:    s_addc_u32 s1, s1, 06084; VI-NEXT:    v_mov_b32_e32 v0, s06085; VI-NEXT:    v_mov_b32_e32 v1, s16086; VI-NEXT:    v_mov_b32_e32 v2, s46087; VI-NEXT:    flat_atomic_xor v[0:1], v26088; VI-NEXT:    s_waitcnt vmcnt(0)6089; VI-NEXT:    buffer_wbinvl1_vol6090; VI-NEXT:    s_endpgm6091;6092; GFX9-LABEL: atomic_xor_i32_addr64_offset:6093; GFX9:       ; %bb.0: ; %entry6094; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x346095; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x246096; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c6097; GFX9-NEXT:    v_mov_b32_e32 v0, 06098; GFX9-NEXT:    s_waitcnt lgkmcnt(0)6099; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 26100; GFX9-NEXT:    s_add_u32 s0, s2, s06101; GFX9-NEXT:    s_addc_u32 s1, s3, s16102; GFX9-NEXT:    v_mov_b32_e32 v1, s66103; GFX9-NEXT:    global_atomic_xor v0, v1, s[0:1] offset:166104; GFX9-NEXT:    s_waitcnt vmcnt(0)6105; GFX9-NEXT:    buffer_wbinvl1_vol6106; GFX9-NEXT:    s_endpgm6107;6108; GFX11-LABEL: atomic_xor_i32_addr64_offset:6109; GFX11:       ; %bb.0: ; %entry6110; GFX11-NEXT:    s_clause 0x26111; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x346112; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c6113; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x246114; GFX11-NEXT:    v_mov_b32_e32 v0, 06115; GFX11-NEXT:    s_waitcnt lgkmcnt(0)6116; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 26117; GFX11-NEXT:    v_mov_b32_e32 v1, s66118; GFX11-NEXT:    s_add_u32 s0, s2, s06119; GFX11-NEXT:    s_addc_u32 s1, s3, s16120; GFX11-NEXT:    global_atomic_xor_b32 v0, v1, s[0:1] offset:166121; GFX11-NEXT:    s_waitcnt_vscnt null, 0x06122; GFX11-NEXT:    buffer_gl1_inv6123; GFX11-NEXT:    buffer_gl0_inv6124; GFX11-NEXT:    s_endpgm6125entry:6126  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index6127  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 46128  %val = atomicrmw volatile xor ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !06129  ret void6130}6131 6132define amdgpu_kernel void @atomic_xor_i32_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) {6133; SI-LABEL: atomic_xor_i32_ret_addr64_offset:6134; SI:       ; %bb.0: ; %entry6135; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96136; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf6137; SI-NEXT:    s_load_dword s6, s[4:5], 0xd6138; SI-NEXT:    s_mov_b32 s7, 0xf0006139; SI-NEXT:    s_waitcnt lgkmcnt(0)6140; SI-NEXT:    s_mov_b32 s4, s26141; SI-NEXT:    s_mov_b32 s5, s36142; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 26143; SI-NEXT:    s_mov_b32 s2, 06144; SI-NEXT:    s_mov_b32 s3, s76145; SI-NEXT:    v_mov_b32_e32 v2, s66146; SI-NEXT:    v_mov_b32_e32 v0, s86147; SI-NEXT:    v_mov_b32_e32 v1, s96148; SI-NEXT:    buffer_atomic_xor v2, v[0:1], s[0:3], 0 addr64 offset:16 glc6149; SI-NEXT:    s_waitcnt vmcnt(0)6150; SI-NEXT:    buffer_wbinvl16151; SI-NEXT:    s_mov_b32 s6, -16152; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 06153; SI-NEXT:    s_endpgm6154;6155; VI-LABEL: atomic_xor_i32_ret_addr64_offset:6156; VI:       ; %bb.0: ; %entry6157; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c6158; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246159; VI-NEXT:    s_load_dword s8, s[4:5], 0x346160; VI-NEXT:    s_waitcnt lgkmcnt(0)6161; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 26162; VI-NEXT:    s_add_u32 s0, s0, s46163; VI-NEXT:    s_addc_u32 s1, s1, s56164; VI-NEXT:    s_add_u32 s0, s0, 166165; VI-NEXT:    s_addc_u32 s1, s1, 06166; VI-NEXT:    v_mov_b32_e32 v0, s06167; VI-NEXT:    v_mov_b32_e32 v1, s16168; VI-NEXT:    v_mov_b32_e32 v2, s86169; VI-NEXT:    flat_atomic_xor v0, v[0:1], v2 glc6170; VI-NEXT:    s_waitcnt vmcnt(0)6171; VI-NEXT:    buffer_wbinvl1_vol6172; VI-NEXT:    s_mov_b32 s7, 0xf0006173; VI-NEXT:    s_mov_b32 s6, -16174; VI-NEXT:    s_mov_b32 s4, s26175; VI-NEXT:    s_mov_b32 s5, s36176; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 06177; VI-NEXT:    s_endpgm6178;6179; GFX9-LABEL: atomic_xor_i32_ret_addr64_offset:6180; GFX9:       ; %bb.0: ; %entry6181; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c6182; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246183; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x346184; GFX9-NEXT:    v_mov_b32_e32 v0, 06185; GFX9-NEXT:    s_waitcnt lgkmcnt(0)6186; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 26187; GFX9-NEXT:    s_add_u32 s0, s0, s46188; GFX9-NEXT:    s_addc_u32 s1, s1, s56189; GFX9-NEXT:    v_mov_b32_e32 v1, s86190; GFX9-NEXT:    global_atomic_xor v1, v0, v1, s[0:1] offset:16 glc6191; GFX9-NEXT:    s_waitcnt vmcnt(0)6192; GFX9-NEXT:    buffer_wbinvl1_vol6193; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]6194; GFX9-NEXT:    s_endpgm6195;6196; GFX11-LABEL: atomic_xor_i32_ret_addr64_offset:6197; GFX11:       ; %bb.0: ; %entry6198; GFX11-NEXT:    s_clause 0x26199; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c6200; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x346201; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246202; GFX11-NEXT:    v_mov_b32_e32 v0, 06203; GFX11-NEXT:    s_waitcnt lgkmcnt(0)6204; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 26205; GFX11-NEXT:    v_mov_b32_e32 v1, s86206; GFX11-NEXT:    s_add_u32 s0, s0, s46207; GFX11-NEXT:    s_addc_u32 s1, s1, s56208; GFX11-NEXT:    global_atomic_xor_b32 v1, v0, v1, s[0:1] offset:16 glc6209; GFX11-NEXT:    s_waitcnt vmcnt(0)6210; GFX11-NEXT:    buffer_gl1_inv6211; GFX11-NEXT:    buffer_gl0_inv6212; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]6213; GFX11-NEXT:    s_endpgm6214entry:6215  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index6216  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 46217  %val = atomicrmw volatile xor ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !06218  store i32 %val, ptr addrspace(1) %out26219  ret void6220}6221 6222define amdgpu_kernel void @atomic_xor_i32(ptr addrspace(1) %out, i32 %in) {6223; SI-LABEL: atomic_xor_i32:6224; SI:       ; %bb.0: ; %entry6225; SI-NEXT:    s_load_dword s6, s[4:5], 0xb6226; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x96227; SI-NEXT:    s_mov_b32 s3, 0xf0006228; SI-NEXT:    s_mov_b32 s2, -16229; SI-NEXT:    s_waitcnt lgkmcnt(0)6230; SI-NEXT:    v_mov_b32_e32 v0, s66231; SI-NEXT:    buffer_atomic_xor v0, off, s[0:3], 06232; SI-NEXT:    s_waitcnt vmcnt(0)6233; SI-NEXT:    buffer_wbinvl16234; SI-NEXT:    s_endpgm6235;6236; VI-LABEL: atomic_xor_i32:6237; VI:       ; %bb.0: ; %entry6238; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c6239; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x246240; VI-NEXT:    s_mov_b32 s3, 0xf0006241; VI-NEXT:    s_mov_b32 s2, -16242; VI-NEXT:    s_waitcnt lgkmcnt(0)6243; VI-NEXT:    v_mov_b32_e32 v0, s66244; VI-NEXT:    buffer_atomic_xor v0, off, s[0:3], 06245; VI-NEXT:    s_waitcnt vmcnt(0)6246; VI-NEXT:    buffer_wbinvl1_vol6247; VI-NEXT:    s_endpgm6248;6249; GFX9-LABEL: atomic_xor_i32:6250; GFX9:       ; %bb.0: ; %entry6251; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c6252; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x246253; GFX9-NEXT:    v_mov_b32_e32 v0, 06254; GFX9-NEXT:    s_waitcnt lgkmcnt(0)6255; GFX9-NEXT:    v_mov_b32_e32 v1, s26256; GFX9-NEXT:    global_atomic_xor v0, v1, s[0:1]6257; GFX9-NEXT:    s_waitcnt vmcnt(0)6258; GFX9-NEXT:    buffer_wbinvl1_vol6259; GFX9-NEXT:    s_endpgm6260;6261; GFX11-LABEL: atomic_xor_i32:6262; GFX11:       ; %bb.0: ; %entry6263; GFX11-NEXT:    s_clause 0x16264; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c6265; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x246266; GFX11-NEXT:    s_waitcnt lgkmcnt(0)6267; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s26268; GFX11-NEXT:    global_atomic_xor_b32 v0, v1, s[0:1]6269; GFX11-NEXT:    s_waitcnt_vscnt null, 0x06270; GFX11-NEXT:    buffer_gl1_inv6271; GFX11-NEXT:    buffer_gl0_inv6272; GFX11-NEXT:    s_endpgm6273entry:6274  %val = atomicrmw volatile xor ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !06275  ret void6276}6277 6278define amdgpu_kernel void @atomic_xor_i32_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) {6279; SI-LABEL: atomic_xor_i32_ret:6280; SI:       ; %bb.0: ; %entry6281; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96282; SI-NEXT:    s_load_dword s8, s[4:5], 0xd6283; SI-NEXT:    s_mov_b32 s7, 0xf0006284; SI-NEXT:    s_mov_b32 s6, -16285; SI-NEXT:    s_waitcnt lgkmcnt(0)6286; SI-NEXT:    s_mov_b32 s4, s06287; SI-NEXT:    s_mov_b32 s5, s16288; SI-NEXT:    v_mov_b32_e32 v0, s86289; SI-NEXT:    buffer_atomic_xor v0, off, s[4:7], 0 glc6290; SI-NEXT:    s_waitcnt vmcnt(0)6291; SI-NEXT:    buffer_wbinvl16292; SI-NEXT:    s_mov_b32 s4, s26293; SI-NEXT:    s_mov_b32 s5, s36294; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 06295; SI-NEXT:    s_endpgm6296;6297; VI-LABEL: atomic_xor_i32_ret:6298; VI:       ; %bb.0: ; %entry6299; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246300; VI-NEXT:    s_load_dword s8, s[4:5], 0x346301; VI-NEXT:    s_mov_b32 s7, 0xf0006302; VI-NEXT:    s_mov_b32 s6, -16303; VI-NEXT:    s_waitcnt lgkmcnt(0)6304; VI-NEXT:    s_mov_b32 s4, s06305; VI-NEXT:    s_mov_b32 s5, s16306; VI-NEXT:    v_mov_b32_e32 v0, s86307; VI-NEXT:    buffer_atomic_xor v0, off, s[4:7], 0 glc6308; VI-NEXT:    s_waitcnt vmcnt(0)6309; VI-NEXT:    buffer_wbinvl1_vol6310; VI-NEXT:    s_mov_b32 s4, s26311; VI-NEXT:    s_mov_b32 s5, s36312; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 06313; VI-NEXT:    s_endpgm6314;6315; GFX9-LABEL: atomic_xor_i32_ret:6316; GFX9:       ; %bb.0: ; %entry6317; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x346318; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246319; GFX9-NEXT:    v_mov_b32_e32 v0, 06320; GFX9-NEXT:    s_waitcnt lgkmcnt(0)6321; GFX9-NEXT:    v_mov_b32_e32 v1, s66322; GFX9-NEXT:    global_atomic_xor v1, v0, v1, s[0:1] glc6323; GFX9-NEXT:    s_waitcnt vmcnt(0)6324; GFX9-NEXT:    buffer_wbinvl1_vol6325; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]6326; GFX9-NEXT:    s_endpgm6327;6328; GFX11-LABEL: atomic_xor_i32_ret:6329; GFX11:       ; %bb.0: ; %entry6330; GFX11-NEXT:    s_clause 0x16331; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x346332; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246333; GFX11-NEXT:    s_waitcnt lgkmcnt(0)6334; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s66335; GFX11-NEXT:    global_atomic_xor_b32 v1, v0, v1, s[0:1] glc6336; GFX11-NEXT:    s_waitcnt vmcnt(0)6337; GFX11-NEXT:    buffer_gl1_inv6338; GFX11-NEXT:    buffer_gl0_inv6339; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]6340; GFX11-NEXT:    s_endpgm6341entry:6342  %val = atomicrmw volatile xor ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !06343  store i32 %val, ptr addrspace(1) %out26344  ret void6345}6346 6347define amdgpu_kernel void @atomic_xor_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) {6348; SI-LABEL: atomic_xor_i32_addr64:6349; SI:       ; %bb.0: ; %entry6350; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd6351; SI-NEXT:    s_load_dword s8, s[4:5], 0xb6352; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x96353; SI-NEXT:    s_mov_b32 s3, 0xf0006354; SI-NEXT:    s_waitcnt lgkmcnt(0)6355; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 26356; SI-NEXT:    s_mov_b32 s2, 06357; SI-NEXT:    v_mov_b32_e32 v2, s86358; SI-NEXT:    v_mov_b32_e32 v0, s46359; SI-NEXT:    v_mov_b32_e32 v1, s56360; SI-NEXT:    buffer_atomic_xor v2, v[0:1], s[0:3], 0 addr646361; SI-NEXT:    s_waitcnt vmcnt(0)6362; SI-NEXT:    buffer_wbinvl16363; SI-NEXT:    s_endpgm6364;6365; VI-LABEL: atomic_xor_i32_addr64:6366; VI:       ; %bb.0: ; %entry6367; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x346368; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x246369; VI-NEXT:    s_load_dword s4, s[4:5], 0x2c6370; VI-NEXT:    s_waitcnt lgkmcnt(0)6371; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 26372; VI-NEXT:    s_add_u32 s0, s2, s06373; VI-NEXT:    s_addc_u32 s1, s3, s16374; VI-NEXT:    v_mov_b32_e32 v0, s06375; VI-NEXT:    v_mov_b32_e32 v1, s16376; VI-NEXT:    v_mov_b32_e32 v2, s46377; VI-NEXT:    flat_atomic_xor v[0:1], v26378; VI-NEXT:    s_waitcnt vmcnt(0)6379; VI-NEXT:    buffer_wbinvl1_vol6380; VI-NEXT:    s_endpgm6381;6382; GFX9-LABEL: atomic_xor_i32_addr64:6383; GFX9:       ; %bb.0: ; %entry6384; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x346385; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x246386; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c6387; GFX9-NEXT:    v_mov_b32_e32 v0, 06388; GFX9-NEXT:    s_waitcnt lgkmcnt(0)6389; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 26390; GFX9-NEXT:    s_add_u32 s0, s2, s06391; GFX9-NEXT:    s_addc_u32 s1, s3, s16392; GFX9-NEXT:    v_mov_b32_e32 v1, s66393; GFX9-NEXT:    global_atomic_xor v0, v1, s[0:1]6394; GFX9-NEXT:    s_waitcnt vmcnt(0)6395; GFX9-NEXT:    buffer_wbinvl1_vol6396; GFX9-NEXT:    s_endpgm6397;6398; GFX11-LABEL: atomic_xor_i32_addr64:6399; GFX11:       ; %bb.0: ; %entry6400; GFX11-NEXT:    s_clause 0x26401; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x346402; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c6403; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x246404; GFX11-NEXT:    v_mov_b32_e32 v0, 06405; GFX11-NEXT:    s_waitcnt lgkmcnt(0)6406; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 26407; GFX11-NEXT:    v_mov_b32_e32 v1, s66408; GFX11-NEXT:    s_add_u32 s0, s2, s06409; GFX11-NEXT:    s_addc_u32 s1, s3, s16410; GFX11-NEXT:    global_atomic_xor_b32 v0, v1, s[0:1]6411; GFX11-NEXT:    s_waitcnt_vscnt null, 0x06412; GFX11-NEXT:    buffer_gl1_inv6413; GFX11-NEXT:    buffer_gl0_inv6414; GFX11-NEXT:    s_endpgm6415entry:6416  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index6417  %val = atomicrmw volatile xor ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !06418  ret void6419}6420 6421define amdgpu_kernel void @atomic_xor_i32_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) {6422; SI-LABEL: atomic_xor_i32_ret_addr64:6423; SI:       ; %bb.0: ; %entry6424; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96425; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf6426; SI-NEXT:    s_load_dword s6, s[4:5], 0xd6427; SI-NEXT:    s_mov_b32 s7, 0xf0006428; SI-NEXT:    s_waitcnt lgkmcnt(0)6429; SI-NEXT:    s_mov_b32 s4, s26430; SI-NEXT:    s_mov_b32 s5, s36431; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 26432; SI-NEXT:    s_mov_b32 s2, 06433; SI-NEXT:    s_mov_b32 s3, s76434; SI-NEXT:    v_mov_b32_e32 v2, s66435; SI-NEXT:    v_mov_b32_e32 v0, s86436; SI-NEXT:    v_mov_b32_e32 v1, s96437; SI-NEXT:    buffer_atomic_xor v2, v[0:1], s[0:3], 0 addr64 glc6438; SI-NEXT:    s_waitcnt vmcnt(0)6439; SI-NEXT:    buffer_wbinvl16440; SI-NEXT:    s_mov_b32 s6, -16441; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 06442; SI-NEXT:    s_endpgm6443;6444; VI-LABEL: atomic_xor_i32_ret_addr64:6445; VI:       ; %bb.0: ; %entry6446; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c6447; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246448; VI-NEXT:    s_load_dword s8, s[4:5], 0x346449; VI-NEXT:    s_waitcnt lgkmcnt(0)6450; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 26451; VI-NEXT:    s_add_u32 s0, s0, s46452; VI-NEXT:    s_addc_u32 s1, s1, s56453; VI-NEXT:    v_mov_b32_e32 v0, s06454; VI-NEXT:    v_mov_b32_e32 v1, s16455; VI-NEXT:    v_mov_b32_e32 v2, s86456; VI-NEXT:    flat_atomic_xor v0, v[0:1], v2 glc6457; VI-NEXT:    s_waitcnt vmcnt(0)6458; VI-NEXT:    buffer_wbinvl1_vol6459; VI-NEXT:    s_mov_b32 s7, 0xf0006460; VI-NEXT:    s_mov_b32 s6, -16461; VI-NEXT:    s_mov_b32 s4, s26462; VI-NEXT:    s_mov_b32 s5, s36463; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 06464; VI-NEXT:    s_endpgm6465;6466; GFX9-LABEL: atomic_xor_i32_ret_addr64:6467; GFX9:       ; %bb.0: ; %entry6468; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c6469; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246470; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x346471; GFX9-NEXT:    v_mov_b32_e32 v0, 06472; GFX9-NEXT:    s_waitcnt lgkmcnt(0)6473; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 26474; GFX9-NEXT:    s_add_u32 s0, s0, s46475; GFX9-NEXT:    s_addc_u32 s1, s1, s56476; GFX9-NEXT:    v_mov_b32_e32 v1, s86477; GFX9-NEXT:    global_atomic_xor v1, v0, v1, s[0:1] glc6478; GFX9-NEXT:    s_waitcnt vmcnt(0)6479; GFX9-NEXT:    buffer_wbinvl1_vol6480; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]6481; GFX9-NEXT:    s_endpgm6482;6483; GFX11-LABEL: atomic_xor_i32_ret_addr64:6484; GFX11:       ; %bb.0: ; %entry6485; GFX11-NEXT:    s_clause 0x26486; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c6487; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x346488; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246489; GFX11-NEXT:    v_mov_b32_e32 v0, 06490; GFX11-NEXT:    s_waitcnt lgkmcnt(0)6491; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 26492; GFX11-NEXT:    v_mov_b32_e32 v1, s86493; GFX11-NEXT:    s_add_u32 s0, s0, s46494; GFX11-NEXT:    s_addc_u32 s1, s1, s56495; GFX11-NEXT:    global_atomic_xor_b32 v1, v0, v1, s[0:1] glc6496; GFX11-NEXT:    s_waitcnt vmcnt(0)6497; GFX11-NEXT:    buffer_gl1_inv6498; GFX11-NEXT:    buffer_gl0_inv6499; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]6500; GFX11-NEXT:    s_endpgm6501entry:6502  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index6503  %val = atomicrmw volatile xor ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !06504  store i32 %val, ptr addrspace(1) %out26505  ret void6506}6507 6508define amdgpu_kernel void @atomic_load_i32_offset(ptr addrspace(1) %in, ptr addrspace(1) %out) {6509; SI-LABEL: atomic_load_i32_offset:6510; SI:       ; %bb.0: ; %entry6511; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96512; SI-NEXT:    s_mov_b32 s7, 0xf0006513; SI-NEXT:    s_mov_b32 s6, -16514; SI-NEXT:    s_waitcnt lgkmcnt(0)6515; SI-NEXT:    s_mov_b32 s4, s26516; SI-NEXT:    s_mov_b32 s5, s36517; SI-NEXT:    s_mov_b32 s2, s66518; SI-NEXT:    s_mov_b32 s3, s76519; SI-NEXT:    buffer_load_dword v0, off, s[0:3], 0 offset:16 glc6520; SI-NEXT:    s_waitcnt vmcnt(0)6521; SI-NEXT:    buffer_wbinvl16522; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 06523; SI-NEXT:    s_endpgm6524;6525; VI-LABEL: atomic_load_i32_offset:6526; VI:       ; %bb.0: ; %entry6527; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246528; VI-NEXT:    s_mov_b32 s7, 0xf0006529; VI-NEXT:    s_mov_b32 s6, -16530; VI-NEXT:    s_waitcnt lgkmcnt(0)6531; VI-NEXT:    s_add_u32 s0, s0, 166532; VI-NEXT:    s_addc_u32 s1, s1, 06533; VI-NEXT:    v_mov_b32_e32 v0, s06534; VI-NEXT:    v_mov_b32_e32 v1, s16535; VI-NEXT:    flat_load_dword v0, v[0:1] glc6536; VI-NEXT:    s_waitcnt vmcnt(0)6537; VI-NEXT:    buffer_wbinvl1_vol6538; VI-NEXT:    s_mov_b32 s4, s26539; VI-NEXT:    s_mov_b32 s5, s36540; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 06541; VI-NEXT:    s_endpgm6542;6543; GFX9-LABEL: atomic_load_i32_offset:6544; GFX9:       ; %bb.0: ; %entry6545; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246546; GFX9-NEXT:    v_mov_b32_e32 v0, 06547; GFX9-NEXT:    s_waitcnt lgkmcnt(0)6548; GFX9-NEXT:    global_load_dword v1, v0, s[0:1] offset:16 glc6549; GFX9-NEXT:    s_waitcnt vmcnt(0)6550; GFX9-NEXT:    buffer_wbinvl1_vol6551; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]6552; GFX9-NEXT:    s_endpgm6553;6554; GFX11-LABEL: atomic_load_i32_offset:6555; GFX11:       ; %bb.0: ; %entry6556; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246557; GFX11-NEXT:    v_mov_b32_e32 v0, 06558; GFX11-NEXT:    s_waitcnt lgkmcnt(0)6559; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1] offset:16 glc6560; GFX11-NEXT:    s_waitcnt vmcnt(0)6561; GFX11-NEXT:    buffer_gl1_inv6562; GFX11-NEXT:    buffer_gl0_inv6563; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]6564; GFX11-NEXT:    s_endpgm6565entry:6566  %gep = getelementptr i32, ptr addrspace(1) %in, i64 46567  %val = load atomic i32, ptr addrspace(1) %gep  seq_cst, align 46568  store i32 %val, ptr addrspace(1) %out6569  ret void6570}6571 6572define amdgpu_kernel void @atomic_load_i32_negoffset(ptr addrspace(1) %in, ptr addrspace(1) %out) {6573; SI-LABEL: atomic_load_i32_negoffset:6574; SI:       ; %bb.0: ; %entry6575; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96576; SI-NEXT:    s_mov_b32 s7, 0xf0006577; SI-NEXT:    s_waitcnt lgkmcnt(0)6578; SI-NEXT:    s_mov_b32 s4, s26579; SI-NEXT:    s_mov_b32 s5, s36580; SI-NEXT:    s_mov_b32 s2, 06581; SI-NEXT:    s_mov_b32 s3, s76582; SI-NEXT:    v_mov_b32_e32 v0, 0xfffffe006583; SI-NEXT:    v_mov_b32_e32 v1, -16584; SI-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc6585; SI-NEXT:    s_waitcnt vmcnt(0)6586; SI-NEXT:    buffer_wbinvl16587; SI-NEXT:    s_mov_b32 s6, -16588; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 06589; SI-NEXT:    s_endpgm6590;6591; VI-LABEL: atomic_load_i32_negoffset:6592; VI:       ; %bb.0: ; %entry6593; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246594; VI-NEXT:    s_mov_b32 s7, 0xf0006595; VI-NEXT:    s_mov_b32 s6, -16596; VI-NEXT:    s_waitcnt lgkmcnt(0)6597; VI-NEXT:    s_add_u32 s0, s0, 0xfffffe006598; VI-NEXT:    s_addc_u32 s1, s1, -16599; VI-NEXT:    v_mov_b32_e32 v0, s06600; VI-NEXT:    v_mov_b32_e32 v1, s16601; VI-NEXT:    flat_load_dword v0, v[0:1] glc6602; VI-NEXT:    s_waitcnt vmcnt(0)6603; VI-NEXT:    buffer_wbinvl1_vol6604; VI-NEXT:    s_mov_b32 s4, s26605; VI-NEXT:    s_mov_b32 s5, s36606; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 06607; VI-NEXT:    s_endpgm6608;6609; GFX9-LABEL: atomic_load_i32_negoffset:6610; GFX9:       ; %bb.0: ; %entry6611; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246612; GFX9-NEXT:    v_mov_b32_e32 v0, 06613; GFX9-NEXT:    s_waitcnt lgkmcnt(0)6614; GFX9-NEXT:    global_load_dword v1, v0, s[0:1] offset:-512 glc6615; GFX9-NEXT:    s_waitcnt vmcnt(0)6616; GFX9-NEXT:    buffer_wbinvl1_vol6617; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]6618; GFX9-NEXT:    s_endpgm6619;6620; GFX11-LABEL: atomic_load_i32_negoffset:6621; GFX11:       ; %bb.0: ; %entry6622; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246623; GFX11-NEXT:    v_mov_b32_e32 v0, 06624; GFX11-NEXT:    s_waitcnt lgkmcnt(0)6625; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1] offset:-512 glc6626; GFX11-NEXT:    s_waitcnt vmcnt(0)6627; GFX11-NEXT:    buffer_gl1_inv6628; GFX11-NEXT:    buffer_gl0_inv6629; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]6630; GFX11-NEXT:    s_endpgm6631entry:6632  %gep = getelementptr i32, ptr addrspace(1) %in, i64 -1286633  %val = load atomic i32, ptr addrspace(1) %gep  seq_cst, align 46634  store i32 %val, ptr addrspace(1) %out6635  ret void6636}6637 6638define amdgpu_kernel void @atomic_load_f32_offset(ptr addrspace(1) %in, ptr addrspace(1) %out) {6639; SI-LABEL: atomic_load_f32_offset:6640; SI:       ; %bb.0: ; %entry6641; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96642; SI-NEXT:    s_mov_b32 s7, 0xf0006643; SI-NEXT:    s_mov_b32 s6, -16644; SI-NEXT:    s_waitcnt lgkmcnt(0)6645; SI-NEXT:    s_mov_b32 s4, s26646; SI-NEXT:    s_mov_b32 s5, s36647; SI-NEXT:    s_mov_b32 s2, s66648; SI-NEXT:    s_mov_b32 s3, s76649; SI-NEXT:    buffer_load_dword v0, off, s[0:3], 0 offset:16 glc6650; SI-NEXT:    s_waitcnt vmcnt(0)6651; SI-NEXT:    buffer_wbinvl16652; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 06653; SI-NEXT:    s_endpgm6654;6655; VI-LABEL: atomic_load_f32_offset:6656; VI:       ; %bb.0: ; %entry6657; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246658; VI-NEXT:    s_mov_b32 s7, 0xf0006659; VI-NEXT:    s_mov_b32 s6, -16660; VI-NEXT:    s_waitcnt lgkmcnt(0)6661; VI-NEXT:    s_add_u32 s0, s0, 166662; VI-NEXT:    s_addc_u32 s1, s1, 06663; VI-NEXT:    v_mov_b32_e32 v0, s06664; VI-NEXT:    v_mov_b32_e32 v1, s16665; VI-NEXT:    flat_load_dword v0, v[0:1] glc6666; VI-NEXT:    s_waitcnt vmcnt(0)6667; VI-NEXT:    buffer_wbinvl1_vol6668; VI-NEXT:    s_mov_b32 s4, s26669; VI-NEXT:    s_mov_b32 s5, s36670; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 06671; VI-NEXT:    s_endpgm6672;6673; GFX9-LABEL: atomic_load_f32_offset:6674; GFX9:       ; %bb.0: ; %entry6675; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246676; GFX9-NEXT:    v_mov_b32_e32 v0, 06677; GFX9-NEXT:    s_waitcnt lgkmcnt(0)6678; GFX9-NEXT:    global_load_dword v1, v0, s[0:1] offset:16 glc6679; GFX9-NEXT:    s_waitcnt vmcnt(0)6680; GFX9-NEXT:    buffer_wbinvl1_vol6681; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]6682; GFX9-NEXT:    s_endpgm6683;6684; GFX11-LABEL: atomic_load_f32_offset:6685; GFX11:       ; %bb.0: ; %entry6686; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246687; GFX11-NEXT:    v_mov_b32_e32 v0, 06688; GFX11-NEXT:    s_waitcnt lgkmcnt(0)6689; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1] offset:16 glc6690; GFX11-NEXT:    s_waitcnt vmcnt(0)6691; GFX11-NEXT:    buffer_gl1_inv6692; GFX11-NEXT:    buffer_gl0_inv6693; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]6694; GFX11-NEXT:    s_endpgm6695entry:6696  %gep = getelementptr float, ptr addrspace(1) %in, i64 46697  %val = load atomic float, ptr addrspace(1) %gep  seq_cst, align 46698  store float %val, ptr addrspace(1) %out6699  ret void6700}6701 6702define amdgpu_kernel void @atomic_load_i32(ptr addrspace(1) %in, ptr addrspace(1) %out) {6703; SI-LABEL: atomic_load_i32:6704; SI:       ; %bb.0: ; %entry6705; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96706; SI-NEXT:    s_mov_b32 s7, 0xf0006707; SI-NEXT:    s_mov_b32 s6, -16708; SI-NEXT:    s_waitcnt lgkmcnt(0)6709; SI-NEXT:    s_mov_b32 s4, s06710; SI-NEXT:    s_mov_b32 s5, s16711; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 glc6712; SI-NEXT:    s_waitcnt vmcnt(0)6713; SI-NEXT:    buffer_wbinvl16714; SI-NEXT:    s_mov_b32 s4, s26715; SI-NEXT:    s_mov_b32 s5, s36716; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 06717; SI-NEXT:    s_endpgm6718;6719; VI-LABEL: atomic_load_i32:6720; VI:       ; %bb.0: ; %entry6721; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246722; VI-NEXT:    s_mov_b32 s7, 0xf0006723; VI-NEXT:    s_mov_b32 s6, -16724; VI-NEXT:    s_waitcnt lgkmcnt(0)6725; VI-NEXT:    v_mov_b32_e32 v0, s06726; VI-NEXT:    v_mov_b32_e32 v1, s16727; VI-NEXT:    flat_load_dword v0, v[0:1] glc6728; VI-NEXT:    s_waitcnt vmcnt(0)6729; VI-NEXT:    buffer_wbinvl1_vol6730; VI-NEXT:    s_mov_b32 s4, s26731; VI-NEXT:    s_mov_b32 s5, s36732; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 06733; VI-NEXT:    s_endpgm6734;6735; GFX9-LABEL: atomic_load_i32:6736; GFX9:       ; %bb.0: ; %entry6737; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246738; GFX9-NEXT:    v_mov_b32_e32 v0, 06739; GFX9-NEXT:    s_waitcnt lgkmcnt(0)6740; GFX9-NEXT:    global_load_dword v1, v0, s[0:1] glc6741; GFX9-NEXT:    s_waitcnt vmcnt(0)6742; GFX9-NEXT:    buffer_wbinvl1_vol6743; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]6744; GFX9-NEXT:    s_endpgm6745;6746; GFX11-LABEL: atomic_load_i32:6747; GFX11:       ; %bb.0: ; %entry6748; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246749; GFX11-NEXT:    v_mov_b32_e32 v0, 06750; GFX11-NEXT:    s_waitcnt lgkmcnt(0)6751; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1] glc6752; GFX11-NEXT:    s_waitcnt vmcnt(0)6753; GFX11-NEXT:    buffer_gl1_inv6754; GFX11-NEXT:    buffer_gl0_inv6755; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]6756; GFX11-NEXT:    s_endpgm6757entry:6758  %val = load atomic i32, ptr addrspace(1) %in syncscope("agent") seq_cst, align 46759  store i32 %val, ptr addrspace(1) %out6760  ret void6761}6762 6763define amdgpu_kernel void @atomic_load_i32_addr64_offset(ptr addrspace(1) %in, ptr addrspace(1) %out, i64 %index) {6764; SI-LABEL: atomic_load_i32_addr64_offset:6765; SI:       ; %bb.0: ; %entry6766; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96767; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd6768; SI-NEXT:    s_mov_b32 s7, 0xf0006769; SI-NEXT:    s_waitcnt lgkmcnt(0)6770; SI-NEXT:    s_mov_b32 s4, s26771; SI-NEXT:    s_mov_b32 s5, s36772; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 26773; SI-NEXT:    s_mov_b32 s2, 06774; SI-NEXT:    s_mov_b32 s3, s76775; SI-NEXT:    v_mov_b32_e32 v0, s86776; SI-NEXT:    v_mov_b32_e32 v1, s96777; SI-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 offset:16 glc6778; SI-NEXT:    s_waitcnt vmcnt(0)6779; SI-NEXT:    buffer_wbinvl16780; SI-NEXT:    s_mov_b32 s6, -16781; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 06782; SI-NEXT:    s_endpgm6783;6784; VI-LABEL: atomic_load_i32_addr64_offset:6785; VI:       ; %bb.0: ; %entry6786; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x346787; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246788; VI-NEXT:    s_waitcnt lgkmcnt(0)6789; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 26790; VI-NEXT:    s_add_u32 s0, s0, s46791; VI-NEXT:    s_addc_u32 s1, s1, s56792; VI-NEXT:    s_add_u32 s0, s0, 166793; VI-NEXT:    s_addc_u32 s1, s1, 06794; VI-NEXT:    v_mov_b32_e32 v0, s06795; VI-NEXT:    v_mov_b32_e32 v1, s16796; VI-NEXT:    flat_load_dword v0, v[0:1] glc6797; VI-NEXT:    s_waitcnt vmcnt(0)6798; VI-NEXT:    buffer_wbinvl1_vol6799; VI-NEXT:    s_mov_b32 s7, 0xf0006800; VI-NEXT:    s_mov_b32 s6, -16801; VI-NEXT:    s_mov_b32 s4, s26802; VI-NEXT:    s_mov_b32 s5, s36803; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 06804; VI-NEXT:    s_endpgm6805;6806; GFX9-LABEL: atomic_load_i32_addr64_offset:6807; GFX9:       ; %bb.0: ; %entry6808; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x346809; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246810; GFX9-NEXT:    v_mov_b32_e32 v0, 06811; GFX9-NEXT:    s_waitcnt lgkmcnt(0)6812; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 26813; GFX9-NEXT:    s_add_u32 s0, s0, s46814; GFX9-NEXT:    s_addc_u32 s1, s1, s56815; GFX9-NEXT:    global_load_dword v1, v0, s[0:1] offset:16 glc6816; GFX9-NEXT:    s_waitcnt vmcnt(0)6817; GFX9-NEXT:    buffer_wbinvl1_vol6818; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]6819; GFX9-NEXT:    s_endpgm6820;6821; GFX11-LABEL: atomic_load_i32_addr64_offset:6822; GFX11:       ; %bb.0: ; %entry6823; GFX11-NEXT:    s_clause 0x16824; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x346825; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246826; GFX11-NEXT:    v_mov_b32_e32 v0, 06827; GFX11-NEXT:    s_waitcnt lgkmcnt(0)6828; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 26829; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)6830; GFX11-NEXT:    s_add_u32 s0, s0, s46831; GFX11-NEXT:    s_addc_u32 s1, s1, s56832; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1] offset:16 glc6833; GFX11-NEXT:    s_waitcnt vmcnt(0)6834; GFX11-NEXT:    buffer_gl1_inv6835; GFX11-NEXT:    buffer_gl0_inv6836; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]6837; GFX11-NEXT:    s_endpgm6838entry:6839  %ptr = getelementptr i32, ptr addrspace(1) %in, i64 %index6840  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 46841  %val = load atomic i32, ptr addrspace(1) %gep seq_cst, align 46842  store i32 %val, ptr addrspace(1) %out6843  ret void6844}6845 6846define amdgpu_kernel void @atomic_load_i32_addr64(ptr addrspace(1) %in, ptr addrspace(1) %out, i64 %index) {6847; SI-LABEL: atomic_load_i32_addr64:6848; SI:       ; %bb.0: ; %entry6849; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96850; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd6851; SI-NEXT:    s_mov_b32 s7, 0xf0006852; SI-NEXT:    s_waitcnt lgkmcnt(0)6853; SI-NEXT:    s_mov_b32 s4, s26854; SI-NEXT:    s_mov_b32 s5, s36855; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 26856; SI-NEXT:    s_mov_b32 s2, 06857; SI-NEXT:    s_mov_b32 s3, s76858; SI-NEXT:    v_mov_b32_e32 v0, s86859; SI-NEXT:    v_mov_b32_e32 v1, s96860; SI-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc6861; SI-NEXT:    s_waitcnt vmcnt(0)6862; SI-NEXT:    buffer_wbinvl16863; SI-NEXT:    s_mov_b32 s6, -16864; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 06865; SI-NEXT:    s_endpgm6866;6867; VI-LABEL: atomic_load_i32_addr64:6868; VI:       ; %bb.0: ; %entry6869; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x346870; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246871; VI-NEXT:    s_waitcnt lgkmcnt(0)6872; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 26873; VI-NEXT:    s_add_u32 s0, s0, s46874; VI-NEXT:    s_addc_u32 s1, s1, s56875; VI-NEXT:    v_mov_b32_e32 v0, s06876; VI-NEXT:    v_mov_b32_e32 v1, s16877; VI-NEXT:    flat_load_dword v0, v[0:1] glc6878; VI-NEXT:    s_waitcnt vmcnt(0)6879; VI-NEXT:    buffer_wbinvl1_vol6880; VI-NEXT:    s_mov_b32 s7, 0xf0006881; VI-NEXT:    s_mov_b32 s6, -16882; VI-NEXT:    s_mov_b32 s4, s26883; VI-NEXT:    s_mov_b32 s5, s36884; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 06885; VI-NEXT:    s_endpgm6886;6887; GFX9-LABEL: atomic_load_i32_addr64:6888; GFX9:       ; %bb.0: ; %entry6889; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x346890; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246891; GFX9-NEXT:    v_mov_b32_e32 v0, 06892; GFX9-NEXT:    s_waitcnt lgkmcnt(0)6893; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 26894; GFX9-NEXT:    s_add_u32 s0, s0, s46895; GFX9-NEXT:    s_addc_u32 s1, s1, s56896; GFX9-NEXT:    global_load_dword v1, v0, s[0:1] glc6897; GFX9-NEXT:    s_waitcnt vmcnt(0)6898; GFX9-NEXT:    buffer_wbinvl1_vol6899; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]6900; GFX9-NEXT:    s_endpgm6901;6902; GFX11-LABEL: atomic_load_i32_addr64:6903; GFX11:       ; %bb.0: ; %entry6904; GFX11-NEXT:    s_clause 0x16905; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x346906; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246907; GFX11-NEXT:    v_mov_b32_e32 v0, 06908; GFX11-NEXT:    s_waitcnt lgkmcnt(0)6909; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 26910; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)6911; GFX11-NEXT:    s_add_u32 s0, s0, s46912; GFX11-NEXT:    s_addc_u32 s1, s1, s56913; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1] glc6914; GFX11-NEXT:    s_waitcnt vmcnt(0)6915; GFX11-NEXT:    buffer_gl1_inv6916; GFX11-NEXT:    buffer_gl0_inv6917; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]6918; GFX11-NEXT:    s_endpgm6919entry:6920  %ptr = getelementptr i32, ptr addrspace(1) %in, i64 %index6921  %val = load atomic i32, ptr addrspace(1) %ptr seq_cst, align 46922  store i32 %val, ptr addrspace(1) %out6923  ret void6924}6925 6926define amdgpu_kernel void @atomic_load_f32_addr64_offset(ptr addrspace(1) %in, ptr addrspace(1) %out, i64 %index) {6927; SI-LABEL: atomic_load_f32_addr64_offset:6928; SI:       ; %bb.0: ; %entry6929; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96930; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd6931; SI-NEXT:    s_mov_b32 s7, 0xf0006932; SI-NEXT:    s_waitcnt lgkmcnt(0)6933; SI-NEXT:    s_mov_b32 s4, s26934; SI-NEXT:    s_mov_b32 s5, s36935; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 26936; SI-NEXT:    s_mov_b32 s2, 06937; SI-NEXT:    s_mov_b32 s3, s76938; SI-NEXT:    v_mov_b32_e32 v0, s86939; SI-NEXT:    v_mov_b32_e32 v1, s96940; SI-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 offset:16 glc6941; SI-NEXT:    s_waitcnt vmcnt(0)6942; SI-NEXT:    buffer_wbinvl16943; SI-NEXT:    s_mov_b32 s6, -16944; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 06945; SI-NEXT:    s_endpgm6946;6947; VI-LABEL: atomic_load_f32_addr64_offset:6948; VI:       ; %bb.0: ; %entry6949; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x346950; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246951; VI-NEXT:    s_waitcnt lgkmcnt(0)6952; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 26953; VI-NEXT:    s_add_u32 s0, s0, s46954; VI-NEXT:    s_addc_u32 s1, s1, s56955; VI-NEXT:    s_add_u32 s0, s0, 166956; VI-NEXT:    s_addc_u32 s1, s1, 06957; VI-NEXT:    v_mov_b32_e32 v0, s06958; VI-NEXT:    v_mov_b32_e32 v1, s16959; VI-NEXT:    flat_load_dword v0, v[0:1] glc6960; VI-NEXT:    s_waitcnt vmcnt(0)6961; VI-NEXT:    buffer_wbinvl1_vol6962; VI-NEXT:    s_mov_b32 s7, 0xf0006963; VI-NEXT:    s_mov_b32 s6, -16964; VI-NEXT:    s_mov_b32 s4, s26965; VI-NEXT:    s_mov_b32 s5, s36966; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 06967; VI-NEXT:    s_endpgm6968;6969; GFX9-LABEL: atomic_load_f32_addr64_offset:6970; GFX9:       ; %bb.0: ; %entry6971; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x346972; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246973; GFX9-NEXT:    v_mov_b32_e32 v0, 06974; GFX9-NEXT:    s_waitcnt lgkmcnt(0)6975; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 26976; GFX9-NEXT:    s_add_u32 s0, s0, s46977; GFX9-NEXT:    s_addc_u32 s1, s1, s56978; GFX9-NEXT:    global_load_dword v1, v0, s[0:1] offset:16 glc6979; GFX9-NEXT:    s_waitcnt vmcnt(0)6980; GFX9-NEXT:    buffer_wbinvl1_vol6981; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]6982; GFX9-NEXT:    s_endpgm6983;6984; GFX11-LABEL: atomic_load_f32_addr64_offset:6985; GFX11:       ; %bb.0: ; %entry6986; GFX11-NEXT:    s_clause 0x16987; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x346988; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x246989; GFX11-NEXT:    v_mov_b32_e32 v0, 06990; GFX11-NEXT:    s_waitcnt lgkmcnt(0)6991; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 26992; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)6993; GFX11-NEXT:    s_add_u32 s0, s0, s46994; GFX11-NEXT:    s_addc_u32 s1, s1, s56995; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1] offset:16 glc6996; GFX11-NEXT:    s_waitcnt vmcnt(0)6997; GFX11-NEXT:    buffer_gl1_inv6998; GFX11-NEXT:    buffer_gl0_inv6999; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]7000; GFX11-NEXT:    s_endpgm7001entry:7002  %ptr = getelementptr float, ptr addrspace(1) %in, i64 %index7003  %gep = getelementptr float, ptr addrspace(1) %ptr, i64 47004  %val = load atomic float, ptr addrspace(1) %gep seq_cst, align 47005  store float %val, ptr addrspace(1) %out7006  ret void7007}7008 7009define amdgpu_kernel void @atomic_store_i32_offset(i32 %in, ptr addrspace(1) %out) {7010; SI-LABEL: atomic_store_i32_offset:7011; SI:       ; %bb.0: ; %entry7012; SI-NEXT:    s_load_dword s6, s[4:5], 0x97013; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb7014; SI-NEXT:    s_mov_b32 s3, 0xf0007015; SI-NEXT:    s_mov_b32 s2, -17016; SI-NEXT:    s_waitcnt lgkmcnt(0)7017; SI-NEXT:    v_mov_b32_e32 v0, s67018; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0 offset:167019; SI-NEXT:    s_endpgm7020;7021; VI-LABEL: atomic_store_i32_offset:7022; VI:       ; %bb.0: ; %entry7023; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c7024; VI-NEXT:    s_load_dword s2, s[4:5], 0x247025; VI-NEXT:    s_waitcnt lgkmcnt(0)7026; VI-NEXT:    s_add_u32 s0, s0, 167027; VI-NEXT:    s_addc_u32 s1, s1, 07028; VI-NEXT:    v_mov_b32_e32 v0, s07029; VI-NEXT:    v_mov_b32_e32 v1, s17030; VI-NEXT:    v_mov_b32_e32 v2, s27031; VI-NEXT:    flat_store_dword v[0:1], v27032; VI-NEXT:    s_endpgm7033;7034; GFX9-LABEL: atomic_store_i32_offset:7035; GFX9:       ; %bb.0: ; %entry7036; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x247037; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c7038; GFX9-NEXT:    v_mov_b32_e32 v0, 07039; GFX9-NEXT:    s_waitcnt lgkmcnt(0)7040; GFX9-NEXT:    v_mov_b32_e32 v1, s27041; GFX9-NEXT:    global_store_dword v0, v1, s[0:1] offset:167042; GFX9-NEXT:    s_endpgm7043;7044; GFX11-LABEL: atomic_store_i32_offset:7045; GFX11:       ; %bb.0: ; %entry7046; GFX11-NEXT:    s_clause 0x17047; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x247048; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c7049; GFX11-NEXT:    s_waitcnt lgkmcnt(0)7050; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s27051; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1] offset:167052; GFX11-NEXT:    s_endpgm7053entry:7054  %gep = getelementptr i32, ptr addrspace(1) %out, i64 47055  store atomic i32 %in, ptr addrspace(1) %gep  seq_cst, align 47056  ret void7057}7058 7059define amdgpu_kernel void @atomic_store_i32(i32 %in, ptr addrspace(1) %out) {7060; SI-LABEL: atomic_store_i32:7061; SI:       ; %bb.0: ; %entry7062; SI-NEXT:    s_load_dword s6, s[4:5], 0x97063; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb7064; SI-NEXT:    s_mov_b32 s3, 0xf0007065; SI-NEXT:    s_mov_b32 s2, -17066; SI-NEXT:    s_waitcnt lgkmcnt(0)7067; SI-NEXT:    v_mov_b32_e32 v0, s67068; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 07069; SI-NEXT:    s_endpgm7070;7071; VI-LABEL: atomic_store_i32:7072; VI:       ; %bb.0: ; %entry7073; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c7074; VI-NEXT:    s_load_dword s2, s[4:5], 0x247075; VI-NEXT:    s_waitcnt lgkmcnt(0)7076; VI-NEXT:    v_mov_b32_e32 v0, s07077; VI-NEXT:    v_mov_b32_e32 v1, s17078; VI-NEXT:    v_mov_b32_e32 v2, s27079; VI-NEXT:    flat_store_dword v[0:1], v27080; VI-NEXT:    s_endpgm7081;7082; GFX9-LABEL: atomic_store_i32:7083; GFX9:       ; %bb.0: ; %entry7084; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x247085; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c7086; GFX9-NEXT:    v_mov_b32_e32 v0, 07087; GFX9-NEXT:    s_waitcnt lgkmcnt(0)7088; GFX9-NEXT:    v_mov_b32_e32 v1, s27089; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]7090; GFX9-NEXT:    s_endpgm7091;7092; GFX11-LABEL: atomic_store_i32:7093; GFX11:       ; %bb.0: ; %entry7094; GFX11-NEXT:    s_clause 0x17095; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x247096; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c7097; GFX11-NEXT:    s_waitcnt lgkmcnt(0)7098; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s27099; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]7100; GFX11-NEXT:    s_endpgm7101entry:7102  store atomic i32 %in, ptr addrspace(1) %out seq_cst, align 47103  ret void7104}7105 7106define amdgpu_kernel void @atomic_store_f32(float %in, ptr addrspace(1) %out) {7107; SI-LABEL: atomic_store_f32:7108; SI:       ; %bb.0: ; %entry7109; SI-NEXT:    s_load_dword s6, s[4:5], 0x97110; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb7111; SI-NEXT:    s_mov_b32 s3, 0xf0007112; SI-NEXT:    s_mov_b32 s2, -17113; SI-NEXT:    s_waitcnt lgkmcnt(0)7114; SI-NEXT:    v_mov_b32_e32 v0, s67115; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 07116; SI-NEXT:    s_endpgm7117;7118; VI-LABEL: atomic_store_f32:7119; VI:       ; %bb.0: ; %entry7120; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c7121; VI-NEXT:    s_load_dword s2, s[4:5], 0x247122; VI-NEXT:    s_waitcnt lgkmcnt(0)7123; VI-NEXT:    v_mov_b32_e32 v0, s07124; VI-NEXT:    v_mov_b32_e32 v1, s17125; VI-NEXT:    v_mov_b32_e32 v2, s27126; VI-NEXT:    flat_store_dword v[0:1], v27127; VI-NEXT:    s_endpgm7128;7129; GFX9-LABEL: atomic_store_f32:7130; GFX9:       ; %bb.0: ; %entry7131; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x247132; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c7133; GFX9-NEXT:    v_mov_b32_e32 v0, 07134; GFX9-NEXT:    s_waitcnt lgkmcnt(0)7135; GFX9-NEXT:    v_mov_b32_e32 v1, s27136; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]7137; GFX9-NEXT:    s_endpgm7138;7139; GFX11-LABEL: atomic_store_f32:7140; GFX11:       ; %bb.0: ; %entry7141; GFX11-NEXT:    s_clause 0x17142; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x247143; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c7144; GFX11-NEXT:    s_waitcnt lgkmcnt(0)7145; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s27146; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]7147; GFX11-NEXT:    s_endpgm7148entry:7149  store atomic float %in, ptr addrspace(1) %out seq_cst, align 47150  ret void7151}7152 7153define amdgpu_kernel void @atomic_store_i32_addr64_offset(i32 %in, ptr addrspace(1) %out, i64 %index) {7154; SI-LABEL: atomic_store_i32_addr64_offset:7155; SI:       ; %bb.0: ; %entry7156; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb7157; SI-NEXT:    s_load_dword s6, s[4:5], 0x97158; SI-NEXT:    s_waitcnt lgkmcnt(0)7159; SI-NEXT:    s_lshl_b64 s[4:5], s[2:3], 27160; SI-NEXT:    s_mov_b32 s3, 0xf0007161; SI-NEXT:    s_mov_b32 s2, 07162; SI-NEXT:    v_mov_b32_e32 v2, s67163; SI-NEXT:    v_mov_b32_e32 v0, s47164; SI-NEXT:    v_mov_b32_e32 v1, s57165; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:167166; SI-NEXT:    s_endpgm7167;7168; VI-LABEL: atomic_store_i32_addr64_offset:7169; VI:       ; %bb.0: ; %entry7170; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c7171; VI-NEXT:    s_load_dword s4, s[4:5], 0x247172; VI-NEXT:    s_waitcnt lgkmcnt(0)7173; VI-NEXT:    s_lshl_b64 s[2:3], s[2:3], 27174; VI-NEXT:    s_add_u32 s0, s0, s27175; VI-NEXT:    s_addc_u32 s1, s1, s37176; VI-NEXT:    s_add_u32 s0, s0, 167177; VI-NEXT:    s_addc_u32 s1, s1, 07178; VI-NEXT:    v_mov_b32_e32 v0, s07179; VI-NEXT:    v_mov_b32_e32 v1, s17180; VI-NEXT:    v_mov_b32_e32 v2, s47181; VI-NEXT:    flat_store_dword v[0:1], v27182; VI-NEXT:    s_endpgm7183;7184; GFX9-LABEL: atomic_store_i32_addr64_offset:7185; GFX9:       ; %bb.0: ; %entry7186; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c7187; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x247188; GFX9-NEXT:    v_mov_b32_e32 v0, 07189; GFX9-NEXT:    s_waitcnt lgkmcnt(0)7190; GFX9-NEXT:    s_lshl_b64 s[2:3], s[2:3], 27191; GFX9-NEXT:    s_add_u32 s0, s0, s27192; GFX9-NEXT:    s_addc_u32 s1, s1, s37193; GFX9-NEXT:    v_mov_b32_e32 v1, s67194; GFX9-NEXT:    global_store_dword v0, v1, s[0:1] offset:167195; GFX9-NEXT:    s_endpgm7196;7197; GFX11-LABEL: atomic_store_i32_addr64_offset:7198; GFX11:       ; %bb.0: ; %entry7199; GFX11-NEXT:    s_clause 0x17200; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c7201; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x247202; GFX11-NEXT:    v_mov_b32_e32 v0, 07203; GFX11-NEXT:    s_waitcnt lgkmcnt(0)7204; GFX11-NEXT:    s_lshl_b64 s[2:3], s[2:3], 27205; GFX11-NEXT:    v_mov_b32_e32 v1, s47206; GFX11-NEXT:    s_add_u32 s0, s0, s27207; GFX11-NEXT:    s_addc_u32 s1, s1, s37208; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1] offset:167209; GFX11-NEXT:    s_endpgm7210entry:7211  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index7212  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 47213  store atomic i32 %in, ptr addrspace(1) %gep seq_cst, align 47214  ret void7215}7216 7217define amdgpu_kernel void @atomic_store_f32_addr64_offset(float %in, ptr addrspace(1) %out, i64 %index) {7218; SI-LABEL: atomic_store_f32_addr64_offset:7219; SI:       ; %bb.0: ; %entry7220; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb7221; SI-NEXT:    s_load_dword s6, s[4:5], 0x97222; SI-NEXT:    s_waitcnt lgkmcnt(0)7223; SI-NEXT:    s_lshl_b64 s[4:5], s[2:3], 27224; SI-NEXT:    s_mov_b32 s3, 0xf0007225; SI-NEXT:    s_mov_b32 s2, 07226; SI-NEXT:    v_mov_b32_e32 v2, s67227; SI-NEXT:    v_mov_b32_e32 v0, s47228; SI-NEXT:    v_mov_b32_e32 v1, s57229; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:167230; SI-NEXT:    s_endpgm7231;7232; VI-LABEL: atomic_store_f32_addr64_offset:7233; VI:       ; %bb.0: ; %entry7234; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c7235; VI-NEXT:    s_load_dword s4, s[4:5], 0x247236; VI-NEXT:    s_waitcnt lgkmcnt(0)7237; VI-NEXT:    s_lshl_b64 s[2:3], s[2:3], 27238; VI-NEXT:    s_add_u32 s0, s0, s27239; VI-NEXT:    s_addc_u32 s1, s1, s37240; VI-NEXT:    s_add_u32 s0, s0, 167241; VI-NEXT:    s_addc_u32 s1, s1, 07242; VI-NEXT:    v_mov_b32_e32 v0, s07243; VI-NEXT:    v_mov_b32_e32 v1, s17244; VI-NEXT:    v_mov_b32_e32 v2, s47245; VI-NEXT:    flat_store_dword v[0:1], v27246; VI-NEXT:    s_endpgm7247;7248; GFX9-LABEL: atomic_store_f32_addr64_offset:7249; GFX9:       ; %bb.0: ; %entry7250; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c7251; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x247252; GFX9-NEXT:    v_mov_b32_e32 v0, 07253; GFX9-NEXT:    s_waitcnt lgkmcnt(0)7254; GFX9-NEXT:    s_lshl_b64 s[2:3], s[2:3], 27255; GFX9-NEXT:    s_add_u32 s0, s0, s27256; GFX9-NEXT:    s_addc_u32 s1, s1, s37257; GFX9-NEXT:    v_mov_b32_e32 v1, s67258; GFX9-NEXT:    global_store_dword v0, v1, s[0:1] offset:167259; GFX9-NEXT:    s_endpgm7260;7261; GFX11-LABEL: atomic_store_f32_addr64_offset:7262; GFX11:       ; %bb.0: ; %entry7263; GFX11-NEXT:    s_clause 0x17264; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c7265; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x247266; GFX11-NEXT:    v_mov_b32_e32 v0, 07267; GFX11-NEXT:    s_waitcnt lgkmcnt(0)7268; GFX11-NEXT:    s_lshl_b64 s[2:3], s[2:3], 27269; GFX11-NEXT:    v_mov_b32_e32 v1, s47270; GFX11-NEXT:    s_add_u32 s0, s0, s27271; GFX11-NEXT:    s_addc_u32 s1, s1, s37272; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1] offset:167273; GFX11-NEXT:    s_endpgm7274entry:7275  %ptr = getelementptr float, ptr addrspace(1) %out, i64 %index7276  %gep = getelementptr float, ptr addrspace(1) %ptr, i64 47277  store atomic float %in, ptr addrspace(1) %gep seq_cst, align 47278  ret void7279}7280 7281define amdgpu_kernel void @atomic_store_i32_addr64(i32 %in, ptr addrspace(1) %out, i64 %index) {7282; SI-LABEL: atomic_store_i32_addr64:7283; SI:       ; %bb.0: ; %entry7284; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb7285; SI-NEXT:    s_load_dword s8, s[4:5], 0x97286; SI-NEXT:    s_mov_b32 s7, 0xf0007287; SI-NEXT:    s_waitcnt lgkmcnt(0)7288; SI-NEXT:    s_lshl_b64 s[2:3], s[2:3], 27289; SI-NEXT:    s_mov_b32 s6, 07290; SI-NEXT:    s_mov_b64 s[4:5], s[0:1]7291; SI-NEXT:    v_mov_b32_e32 v2, s87292; SI-NEXT:    v_mov_b32_e32 v0, s27293; SI-NEXT:    v_mov_b32_e32 v1, s37294; SI-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr647295; SI-NEXT:    s_endpgm7296;7297; VI-LABEL: atomic_store_i32_addr64:7298; VI:       ; %bb.0: ; %entry7299; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c7300; VI-NEXT:    s_load_dword s4, s[4:5], 0x247301; VI-NEXT:    s_waitcnt lgkmcnt(0)7302; VI-NEXT:    s_lshl_b64 s[2:3], s[2:3], 27303; VI-NEXT:    s_add_u32 s0, s0, s27304; VI-NEXT:    s_addc_u32 s1, s1, s37305; VI-NEXT:    v_mov_b32_e32 v0, s07306; VI-NEXT:    v_mov_b32_e32 v1, s17307; VI-NEXT:    v_mov_b32_e32 v2, s47308; VI-NEXT:    flat_store_dword v[0:1], v27309; VI-NEXT:    s_endpgm7310;7311; GFX9-LABEL: atomic_store_i32_addr64:7312; GFX9:       ; %bb.0: ; %entry7313; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c7314; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x247315; GFX9-NEXT:    v_mov_b32_e32 v0, 07316; GFX9-NEXT:    s_waitcnt lgkmcnt(0)7317; GFX9-NEXT:    s_lshl_b64 s[2:3], s[2:3], 27318; GFX9-NEXT:    s_add_u32 s0, s0, s27319; GFX9-NEXT:    s_addc_u32 s1, s1, s37320; GFX9-NEXT:    v_mov_b32_e32 v1, s67321; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]7322; GFX9-NEXT:    s_endpgm7323;7324; GFX11-LABEL: atomic_store_i32_addr64:7325; GFX11:       ; %bb.0: ; %entry7326; GFX11-NEXT:    s_clause 0x17327; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c7328; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x247329; GFX11-NEXT:    v_mov_b32_e32 v0, 07330; GFX11-NEXT:    s_waitcnt lgkmcnt(0)7331; GFX11-NEXT:    s_lshl_b64 s[2:3], s[2:3], 27332; GFX11-NEXT:    v_mov_b32_e32 v1, s47333; GFX11-NEXT:    s_add_u32 s0, s0, s27334; GFX11-NEXT:    s_addc_u32 s1, s1, s37335; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]7336; GFX11-NEXT:    s_endpgm7337entry:7338  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index7339  store atomic i32 %in, ptr addrspace(1) %ptr seq_cst, align 47340  ret void7341}7342 7343define amdgpu_kernel void @atomic_store_f32_addr64(float %in, ptr addrspace(1) %out, i64 %index) {7344; SI-LABEL: atomic_store_f32_addr64:7345; SI:       ; %bb.0: ; %entry7346; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb7347; SI-NEXT:    s_load_dword s8, s[4:5], 0x97348; SI-NEXT:    s_mov_b32 s7, 0xf0007349; SI-NEXT:    s_waitcnt lgkmcnt(0)7350; SI-NEXT:    s_lshl_b64 s[2:3], s[2:3], 27351; SI-NEXT:    s_mov_b32 s6, 07352; SI-NEXT:    s_mov_b64 s[4:5], s[0:1]7353; SI-NEXT:    v_mov_b32_e32 v2, s87354; SI-NEXT:    v_mov_b32_e32 v0, s27355; SI-NEXT:    v_mov_b32_e32 v1, s37356; SI-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr647357; SI-NEXT:    s_endpgm7358;7359; VI-LABEL: atomic_store_f32_addr64:7360; VI:       ; %bb.0: ; %entry7361; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c7362; VI-NEXT:    s_load_dword s4, s[4:5], 0x247363; VI-NEXT:    s_waitcnt lgkmcnt(0)7364; VI-NEXT:    s_lshl_b64 s[2:3], s[2:3], 27365; VI-NEXT:    s_add_u32 s0, s0, s27366; VI-NEXT:    s_addc_u32 s1, s1, s37367; VI-NEXT:    v_mov_b32_e32 v0, s07368; VI-NEXT:    v_mov_b32_e32 v1, s17369; VI-NEXT:    v_mov_b32_e32 v2, s47370; VI-NEXT:    flat_store_dword v[0:1], v27371; VI-NEXT:    s_endpgm7372;7373; GFX9-LABEL: atomic_store_f32_addr64:7374; GFX9:       ; %bb.0: ; %entry7375; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c7376; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x247377; GFX9-NEXT:    v_mov_b32_e32 v0, 07378; GFX9-NEXT:    s_waitcnt lgkmcnt(0)7379; GFX9-NEXT:    s_lshl_b64 s[2:3], s[2:3], 27380; GFX9-NEXT:    s_add_u32 s0, s0, s27381; GFX9-NEXT:    s_addc_u32 s1, s1, s37382; GFX9-NEXT:    v_mov_b32_e32 v1, s67383; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]7384; GFX9-NEXT:    s_endpgm7385;7386; GFX11-LABEL: atomic_store_f32_addr64:7387; GFX11:       ; %bb.0: ; %entry7388; GFX11-NEXT:    s_clause 0x17389; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c7390; GFX11-NEXT:    s_load_b32 s4, s[4:5], 0x247391; GFX11-NEXT:    v_mov_b32_e32 v0, 07392; GFX11-NEXT:    s_waitcnt lgkmcnt(0)7393; GFX11-NEXT:    s_lshl_b64 s[2:3], s[2:3], 27394; GFX11-NEXT:    v_mov_b32_e32 v1, s47395; GFX11-NEXT:    s_add_u32 s0, s0, s27396; GFX11-NEXT:    s_addc_u32 s1, s1, s37397; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]7398; GFX11-NEXT:    s_endpgm7399entry:7400  %ptr = getelementptr float, ptr addrspace(1) %out, i64 %index7401  store atomic float %in, ptr addrspace(1) %ptr seq_cst, align 47402  ret void7403}7404 7405define amdgpu_kernel void @atomic_load_i8_offset(ptr addrspace(1) %in, ptr addrspace(1) %out) {7406; SI-LABEL: atomic_load_i8_offset:7407; SI:       ; %bb.0: ; %entry7408; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x97409; SI-NEXT:    s_mov_b32 s7, 0xf0007410; SI-NEXT:    s_mov_b32 s6, -17411; SI-NEXT:    s_waitcnt lgkmcnt(0)7412; SI-NEXT:    s_mov_b32 s4, s27413; SI-NEXT:    s_mov_b32 s5, s37414; SI-NEXT:    s_mov_b32 s2, s67415; SI-NEXT:    s_mov_b32 s3, s77416; SI-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0 offset:16 glc7417; SI-NEXT:    s_waitcnt vmcnt(0)7418; SI-NEXT:    buffer_wbinvl17419; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 07420; SI-NEXT:    s_endpgm7421;7422; VI-LABEL: atomic_load_i8_offset:7423; VI:       ; %bb.0: ; %entry7424; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247425; VI-NEXT:    s_mov_b32 s7, 0xf0007426; VI-NEXT:    s_mov_b32 s6, -17427; VI-NEXT:    s_waitcnt lgkmcnt(0)7428; VI-NEXT:    s_mov_b32 s4, s27429; VI-NEXT:    s_mov_b32 s5, s37430; VI-NEXT:    s_mov_b32 s2, s67431; VI-NEXT:    s_mov_b32 s3, s77432; VI-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0 offset:16 glc7433; VI-NEXT:    s_waitcnt vmcnt(0)7434; VI-NEXT:    buffer_wbinvl1_vol7435; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 07436; VI-NEXT:    s_endpgm7437;7438; GFX9-LABEL: atomic_load_i8_offset:7439; GFX9:       ; %bb.0: ; %entry7440; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247441; GFX9-NEXT:    v_mov_b32_e32 v0, 07442; GFX9-NEXT:    s_waitcnt lgkmcnt(0)7443; GFX9-NEXT:    global_load_ubyte v1, v0, s[0:1] offset:16 glc7444; GFX9-NEXT:    s_waitcnt vmcnt(0)7445; GFX9-NEXT:    buffer_wbinvl1_vol7446; GFX9-NEXT:    global_store_byte v0, v1, s[2:3]7447; GFX9-NEXT:    s_endpgm7448;7449; GFX11-TRUE16-LABEL: atomic_load_i8_offset:7450; GFX11-TRUE16:       ; %bb.0: ; %entry7451; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x247452; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 07453; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)7454; GFX11-TRUE16-NEXT:    global_load_d16_u8 v0, v1, s[0:1] offset:16 glc7455; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)7456; GFX11-TRUE16-NEXT:    buffer_gl1_inv7457; GFX11-TRUE16-NEXT:    buffer_gl0_inv7458; GFX11-TRUE16-NEXT:    global_store_b8 v1, v0, s[2:3]7459; GFX11-TRUE16-NEXT:    s_endpgm7460;7461; GFX11-FAKE16-LABEL: atomic_load_i8_offset:7462; GFX11-FAKE16:       ; %bb.0: ; %entry7463; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x247464; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 07465; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)7466; GFX11-FAKE16-NEXT:    global_load_u8 v1, v0, s[0:1] offset:16 glc7467; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)7468; GFX11-FAKE16-NEXT:    buffer_gl1_inv7469; GFX11-FAKE16-NEXT:    buffer_gl0_inv7470; GFX11-FAKE16-NEXT:    global_store_b8 v0, v1, s[2:3]7471; GFX11-FAKE16-NEXT:    s_endpgm7472entry:7473  %gep = getelementptr i8, ptr addrspace(1) %in, i64 167474  %val = load atomic i8, ptr addrspace(1) %gep  seq_cst, align 17475  store i8 %val, ptr addrspace(1) %out7476  ret void7477}7478 7479define amdgpu_kernel void @atomic_load_i8_negoffset(ptr addrspace(1) %in, ptr addrspace(1) %out) {7480; SI-LABEL: atomic_load_i8_negoffset:7481; SI:       ; %bb.0: ; %entry7482; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x97483; SI-NEXT:    s_mov_b32 s7, 0xf0007484; SI-NEXT:    s_waitcnt lgkmcnt(0)7485; SI-NEXT:    s_mov_b32 s4, s27486; SI-NEXT:    s_mov_b32 s5, s37487; SI-NEXT:    s_mov_b32 s2, 07488; SI-NEXT:    s_mov_b32 s3, s77489; SI-NEXT:    v_mov_b32_e32 v0, 0xfffffe007490; SI-NEXT:    v_mov_b32_e32 v1, -17491; SI-NEXT:    buffer_load_ubyte v0, v[0:1], s[0:3], 0 addr64 glc7492; SI-NEXT:    s_waitcnt vmcnt(0)7493; SI-NEXT:    buffer_wbinvl17494; SI-NEXT:    s_mov_b32 s6, -17495; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 07496; SI-NEXT:    s_endpgm7497;7498; VI-LABEL: atomic_load_i8_negoffset:7499; VI:       ; %bb.0: ; %entry7500; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247501; VI-NEXT:    s_mov_b32 s7, 0xf0007502; VI-NEXT:    s_mov_b32 s6, -17503; VI-NEXT:    s_waitcnt lgkmcnt(0)7504; VI-NEXT:    s_add_u32 s0, s0, 0xfffffe007505; VI-NEXT:    s_addc_u32 s1, s1, -17506; VI-NEXT:    v_mov_b32_e32 v0, s07507; VI-NEXT:    v_mov_b32_e32 v1, s17508; VI-NEXT:    flat_load_ubyte v0, v[0:1] glc7509; VI-NEXT:    s_waitcnt vmcnt(0)7510; VI-NEXT:    buffer_wbinvl1_vol7511; VI-NEXT:    s_mov_b32 s4, s27512; VI-NEXT:    s_mov_b32 s5, s37513; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 07514; VI-NEXT:    s_endpgm7515;7516; GFX9-LABEL: atomic_load_i8_negoffset:7517; GFX9:       ; %bb.0: ; %entry7518; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247519; GFX9-NEXT:    v_mov_b32_e32 v0, 07520; GFX9-NEXT:    s_waitcnt lgkmcnt(0)7521; GFX9-NEXT:    global_load_ubyte v1, v0, s[0:1] offset:-512 glc7522; GFX9-NEXT:    s_waitcnt vmcnt(0)7523; GFX9-NEXT:    buffer_wbinvl1_vol7524; GFX9-NEXT:    global_store_byte v0, v1, s[2:3]7525; GFX9-NEXT:    s_endpgm7526;7527; GFX11-TRUE16-LABEL: atomic_load_i8_negoffset:7528; GFX11-TRUE16:       ; %bb.0: ; %entry7529; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x247530; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 07531; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)7532; GFX11-TRUE16-NEXT:    global_load_d16_u8 v0, v1, s[0:1] offset:-512 glc7533; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)7534; GFX11-TRUE16-NEXT:    buffer_gl1_inv7535; GFX11-TRUE16-NEXT:    buffer_gl0_inv7536; GFX11-TRUE16-NEXT:    global_store_b8 v1, v0, s[2:3]7537; GFX11-TRUE16-NEXT:    s_endpgm7538;7539; GFX11-FAKE16-LABEL: atomic_load_i8_negoffset:7540; GFX11-FAKE16:       ; %bb.0: ; %entry7541; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x247542; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 07543; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)7544; GFX11-FAKE16-NEXT:    global_load_u8 v1, v0, s[0:1] offset:-512 glc7545; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)7546; GFX11-FAKE16-NEXT:    buffer_gl1_inv7547; GFX11-FAKE16-NEXT:    buffer_gl0_inv7548; GFX11-FAKE16-NEXT:    global_store_b8 v0, v1, s[2:3]7549; GFX11-FAKE16-NEXT:    s_endpgm7550entry:7551  %gep = getelementptr i8, ptr addrspace(1) %in, i64 -5127552  %val = load atomic i8, ptr addrspace(1) %gep  seq_cst, align 17553  store i8 %val, ptr addrspace(1) %out7554  ret void7555}7556 7557define amdgpu_kernel void @atomic_store_i8_offset(i8 %in, ptr addrspace(1) %out) {7558; SI-LABEL: atomic_store_i8_offset:7559; SI:       ; %bb.0: ; %entry7560; SI-NEXT:    s_load_dword s6, s[4:5], 0x97561; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb7562; SI-NEXT:    s_mov_b32 s3, 0xf0007563; SI-NEXT:    s_mov_b32 s2, -17564; SI-NEXT:    s_waitcnt lgkmcnt(0)7565; SI-NEXT:    v_mov_b32_e32 v0, s67566; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 0 offset:167567; SI-NEXT:    s_endpgm7568;7569; VI-LABEL: atomic_store_i8_offset:7570; VI:       ; %bb.0: ; %entry7571; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c7572; VI-NEXT:    s_load_dword s2, s[4:5], 0x247573; VI-NEXT:    s_waitcnt lgkmcnt(0)7574; VI-NEXT:    s_add_u32 s0, s0, 167575; VI-NEXT:    s_addc_u32 s1, s1, 07576; VI-NEXT:    v_mov_b32_e32 v0, s07577; VI-NEXT:    v_mov_b32_e32 v1, s17578; VI-NEXT:    v_mov_b32_e32 v2, s27579; VI-NEXT:    flat_store_byte v[0:1], v27580; VI-NEXT:    s_endpgm7581;7582; GFX9-LABEL: atomic_store_i8_offset:7583; GFX9:       ; %bb.0: ; %entry7584; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x247585; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c7586; GFX9-NEXT:    v_mov_b32_e32 v0, 07587; GFX9-NEXT:    s_waitcnt lgkmcnt(0)7588; GFX9-NEXT:    v_mov_b32_e32 v1, s27589; GFX9-NEXT:    global_store_byte v0, v1, s[0:1] offset:167590; GFX9-NEXT:    s_endpgm7591;7592; GFX11-TRUE16-LABEL: atomic_store_i8_offset:7593; GFX11-TRUE16:       ; %bb.0: ; %entry7594; GFX11-TRUE16-NEXT:    s_clause 0x17595; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x247596; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c7597; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 07598; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)7599; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s27600; GFX11-TRUE16-NEXT:    global_store_b8 v1, v0, s[0:1] offset:167601; GFX11-TRUE16-NEXT:    s_endpgm7602;7603; GFX11-FAKE16-LABEL: atomic_store_i8_offset:7604; GFX11-FAKE16:       ; %bb.0: ; %entry7605; GFX11-FAKE16-NEXT:    s_clause 0x17606; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x247607; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c7608; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)7609; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s27610; GFX11-FAKE16-NEXT:    global_store_b8 v0, v1, s[0:1] offset:167611; GFX11-FAKE16-NEXT:    s_endpgm7612entry:7613  %gep = getelementptr i8, ptr addrspace(1) %out, i64 167614  store atomic i8 %in, ptr addrspace(1) %gep  seq_cst, align 17615  ret void7616}7617 7618define amdgpu_kernel void @atomic_store_i8(i8 %in, ptr addrspace(1) %out) {7619; SI-LABEL: atomic_store_i8:7620; SI:       ; %bb.0: ; %entry7621; SI-NEXT:    s_load_dword s6, s[4:5], 0x97622; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb7623; SI-NEXT:    s_mov_b32 s3, 0xf0007624; SI-NEXT:    s_mov_b32 s2, -17625; SI-NEXT:    s_waitcnt lgkmcnt(0)7626; SI-NEXT:    v_mov_b32_e32 v0, s67627; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 07628; SI-NEXT:    s_endpgm7629;7630; VI-LABEL: atomic_store_i8:7631; VI:       ; %bb.0: ; %entry7632; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c7633; VI-NEXT:    s_load_dword s2, s[4:5], 0x247634; VI-NEXT:    s_waitcnt lgkmcnt(0)7635; VI-NEXT:    v_mov_b32_e32 v0, s07636; VI-NEXT:    v_mov_b32_e32 v1, s17637; VI-NEXT:    v_mov_b32_e32 v2, s27638; VI-NEXT:    flat_store_byte v[0:1], v27639; VI-NEXT:    s_endpgm7640;7641; GFX9-LABEL: atomic_store_i8:7642; GFX9:       ; %bb.0: ; %entry7643; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x247644; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c7645; GFX9-NEXT:    v_mov_b32_e32 v0, 07646; GFX9-NEXT:    s_waitcnt lgkmcnt(0)7647; GFX9-NEXT:    v_mov_b32_e32 v1, s27648; GFX9-NEXT:    global_store_byte v0, v1, s[0:1]7649; GFX9-NEXT:    s_endpgm7650;7651; GFX11-TRUE16-LABEL: atomic_store_i8:7652; GFX11-TRUE16:       ; %bb.0: ; %entry7653; GFX11-TRUE16-NEXT:    s_clause 0x17654; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x247655; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c7656; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 07657; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)7658; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s27659; GFX11-TRUE16-NEXT:    global_store_b8 v1, v0, s[0:1]7660; GFX11-TRUE16-NEXT:    s_endpgm7661;7662; GFX11-FAKE16-LABEL: atomic_store_i8:7663; GFX11-FAKE16:       ; %bb.0: ; %entry7664; GFX11-FAKE16-NEXT:    s_clause 0x17665; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x247666; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c7667; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)7668; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s27669; GFX11-FAKE16-NEXT:    global_store_b8 v0, v1, s[0:1]7670; GFX11-FAKE16-NEXT:    s_endpgm7671entry:7672  store atomic i8 %in, ptr addrspace(1) %out seq_cst, align 17673  ret void7674}7675 7676define amdgpu_kernel void @atomic_load_i16_offset(ptr addrspace(1) %in, ptr addrspace(1) %out) {7677; SI-LABEL: atomic_load_i16_offset:7678; SI:       ; %bb.0: ; %entry7679; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x97680; SI-NEXT:    s_mov_b32 s7, 0xf0007681; SI-NEXT:    s_mov_b32 s6, -17682; SI-NEXT:    s_waitcnt lgkmcnt(0)7683; SI-NEXT:    s_mov_b32 s4, s27684; SI-NEXT:    s_mov_b32 s5, s37685; SI-NEXT:    s_mov_b32 s2, s67686; SI-NEXT:    s_mov_b32 s3, s77687; SI-NEXT:    buffer_load_ushort v0, off, s[0:3], 0 offset:16 glc7688; SI-NEXT:    s_waitcnt vmcnt(0)7689; SI-NEXT:    buffer_wbinvl17690; SI-NEXT:    buffer_store_short v0, off, s[4:7], 07691; SI-NEXT:    s_endpgm7692;7693; VI-LABEL: atomic_load_i16_offset:7694; VI:       ; %bb.0: ; %entry7695; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247696; VI-NEXT:    s_mov_b32 s7, 0xf0007697; VI-NEXT:    s_mov_b32 s6, -17698; VI-NEXT:    s_waitcnt lgkmcnt(0)7699; VI-NEXT:    s_mov_b32 s4, s27700; VI-NEXT:    s_mov_b32 s5, s37701; VI-NEXT:    s_mov_b32 s2, s67702; VI-NEXT:    s_mov_b32 s3, s77703; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], 0 offset:16 glc7704; VI-NEXT:    s_waitcnt vmcnt(0)7705; VI-NEXT:    buffer_wbinvl1_vol7706; VI-NEXT:    buffer_store_short v0, off, s[4:7], 07707; VI-NEXT:    s_endpgm7708;7709; GFX9-LABEL: atomic_load_i16_offset:7710; GFX9:       ; %bb.0: ; %entry7711; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247712; GFX9-NEXT:    v_mov_b32_e32 v0, 07713; GFX9-NEXT:    s_waitcnt lgkmcnt(0)7714; GFX9-NEXT:    global_load_ushort v1, v0, s[0:1] offset:16 glc7715; GFX9-NEXT:    s_waitcnt vmcnt(0)7716; GFX9-NEXT:    buffer_wbinvl1_vol7717; GFX9-NEXT:    global_store_short v0, v1, s[2:3]7718; GFX9-NEXT:    s_endpgm7719;7720; GFX11-TRUE16-LABEL: atomic_load_i16_offset:7721; GFX11-TRUE16:       ; %bb.0: ; %entry7722; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x247723; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 07724; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)7725; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1] offset:16 glc7726; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)7727; GFX11-TRUE16-NEXT:    buffer_gl1_inv7728; GFX11-TRUE16-NEXT:    buffer_gl0_inv7729; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[2:3]7730; GFX11-TRUE16-NEXT:    s_endpgm7731;7732; GFX11-FAKE16-LABEL: atomic_load_i16_offset:7733; GFX11-FAKE16:       ; %bb.0: ; %entry7734; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x247735; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 07736; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)7737; GFX11-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1] offset:16 glc7738; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)7739; GFX11-FAKE16-NEXT:    buffer_gl1_inv7740; GFX11-FAKE16-NEXT:    buffer_gl0_inv7741; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[2:3]7742; GFX11-FAKE16-NEXT:    s_endpgm7743entry:7744  %gep = getelementptr i16, ptr addrspace(1) %in, i64 87745  %val = load atomic i16, ptr addrspace(1) %gep  seq_cst, align 27746  store i16 %val, ptr addrspace(1) %out7747  ret void7748}7749 7750define amdgpu_kernel void @atomic_load_i16_negoffset(ptr addrspace(1) %in, ptr addrspace(1) %out) {7751; SI-LABEL: atomic_load_i16_negoffset:7752; SI:       ; %bb.0: ; %entry7753; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x97754; SI-NEXT:    s_mov_b32 s7, 0xf0007755; SI-NEXT:    s_waitcnt lgkmcnt(0)7756; SI-NEXT:    s_mov_b32 s4, s27757; SI-NEXT:    s_mov_b32 s5, s37758; SI-NEXT:    s_mov_b32 s2, 07759; SI-NEXT:    s_mov_b32 s3, s77760; SI-NEXT:    v_mov_b32_e32 v0, 0xfffffe007761; SI-NEXT:    v_mov_b32_e32 v1, -17762; SI-NEXT:    buffer_load_ushort v0, v[0:1], s[0:3], 0 addr64 glc7763; SI-NEXT:    s_waitcnt vmcnt(0)7764; SI-NEXT:    buffer_wbinvl17765; SI-NEXT:    s_mov_b32 s6, -17766; SI-NEXT:    buffer_store_short v0, off, s[4:7], 07767; SI-NEXT:    s_endpgm7768;7769; VI-LABEL: atomic_load_i16_negoffset:7770; VI:       ; %bb.0: ; %entry7771; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247772; VI-NEXT:    s_mov_b32 s7, 0xf0007773; VI-NEXT:    s_mov_b32 s6, -17774; VI-NEXT:    s_waitcnt lgkmcnt(0)7775; VI-NEXT:    s_add_u32 s0, s0, 0xfffffe007776; VI-NEXT:    s_addc_u32 s1, s1, -17777; VI-NEXT:    v_mov_b32_e32 v0, s07778; VI-NEXT:    v_mov_b32_e32 v1, s17779; VI-NEXT:    flat_load_ushort v0, v[0:1] glc7780; VI-NEXT:    s_waitcnt vmcnt(0)7781; VI-NEXT:    buffer_wbinvl1_vol7782; VI-NEXT:    s_mov_b32 s4, s27783; VI-NEXT:    s_mov_b32 s5, s37784; VI-NEXT:    buffer_store_short v0, off, s[4:7], 07785; VI-NEXT:    s_endpgm7786;7787; GFX9-LABEL: atomic_load_i16_negoffset:7788; GFX9:       ; %bb.0: ; %entry7789; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247790; GFX9-NEXT:    v_mov_b32_e32 v0, 07791; GFX9-NEXT:    s_waitcnt lgkmcnt(0)7792; GFX9-NEXT:    global_load_ushort v1, v0, s[0:1] offset:-512 glc7793; GFX9-NEXT:    s_waitcnt vmcnt(0)7794; GFX9-NEXT:    buffer_wbinvl1_vol7795; GFX9-NEXT:    global_store_short v0, v1, s[2:3]7796; GFX9-NEXT:    s_endpgm7797;7798; GFX11-TRUE16-LABEL: atomic_load_i16_negoffset:7799; GFX11-TRUE16:       ; %bb.0: ; %entry7800; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x247801; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 07802; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)7803; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1] offset:-512 glc7804; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)7805; GFX11-TRUE16-NEXT:    buffer_gl1_inv7806; GFX11-TRUE16-NEXT:    buffer_gl0_inv7807; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[2:3]7808; GFX11-TRUE16-NEXT:    s_endpgm7809;7810; GFX11-FAKE16-LABEL: atomic_load_i16_negoffset:7811; GFX11-FAKE16:       ; %bb.0: ; %entry7812; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x247813; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 07814; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)7815; GFX11-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1] offset:-512 glc7816; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)7817; GFX11-FAKE16-NEXT:    buffer_gl1_inv7818; GFX11-FAKE16-NEXT:    buffer_gl0_inv7819; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[2:3]7820; GFX11-FAKE16-NEXT:    s_endpgm7821entry:7822  %gep = getelementptr i16, ptr addrspace(1) %in, i64 -2567823  %val = load atomic i16, ptr addrspace(1) %gep  seq_cst, align 27824  store i16 %val, ptr addrspace(1) %out7825  ret void7826}7827 7828define amdgpu_kernel void @atomic_store_i16_offset(i16 %in, ptr addrspace(1) %out) {7829; SI-LABEL: atomic_store_i16_offset:7830; SI:       ; %bb.0: ; %entry7831; SI-NEXT:    s_load_dword s6, s[4:5], 0x97832; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb7833; SI-NEXT:    s_mov_b32 s3, 0xf0007834; SI-NEXT:    s_mov_b32 s2, -17835; SI-NEXT:    s_waitcnt lgkmcnt(0)7836; SI-NEXT:    v_mov_b32_e32 v0, s67837; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0 offset:167838; SI-NEXT:    s_endpgm7839;7840; VI-LABEL: atomic_store_i16_offset:7841; VI:       ; %bb.0: ; %entry7842; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c7843; VI-NEXT:    s_load_dword s2, s[4:5], 0x247844; VI-NEXT:    s_waitcnt lgkmcnt(0)7845; VI-NEXT:    s_add_u32 s0, s0, 167846; VI-NEXT:    s_addc_u32 s1, s1, 07847; VI-NEXT:    v_mov_b32_e32 v0, s07848; VI-NEXT:    v_mov_b32_e32 v1, s17849; VI-NEXT:    v_mov_b32_e32 v2, s27850; VI-NEXT:    flat_store_short v[0:1], v27851; VI-NEXT:    s_endpgm7852;7853; GFX9-LABEL: atomic_store_i16_offset:7854; GFX9:       ; %bb.0: ; %entry7855; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x247856; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c7857; GFX9-NEXT:    v_mov_b32_e32 v0, 07858; GFX9-NEXT:    s_waitcnt lgkmcnt(0)7859; GFX9-NEXT:    v_mov_b32_e32 v1, s27860; GFX9-NEXT:    global_store_short v0, v1, s[0:1] offset:167861; GFX9-NEXT:    s_endpgm7862;7863; GFX11-TRUE16-LABEL: atomic_store_i16_offset:7864; GFX11-TRUE16:       ; %bb.0: ; %entry7865; GFX11-TRUE16-NEXT:    s_clause 0x17866; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x247867; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c7868; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 07869; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)7870; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s27871; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1] offset:167872; GFX11-TRUE16-NEXT:    s_endpgm7873;7874; GFX11-FAKE16-LABEL: atomic_store_i16_offset:7875; GFX11-FAKE16:       ; %bb.0: ; %entry7876; GFX11-FAKE16-NEXT:    s_clause 0x17877; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x247878; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c7879; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)7880; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s27881; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1] offset:167882; GFX11-FAKE16-NEXT:    s_endpgm7883entry:7884  %gep = getelementptr i16, ptr addrspace(1) %out, i64 87885  store atomic i16 %in, ptr addrspace(1) %gep  seq_cst, align 27886  ret void7887}7888 7889define amdgpu_kernel void @atomic_store_i16(i16 %in, ptr addrspace(1) %out) {7890; SI-LABEL: atomic_store_i16:7891; SI:       ; %bb.0: ; %entry7892; SI-NEXT:    s_load_dword s6, s[4:5], 0x97893; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb7894; SI-NEXT:    s_mov_b32 s3, 0xf0007895; SI-NEXT:    s_mov_b32 s2, -17896; SI-NEXT:    s_waitcnt lgkmcnt(0)7897; SI-NEXT:    v_mov_b32_e32 v0, s67898; SI-NEXT:    buffer_store_short v0, off, s[0:3], 07899; SI-NEXT:    s_endpgm7900;7901; VI-LABEL: atomic_store_i16:7902; VI:       ; %bb.0: ; %entry7903; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c7904; VI-NEXT:    s_load_dword s2, s[4:5], 0x247905; VI-NEXT:    s_waitcnt lgkmcnt(0)7906; VI-NEXT:    v_mov_b32_e32 v0, s07907; VI-NEXT:    v_mov_b32_e32 v1, s17908; VI-NEXT:    v_mov_b32_e32 v2, s27909; VI-NEXT:    flat_store_short v[0:1], v27910; VI-NEXT:    s_endpgm7911;7912; GFX9-LABEL: atomic_store_i16:7913; GFX9:       ; %bb.0: ; %entry7914; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x247915; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c7916; GFX9-NEXT:    v_mov_b32_e32 v0, 07917; GFX9-NEXT:    s_waitcnt lgkmcnt(0)7918; GFX9-NEXT:    v_mov_b32_e32 v1, s27919; GFX9-NEXT:    global_store_short v0, v1, s[0:1]7920; GFX9-NEXT:    s_endpgm7921;7922; GFX11-TRUE16-LABEL: atomic_store_i16:7923; GFX11-TRUE16:       ; %bb.0: ; %entry7924; GFX11-TRUE16-NEXT:    s_clause 0x17925; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x247926; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c7927; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 07928; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)7929; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s27930; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]7931; GFX11-TRUE16-NEXT:    s_endpgm7932;7933; GFX11-FAKE16-LABEL: atomic_store_i16:7934; GFX11-FAKE16:       ; %bb.0: ; %entry7935; GFX11-FAKE16-NEXT:    s_clause 0x17936; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x247937; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c7938; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)7939; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s27940; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]7941; GFX11-FAKE16-NEXT:    s_endpgm7942entry:7943  store atomic i16 %in, ptr addrspace(1) %out seq_cst, align 27944  ret void7945}7946 7947define amdgpu_kernel void @atomic_store_f16_offset(half %in, ptr addrspace(1) %out) {7948; SI-LABEL: atomic_store_f16_offset:7949; SI:       ; %bb.0: ; %entry7950; SI-NEXT:    s_load_dword s6, s[4:5], 0x97951; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb7952; SI-NEXT:    s_mov_b32 s3, 0xf0007953; SI-NEXT:    s_mov_b32 s2, -17954; SI-NEXT:    s_waitcnt lgkmcnt(0)7955; SI-NEXT:    v_mov_b32_e32 v0, s67956; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0 offset:167957; SI-NEXT:    s_endpgm7958;7959; VI-LABEL: atomic_store_f16_offset:7960; VI:       ; %bb.0: ; %entry7961; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c7962; VI-NEXT:    s_load_dword s2, s[4:5], 0x247963; VI-NEXT:    s_waitcnt lgkmcnt(0)7964; VI-NEXT:    s_add_u32 s0, s0, 167965; VI-NEXT:    s_addc_u32 s1, s1, 07966; VI-NEXT:    v_mov_b32_e32 v0, s07967; VI-NEXT:    v_mov_b32_e32 v1, s17968; VI-NEXT:    v_mov_b32_e32 v2, s27969; VI-NEXT:    flat_store_short v[0:1], v27970; VI-NEXT:    s_endpgm7971;7972; GFX9-LABEL: atomic_store_f16_offset:7973; GFX9:       ; %bb.0: ; %entry7974; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x247975; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c7976; GFX9-NEXT:    v_mov_b32_e32 v0, 07977; GFX9-NEXT:    s_waitcnt lgkmcnt(0)7978; GFX9-NEXT:    v_mov_b32_e32 v1, s27979; GFX9-NEXT:    global_store_short v0, v1, s[0:1] offset:167980; GFX9-NEXT:    s_endpgm7981;7982; GFX11-TRUE16-LABEL: atomic_store_f16_offset:7983; GFX11-TRUE16:       ; %bb.0: ; %entry7984; GFX11-TRUE16-NEXT:    s_clause 0x17985; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x247986; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c7987; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 07988; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)7989; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s27990; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1] offset:167991; GFX11-TRUE16-NEXT:    s_endpgm7992;7993; GFX11-FAKE16-LABEL: atomic_store_f16_offset:7994; GFX11-FAKE16:       ; %bb.0: ; %entry7995; GFX11-FAKE16-NEXT:    s_clause 0x17996; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x247997; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c7998; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)7999; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s28000; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1] offset:168001; GFX11-FAKE16-NEXT:    s_endpgm8002entry:8003  %gep = getelementptr half, ptr addrspace(1) %out, i64 88004  store atomic half %in, ptr addrspace(1) %gep  seq_cst, align 28005  ret void8006}8007 8008define amdgpu_kernel void @atomic_store_f16(half %in, ptr addrspace(1) %out) {8009; SI-LABEL: atomic_store_f16:8010; SI:       ; %bb.0: ; %entry8011; SI-NEXT:    s_load_dword s6, s[4:5], 0x98012; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb8013; SI-NEXT:    s_mov_b32 s3, 0xf0008014; SI-NEXT:    s_mov_b32 s2, -18015; SI-NEXT:    s_waitcnt lgkmcnt(0)8016; SI-NEXT:    v_mov_b32_e32 v0, s68017; SI-NEXT:    buffer_store_short v0, off, s[0:3], 08018; SI-NEXT:    s_endpgm8019;8020; VI-LABEL: atomic_store_f16:8021; VI:       ; %bb.0: ; %entry8022; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c8023; VI-NEXT:    s_load_dword s2, s[4:5], 0x248024; VI-NEXT:    s_waitcnt lgkmcnt(0)8025; VI-NEXT:    v_mov_b32_e32 v0, s08026; VI-NEXT:    v_mov_b32_e32 v1, s18027; VI-NEXT:    v_mov_b32_e32 v2, s28028; VI-NEXT:    flat_store_short v[0:1], v28029; VI-NEXT:    s_endpgm8030;8031; GFX9-LABEL: atomic_store_f16:8032; GFX9:       ; %bb.0: ; %entry8033; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x248034; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c8035; GFX9-NEXT:    v_mov_b32_e32 v0, 08036; GFX9-NEXT:    s_waitcnt lgkmcnt(0)8037; GFX9-NEXT:    v_mov_b32_e32 v1, s28038; GFX9-NEXT:    global_store_short v0, v1, s[0:1]8039; GFX9-NEXT:    s_endpgm8040;8041; GFX11-TRUE16-LABEL: atomic_store_f16:8042; GFX11-TRUE16:       ; %bb.0: ; %entry8043; GFX11-TRUE16-NEXT:    s_clause 0x18044; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x248045; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c8046; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 08047; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)8048; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s28049; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]8050; GFX11-TRUE16-NEXT:    s_endpgm8051;8052; GFX11-FAKE16-LABEL: atomic_store_f16:8053; GFX11-FAKE16:       ; %bb.0: ; %entry8054; GFX11-FAKE16-NEXT:    s_clause 0x18055; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x248056; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c8057; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)8058; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s28059; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]8060; GFX11-FAKE16-NEXT:    s_endpgm8061entry:8062  store atomic half %in, ptr addrspace(1) %out seq_cst, align 28063  ret void8064}8065 8066define amdgpu_kernel void @atomic_store_bf16_offset(bfloat %in, ptr addrspace(1) %out) {8067; SI-LABEL: atomic_store_bf16_offset:8068; SI:       ; %bb.0:8069; SI-NEXT:    s_load_dword s6, s[4:5], 0x98070; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb8071; SI-NEXT:    s_mov_b32 s3, 0xf0008072; SI-NEXT:    s_mov_b32 s2, -18073; SI-NEXT:    s_waitcnt lgkmcnt(0)8074; SI-NEXT:    v_mov_b32_e32 v0, s68075; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0 offset:168076; SI-NEXT:    s_endpgm8077;8078; VI-LABEL: atomic_store_bf16_offset:8079; VI:       ; %bb.0:8080; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c8081; VI-NEXT:    s_load_dword s2, s[4:5], 0x248082; VI-NEXT:    s_waitcnt lgkmcnt(0)8083; VI-NEXT:    s_add_u32 s0, s0, 168084; VI-NEXT:    s_addc_u32 s1, s1, 08085; VI-NEXT:    v_mov_b32_e32 v0, s08086; VI-NEXT:    v_mov_b32_e32 v1, s18087; VI-NEXT:    v_mov_b32_e32 v2, s28088; VI-NEXT:    flat_store_short v[0:1], v28089; VI-NEXT:    s_endpgm8090;8091; GFX9-LABEL: atomic_store_bf16_offset:8092; GFX9:       ; %bb.0:8093; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x248094; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c8095; GFX9-NEXT:    v_mov_b32_e32 v0, 08096; GFX9-NEXT:    s_waitcnt lgkmcnt(0)8097; GFX9-NEXT:    v_mov_b32_e32 v1, s28098; GFX9-NEXT:    global_store_short v0, v1, s[0:1] offset:168099; GFX9-NEXT:    s_endpgm8100;8101; GFX11-TRUE16-LABEL: atomic_store_bf16_offset:8102; GFX11-TRUE16:       ; %bb.0:8103; GFX11-TRUE16-NEXT:    s_clause 0x18104; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x248105; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c8106; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 08107; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)8108; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s28109; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1] offset:168110; GFX11-TRUE16-NEXT:    s_endpgm8111;8112; GFX11-FAKE16-LABEL: atomic_store_bf16_offset:8113; GFX11-FAKE16:       ; %bb.0:8114; GFX11-FAKE16-NEXT:    s_clause 0x18115; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x248116; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c8117; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)8118; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s28119; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1] offset:168120; GFX11-FAKE16-NEXT:    s_endpgm8121  %gep = getelementptr bfloat, ptr addrspace(1) %out, i64 88122  store atomic bfloat %in, ptr addrspace(1) %gep seq_cst, align 28123  ret void8124}8125 8126define amdgpu_kernel void @atomic_store_bf16(bfloat %in, ptr addrspace(1) %out) {8127; SI-LABEL: atomic_store_bf16:8128; SI:       ; %bb.0:8129; SI-NEXT:    s_load_dword s6, s[4:5], 0x98130; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb8131; SI-NEXT:    s_mov_b32 s3, 0xf0008132; SI-NEXT:    s_mov_b32 s2, -18133; SI-NEXT:    s_waitcnt lgkmcnt(0)8134; SI-NEXT:    v_mov_b32_e32 v0, s68135; SI-NEXT:    buffer_store_short v0, off, s[0:3], 08136; SI-NEXT:    s_endpgm8137;8138; VI-LABEL: atomic_store_bf16:8139; VI:       ; %bb.0:8140; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c8141; VI-NEXT:    s_load_dword s2, s[4:5], 0x248142; VI-NEXT:    s_waitcnt lgkmcnt(0)8143; VI-NEXT:    v_mov_b32_e32 v0, s08144; VI-NEXT:    v_mov_b32_e32 v1, s18145; VI-NEXT:    v_mov_b32_e32 v2, s28146; VI-NEXT:    flat_store_short v[0:1], v28147; VI-NEXT:    s_endpgm8148;8149; GFX9-LABEL: atomic_store_bf16:8150; GFX9:       ; %bb.0:8151; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x248152; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c8153; GFX9-NEXT:    v_mov_b32_e32 v0, 08154; GFX9-NEXT:    s_waitcnt lgkmcnt(0)8155; GFX9-NEXT:    v_mov_b32_e32 v1, s28156; GFX9-NEXT:    global_store_short v0, v1, s[0:1]8157; GFX9-NEXT:    s_endpgm8158;8159; GFX11-TRUE16-LABEL: atomic_store_bf16:8160; GFX11-TRUE16:       ; %bb.0:8161; GFX11-TRUE16-NEXT:    s_clause 0x18162; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x248163; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c8164; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 08165; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)8166; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, s28167; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]8168; GFX11-TRUE16-NEXT:    s_endpgm8169;8170; GFX11-FAKE16-LABEL: atomic_store_bf16:8171; GFX11-FAKE16:       ; %bb.0:8172; GFX11-FAKE16-NEXT:    s_clause 0x18173; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x248174; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c8175; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)8176; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s28177; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]8178; GFX11-FAKE16-NEXT:    s_endpgm8179  store atomic bfloat %in, ptr addrspace(1) %out seq_cst, align 28180  ret void8181}8182 8183define amdgpu_kernel void @atomic_inc_i32_offset(ptr addrspace(1) %out, i32 %in) {8184; SI-LABEL: atomic_inc_i32_offset:8185; SI:       ; %bb.0: ; %entry8186; SI-NEXT:    s_load_dword s6, s[4:5], 0xb8187; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x98188; SI-NEXT:    s_mov_b32 s3, 0xf0008189; SI-NEXT:    s_mov_b32 s2, -18190; SI-NEXT:    s_waitcnt lgkmcnt(0)8191; SI-NEXT:    v_mov_b32_e32 v0, s68192; SI-NEXT:    buffer_atomic_inc v0, off, s[0:3], 0 offset:168193; SI-NEXT:    s_waitcnt vmcnt(0)8194; SI-NEXT:    buffer_wbinvl18195; SI-NEXT:    s_endpgm8196;8197; VI-LABEL: atomic_inc_i32_offset:8198; VI:       ; %bb.0: ; %entry8199; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c8200; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x248201; VI-NEXT:    s_mov_b32 s3, 0xf0008202; VI-NEXT:    s_mov_b32 s2, -18203; VI-NEXT:    s_waitcnt lgkmcnt(0)8204; VI-NEXT:    v_mov_b32_e32 v0, s68205; VI-NEXT:    buffer_atomic_inc v0, off, s[0:3], 0 offset:168206; VI-NEXT:    s_waitcnt vmcnt(0)8207; VI-NEXT:    buffer_wbinvl1_vol8208; VI-NEXT:    s_endpgm8209;8210; GFX9-LABEL: atomic_inc_i32_offset:8211; GFX9:       ; %bb.0: ; %entry8212; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c8213; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x248214; GFX9-NEXT:    v_mov_b32_e32 v0, 08215; GFX9-NEXT:    s_waitcnt lgkmcnt(0)8216; GFX9-NEXT:    v_mov_b32_e32 v1, s28217; GFX9-NEXT:    global_atomic_inc v0, v1, s[0:1] offset:168218; GFX9-NEXT:    s_waitcnt vmcnt(0)8219; GFX9-NEXT:    buffer_wbinvl1_vol8220; GFX9-NEXT:    s_endpgm8221;8222; GFX11-LABEL: atomic_inc_i32_offset:8223; GFX11:       ; %bb.0: ; %entry8224; GFX11-NEXT:    s_clause 0x18225; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c8226; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x248227; GFX11-NEXT:    s_waitcnt lgkmcnt(0)8228; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s28229; GFX11-NEXT:    global_atomic_inc_u32 v0, v1, s[0:1] offset:168230; GFX11-NEXT:    s_waitcnt_vscnt null, 0x08231; GFX11-NEXT:    buffer_gl1_inv8232; GFX11-NEXT:    buffer_gl0_inv8233; GFX11-NEXT:    s_endpgm8234entry:8235  %gep = getelementptr i32, ptr addrspace(1) %out, i64 48236  %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !08237  ret void8238}8239 8240define amdgpu_kernel void @atomic_inc_i32_max_neg_offset(ptr addrspace(1) %out, i32 %in) {8241; SI-LABEL: atomic_inc_i32_max_neg_offset:8242; SI:       ; %bb.0: ; %entry8243; SI-NEXT:    s_load_dword s6, s[4:5], 0xb8244; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x98245; SI-NEXT:    s_mov_b32 s3, 0xf0008246; SI-NEXT:    s_mov_b32 s2, 08247; SI-NEXT:    v_mov_b32_e32 v0, 0xfffff0008248; SI-NEXT:    v_mov_b32_e32 v1, -18249; SI-NEXT:    s_waitcnt lgkmcnt(0)8250; SI-NEXT:    v_mov_b32_e32 v2, s68251; SI-NEXT:    buffer_atomic_inc v2, v[0:1], s[0:3], 0 addr648252; SI-NEXT:    s_waitcnt vmcnt(0)8253; SI-NEXT:    buffer_wbinvl18254; SI-NEXT:    s_endpgm8255;8256; VI-LABEL: atomic_inc_i32_max_neg_offset:8257; VI:       ; %bb.0: ; %entry8258; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x248259; VI-NEXT:    s_load_dword s2, s[4:5], 0x2c8260; VI-NEXT:    s_waitcnt lgkmcnt(0)8261; VI-NEXT:    s_add_u32 s0, s0, 0xfffff0008262; VI-NEXT:    s_addc_u32 s1, s1, -18263; VI-NEXT:    v_mov_b32_e32 v0, s08264; VI-NEXT:    v_mov_b32_e32 v1, s18265; VI-NEXT:    v_mov_b32_e32 v2, s28266; VI-NEXT:    flat_atomic_inc v[0:1], v28267; VI-NEXT:    s_waitcnt vmcnt(0)8268; VI-NEXT:    buffer_wbinvl1_vol8269; VI-NEXT:    s_endpgm8270;8271; GFX9-LABEL: atomic_inc_i32_max_neg_offset:8272; GFX9:       ; %bb.0: ; %entry8273; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c8274; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x248275; GFX9-NEXT:    v_mov_b32_e32 v0, 08276; GFX9-NEXT:    s_waitcnt lgkmcnt(0)8277; GFX9-NEXT:    v_mov_b32_e32 v1, s28278; GFX9-NEXT:    global_atomic_inc v0, v1, s[0:1] offset:-40968279; GFX9-NEXT:    s_waitcnt vmcnt(0)8280; GFX9-NEXT:    buffer_wbinvl1_vol8281; GFX9-NEXT:    s_endpgm8282;8283; GFX11-LABEL: atomic_inc_i32_max_neg_offset:8284; GFX11:       ; %bb.0: ; %entry8285; GFX11-NEXT:    s_clause 0x18286; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c8287; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x248288; GFX11-NEXT:    s_waitcnt lgkmcnt(0)8289; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s28290; GFX11-NEXT:    global_atomic_inc_u32 v0, v1, s[0:1] offset:-40968291; GFX11-NEXT:    s_waitcnt_vscnt null, 0x08292; GFX11-NEXT:    buffer_gl1_inv8293; GFX11-NEXT:    buffer_gl0_inv8294; GFX11-NEXT:    s_endpgm8295entry:8296  %gep = getelementptr i32, ptr addrspace(1) %out, i64 -10248297  %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !08298  ret void8299}8300 8301define amdgpu_kernel void @atomic_inc_i32_soffset(ptr addrspace(1) %out, i32 %in) {8302; SI-LABEL: atomic_inc_i32_soffset:8303; SI:       ; %bb.0: ; %entry8304; SI-NEXT:    s_load_dword s6, s[4:5], 0xb8305; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x98306; SI-NEXT:    s_mov_b32 s3, 0xf0008307; SI-NEXT:    s_mov_b32 s2, -18308; SI-NEXT:    s_mov_b32 s4, 0x8ca08309; SI-NEXT:    s_waitcnt lgkmcnt(0)8310; SI-NEXT:    v_mov_b32_e32 v0, s68311; SI-NEXT:    buffer_atomic_inc v0, off, s[0:3], s48312; SI-NEXT:    s_waitcnt vmcnt(0)8313; SI-NEXT:    buffer_wbinvl18314; SI-NEXT:    s_endpgm8315;8316; VI-LABEL: atomic_inc_i32_soffset:8317; VI:       ; %bb.0: ; %entry8318; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c8319; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x248320; VI-NEXT:    s_mov_b32 s3, 0xf0008321; VI-NEXT:    s_mov_b32 s2, -18322; VI-NEXT:    s_mov_b32 s4, 0x8ca08323; VI-NEXT:    s_waitcnt lgkmcnt(0)8324; VI-NEXT:    v_mov_b32_e32 v0, s68325; VI-NEXT:    buffer_atomic_inc v0, off, s[0:3], s48326; VI-NEXT:    s_waitcnt vmcnt(0)8327; VI-NEXT:    buffer_wbinvl1_vol8328; VI-NEXT:    s_endpgm8329;8330; GFX9-LABEL: atomic_inc_i32_soffset:8331; GFX9:       ; %bb.0: ; %entry8332; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c8333; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x248334; GFX9-NEXT:    v_mov_b32_e32 v0, 0x80008335; GFX9-NEXT:    s_waitcnt lgkmcnt(0)8336; GFX9-NEXT:    v_mov_b32_e32 v1, s28337; GFX9-NEXT:    global_atomic_inc v0, v1, s[0:1] offset:32328338; GFX9-NEXT:    s_waitcnt vmcnt(0)8339; GFX9-NEXT:    buffer_wbinvl1_vol8340; GFX9-NEXT:    s_endpgm8341;8342; GFX11-LABEL: atomic_inc_i32_soffset:8343; GFX11:       ; %bb.0: ; %entry8344; GFX11-NEXT:    s_clause 0x18345; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c8346; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x248347; GFX11-NEXT:    s_waitcnt lgkmcnt(0)8348; GFX11-NEXT:    v_dual_mov_b32 v0, 0x8000 :: v_dual_mov_b32 v1, s28349; GFX11-NEXT:    global_atomic_inc_u32 v0, v1, s[0:1] offset:32328350; GFX11-NEXT:    s_waitcnt_vscnt null, 0x08351; GFX11-NEXT:    buffer_gl1_inv8352; GFX11-NEXT:    buffer_gl0_inv8353; GFX11-NEXT:    s_endpgm8354entry:8355  %gep = getelementptr i32, ptr addrspace(1) %out, i64 90008356  %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !08357  ret void8358}8359 8360define amdgpu_kernel void @atomic_inc_i32_huge_offset(ptr addrspace(1) %out, i32 %in) {8361; SI-LABEL: atomic_inc_i32_huge_offset:8362; SI:       ; %bb.0: ; %entry8363; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x98364; SI-NEXT:    s_load_dword s4, s[4:5], 0xb8365; SI-NEXT:    s_mov_b32 s3, 0xf0008366; SI-NEXT:    s_mov_b32 s2, 08367; SI-NEXT:    v_mov_b32_e32 v0, 0xdeac8368; SI-NEXT:    v_mov_b32_e32 v1, 0xabcd8369; SI-NEXT:    s_waitcnt lgkmcnt(0)8370; SI-NEXT:    v_mov_b32_e32 v2, s48371; SI-NEXT:    buffer_atomic_inc v2, v[0:1], s[0:3], 0 addr648372; SI-NEXT:    s_waitcnt vmcnt(0)8373; SI-NEXT:    buffer_wbinvl18374; SI-NEXT:    s_endpgm8375;8376; VI-LABEL: atomic_inc_i32_huge_offset:8377; VI:       ; %bb.0: ; %entry8378; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x248379; VI-NEXT:    s_load_dword s2, s[4:5], 0x2c8380; VI-NEXT:    s_waitcnt lgkmcnt(0)8381; VI-NEXT:    s_add_u32 s0, s0, 0xdeac8382; VI-NEXT:    s_addc_u32 s1, s1, 0xabcd8383; VI-NEXT:    v_mov_b32_e32 v0, s08384; VI-NEXT:    v_mov_b32_e32 v1, s18385; VI-NEXT:    v_mov_b32_e32 v2, s28386; VI-NEXT:    flat_atomic_inc v[0:1], v28387; VI-NEXT:    s_waitcnt vmcnt(0)8388; VI-NEXT:    buffer_wbinvl1_vol8389; VI-NEXT:    s_endpgm8390;8391; GFX9-LABEL: atomic_inc_i32_huge_offset:8392; GFX9:       ; %bb.0: ; %entry8393; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x248394; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c8395; GFX9-NEXT:    v_mov_b32_e32 v0, 08396; GFX9-NEXT:    s_waitcnt lgkmcnt(0)8397; GFX9-NEXT:    s_add_u32 s0, s0, 0xdeac8398; GFX9-NEXT:    s_addc_u32 s1, s1, 0xabcd8399; GFX9-NEXT:    v_mov_b32_e32 v1, s28400; GFX9-NEXT:    global_atomic_inc v0, v1, s[0:1]8401; GFX9-NEXT:    s_waitcnt vmcnt(0)8402; GFX9-NEXT:    buffer_wbinvl1_vol8403; GFX9-NEXT:    s_endpgm8404;8405; GFX11-LABEL: atomic_inc_i32_huge_offset:8406; GFX11:       ; %bb.0: ; %entry8407; GFX11-NEXT:    s_clause 0x18408; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c8409; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x248410; GFX11-NEXT:    s_waitcnt lgkmcnt(0)8411; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s28412; GFX11-NEXT:    s_add_u32 s0, s0, 0xdeac8413; GFX11-NEXT:    s_addc_u32 s1, s1, 0xabcd8414; GFX11-NEXT:    global_atomic_inc_u32 v0, v1, s[0:1]8415; GFX11-NEXT:    s_waitcnt_vscnt null, 0x08416; GFX11-NEXT:    buffer_gl1_inv8417; GFX11-NEXT:    buffer_gl0_inv8418; GFX11-NEXT:    s_endpgm8419entry:8420  %gep = getelementptr i32, ptr addrspace(1) %out, i64 472242391755958421  %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !08422  ret void8423}8424 8425define amdgpu_kernel void @atomic_inc_i32_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) {8426; SI-LABEL: atomic_inc_i32_ret_offset:8427; SI:       ; %bb.0: ; %entry8428; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x98429; SI-NEXT:    s_load_dword s8, s[4:5], 0xd8430; SI-NEXT:    s_mov_b32 s7, 0xf0008431; SI-NEXT:    s_mov_b32 s6, -18432; SI-NEXT:    s_waitcnt lgkmcnt(0)8433; SI-NEXT:    s_mov_b32 s4, s28434; SI-NEXT:    s_mov_b32 s5, s38435; SI-NEXT:    s_mov_b32 s2, s68436; SI-NEXT:    s_mov_b32 s3, s78437; SI-NEXT:    v_mov_b32_e32 v0, s88438; SI-NEXT:    buffer_atomic_inc v0, off, s[0:3], 0 offset:16 glc8439; SI-NEXT:    s_waitcnt vmcnt(0)8440; SI-NEXT:    buffer_wbinvl18441; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 08442; SI-NEXT:    s_endpgm8443;8444; VI-LABEL: atomic_inc_i32_ret_offset:8445; VI:       ; %bb.0: ; %entry8446; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x248447; VI-NEXT:    s_load_dword s8, s[4:5], 0x348448; VI-NEXT:    s_mov_b32 s7, 0xf0008449; VI-NEXT:    s_mov_b32 s6, -18450; VI-NEXT:    s_waitcnt lgkmcnt(0)8451; VI-NEXT:    s_mov_b32 s4, s28452; VI-NEXT:    s_mov_b32 s5, s38453; VI-NEXT:    s_mov_b32 s2, s68454; VI-NEXT:    s_mov_b32 s3, s78455; VI-NEXT:    v_mov_b32_e32 v0, s88456; VI-NEXT:    buffer_atomic_inc v0, off, s[0:3], 0 offset:16 glc8457; VI-NEXT:    s_waitcnt vmcnt(0)8458; VI-NEXT:    buffer_wbinvl1_vol8459; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 08460; VI-NEXT:    s_endpgm8461;8462; GFX9-LABEL: atomic_inc_i32_ret_offset:8463; GFX9:       ; %bb.0: ; %entry8464; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x348465; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x248466; GFX9-NEXT:    v_mov_b32_e32 v0, 08467; GFX9-NEXT:    s_waitcnt lgkmcnt(0)8468; GFX9-NEXT:    v_mov_b32_e32 v1, s68469; GFX9-NEXT:    global_atomic_inc v1, v0, v1, s[0:1] offset:16 glc8470; GFX9-NEXT:    s_waitcnt vmcnt(0)8471; GFX9-NEXT:    buffer_wbinvl1_vol8472; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]8473; GFX9-NEXT:    s_endpgm8474;8475; GFX11-LABEL: atomic_inc_i32_ret_offset:8476; GFX11:       ; %bb.0: ; %entry8477; GFX11-NEXT:    s_clause 0x18478; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x348479; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x248480; GFX11-NEXT:    s_waitcnt lgkmcnt(0)8481; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s68482; GFX11-NEXT:    global_atomic_inc_u32 v1, v0, v1, s[0:1] offset:16 glc8483; GFX11-NEXT:    s_waitcnt vmcnt(0)8484; GFX11-NEXT:    buffer_gl1_inv8485; GFX11-NEXT:    buffer_gl0_inv8486; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]8487; GFX11-NEXT:    s_endpgm8488entry:8489  %gep = getelementptr i32, ptr addrspace(1) %out, i64 48490  %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !08491  store i32 %val, ptr addrspace(1) %out28492  ret void8493}8494 8495define amdgpu_kernel void @atomic_inc_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {8496; SI-LABEL: atomic_inc_i32_addr64_offset:8497; SI:       ; %bb.0: ; %entry8498; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd8499; SI-NEXT:    s_load_dword s8, s[4:5], 0xb8500; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x98501; SI-NEXT:    s_mov_b32 s3, 0xf0008502; SI-NEXT:    s_waitcnt lgkmcnt(0)8503; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 28504; SI-NEXT:    s_mov_b32 s2, 08505; SI-NEXT:    v_mov_b32_e32 v2, s88506; SI-NEXT:    v_mov_b32_e32 v0, s48507; SI-NEXT:    v_mov_b32_e32 v1, s58508; SI-NEXT:    buffer_atomic_inc v2, v[0:1], s[0:3], 0 addr64 offset:168509; SI-NEXT:    s_waitcnt vmcnt(0)8510; SI-NEXT:    buffer_wbinvl18511; SI-NEXT:    s_endpgm8512;8513; VI-LABEL: atomic_inc_i32_addr64_offset:8514; VI:       ; %bb.0: ; %entry8515; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x348516; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x248517; VI-NEXT:    s_load_dword s4, s[4:5], 0x2c8518; VI-NEXT:    s_waitcnt lgkmcnt(0)8519; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 28520; VI-NEXT:    s_add_u32 s0, s2, s08521; VI-NEXT:    s_addc_u32 s1, s3, s18522; VI-NEXT:    s_add_u32 s0, s0, 168523; VI-NEXT:    s_addc_u32 s1, s1, 08524; VI-NEXT:    v_mov_b32_e32 v0, s08525; VI-NEXT:    v_mov_b32_e32 v1, s18526; VI-NEXT:    v_mov_b32_e32 v2, s48527; VI-NEXT:    flat_atomic_inc v[0:1], v28528; VI-NEXT:    s_waitcnt vmcnt(0)8529; VI-NEXT:    buffer_wbinvl1_vol8530; VI-NEXT:    s_endpgm8531;8532; GFX9-LABEL: atomic_inc_i32_addr64_offset:8533; GFX9:       ; %bb.0: ; %entry8534; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x348535; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x248536; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c8537; GFX9-NEXT:    v_mov_b32_e32 v0, 08538; GFX9-NEXT:    s_waitcnt lgkmcnt(0)8539; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 28540; GFX9-NEXT:    s_add_u32 s0, s2, s08541; GFX9-NEXT:    s_addc_u32 s1, s3, s18542; GFX9-NEXT:    v_mov_b32_e32 v1, s68543; GFX9-NEXT:    global_atomic_inc v0, v1, s[0:1] offset:168544; GFX9-NEXT:    s_waitcnt vmcnt(0)8545; GFX9-NEXT:    buffer_wbinvl1_vol8546; GFX9-NEXT:    s_endpgm8547;8548; GFX11-LABEL: atomic_inc_i32_addr64_offset:8549; GFX11:       ; %bb.0: ; %entry8550; GFX11-NEXT:    s_clause 0x28551; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x348552; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c8553; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x248554; GFX11-NEXT:    v_mov_b32_e32 v0, 08555; GFX11-NEXT:    s_waitcnt lgkmcnt(0)8556; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 28557; GFX11-NEXT:    v_mov_b32_e32 v1, s68558; GFX11-NEXT:    s_add_u32 s0, s2, s08559; GFX11-NEXT:    s_addc_u32 s1, s3, s18560; GFX11-NEXT:    global_atomic_inc_u32 v0, v1, s[0:1] offset:168561; GFX11-NEXT:    s_waitcnt_vscnt null, 0x08562; GFX11-NEXT:    buffer_gl1_inv8563; GFX11-NEXT:    buffer_gl0_inv8564; GFX11-NEXT:    s_endpgm8565entry:8566  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index8567  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 48568  %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !08569  ret void8570}8571 8572define amdgpu_kernel void @atomic_inc_i32_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) {8573; SI-LABEL: atomic_inc_i32_ret_addr64_offset:8574; SI:       ; %bb.0: ; %entry8575; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x98576; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf8577; SI-NEXT:    s_load_dword s6, s[4:5], 0xd8578; SI-NEXT:    s_mov_b32 s7, 0xf0008579; SI-NEXT:    s_waitcnt lgkmcnt(0)8580; SI-NEXT:    s_mov_b32 s4, s28581; SI-NEXT:    s_mov_b32 s5, s38582; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 28583; SI-NEXT:    s_mov_b32 s2, 08584; SI-NEXT:    s_mov_b32 s3, s78585; SI-NEXT:    v_mov_b32_e32 v2, s68586; SI-NEXT:    v_mov_b32_e32 v0, s88587; SI-NEXT:    v_mov_b32_e32 v1, s98588; SI-NEXT:    buffer_atomic_inc v2, v[0:1], s[0:3], 0 addr64 offset:16 glc8589; SI-NEXT:    s_waitcnt vmcnt(0)8590; SI-NEXT:    buffer_wbinvl18591; SI-NEXT:    s_mov_b32 s6, -18592; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 08593; SI-NEXT:    s_endpgm8594;8595; VI-LABEL: atomic_inc_i32_ret_addr64_offset:8596; VI:       ; %bb.0: ; %entry8597; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c8598; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x248599; VI-NEXT:    s_load_dword s8, s[4:5], 0x348600; VI-NEXT:    s_waitcnt lgkmcnt(0)8601; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 28602; VI-NEXT:    s_add_u32 s0, s0, s48603; VI-NEXT:    s_addc_u32 s1, s1, s58604; VI-NEXT:    s_add_u32 s0, s0, 168605; VI-NEXT:    s_addc_u32 s1, s1, 08606; VI-NEXT:    v_mov_b32_e32 v0, s08607; VI-NEXT:    v_mov_b32_e32 v1, s18608; VI-NEXT:    v_mov_b32_e32 v2, s88609; VI-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc8610; VI-NEXT:    s_waitcnt vmcnt(0)8611; VI-NEXT:    buffer_wbinvl1_vol8612; VI-NEXT:    s_mov_b32 s7, 0xf0008613; VI-NEXT:    s_mov_b32 s6, -18614; VI-NEXT:    s_mov_b32 s4, s28615; VI-NEXT:    s_mov_b32 s5, s38616; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 08617; VI-NEXT:    s_endpgm8618;8619; GFX9-LABEL: atomic_inc_i32_ret_addr64_offset:8620; GFX9:       ; %bb.0: ; %entry8621; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c8622; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x248623; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x348624; GFX9-NEXT:    v_mov_b32_e32 v0, 08625; GFX9-NEXT:    s_waitcnt lgkmcnt(0)8626; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 28627; GFX9-NEXT:    s_add_u32 s0, s0, s48628; GFX9-NEXT:    s_addc_u32 s1, s1, s58629; GFX9-NEXT:    v_mov_b32_e32 v1, s88630; GFX9-NEXT:    global_atomic_inc v1, v0, v1, s[0:1] offset:16 glc8631; GFX9-NEXT:    s_waitcnt vmcnt(0)8632; GFX9-NEXT:    buffer_wbinvl1_vol8633; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]8634; GFX9-NEXT:    s_endpgm8635;8636; GFX11-LABEL: atomic_inc_i32_ret_addr64_offset:8637; GFX11:       ; %bb.0: ; %entry8638; GFX11-NEXT:    s_clause 0x28639; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c8640; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x348641; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x248642; GFX11-NEXT:    v_mov_b32_e32 v0, 08643; GFX11-NEXT:    s_waitcnt lgkmcnt(0)8644; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 28645; GFX11-NEXT:    v_mov_b32_e32 v1, s88646; GFX11-NEXT:    s_add_u32 s0, s0, s48647; GFX11-NEXT:    s_addc_u32 s1, s1, s58648; GFX11-NEXT:    global_atomic_inc_u32 v1, v0, v1, s[0:1] offset:16 glc8649; GFX11-NEXT:    s_waitcnt vmcnt(0)8650; GFX11-NEXT:    buffer_gl1_inv8651; GFX11-NEXT:    buffer_gl0_inv8652; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]8653; GFX11-NEXT:    s_endpgm8654entry:8655  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index8656  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 48657  %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !08658  store i32 %val, ptr addrspace(1) %out28659  ret void8660}8661 8662define amdgpu_kernel void @atomic_dec_i32_offset(ptr addrspace(1) %out, i32 %in) {8663; SI-LABEL: atomic_dec_i32_offset:8664; SI:       ; %bb.0: ; %entry8665; SI-NEXT:    s_load_dword s6, s[4:5], 0xb8666; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x98667; SI-NEXT:    s_mov_b32 s3, 0xf0008668; SI-NEXT:    s_mov_b32 s2, -18669; SI-NEXT:    s_waitcnt lgkmcnt(0)8670; SI-NEXT:    v_mov_b32_e32 v0, s68671; SI-NEXT:    buffer_atomic_dec v0, off, s[0:3], 0 offset:168672; SI-NEXT:    s_waitcnt vmcnt(0)8673; SI-NEXT:    buffer_wbinvl18674; SI-NEXT:    s_endpgm8675;8676; VI-LABEL: atomic_dec_i32_offset:8677; VI:       ; %bb.0: ; %entry8678; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c8679; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x248680; VI-NEXT:    s_mov_b32 s3, 0xf0008681; VI-NEXT:    s_mov_b32 s2, -18682; VI-NEXT:    s_waitcnt lgkmcnt(0)8683; VI-NEXT:    v_mov_b32_e32 v0, s68684; VI-NEXT:    buffer_atomic_dec v0, off, s[0:3], 0 offset:168685; VI-NEXT:    s_waitcnt vmcnt(0)8686; VI-NEXT:    buffer_wbinvl1_vol8687; VI-NEXT:    s_endpgm8688;8689; GFX9-LABEL: atomic_dec_i32_offset:8690; GFX9:       ; %bb.0: ; %entry8691; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c8692; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x248693; GFX9-NEXT:    v_mov_b32_e32 v0, 08694; GFX9-NEXT:    s_waitcnt lgkmcnt(0)8695; GFX9-NEXT:    v_mov_b32_e32 v1, s28696; GFX9-NEXT:    global_atomic_dec v0, v1, s[0:1] offset:168697; GFX9-NEXT:    s_waitcnt vmcnt(0)8698; GFX9-NEXT:    buffer_wbinvl1_vol8699; GFX9-NEXT:    s_endpgm8700;8701; GFX11-LABEL: atomic_dec_i32_offset:8702; GFX11:       ; %bb.0: ; %entry8703; GFX11-NEXT:    s_clause 0x18704; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c8705; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x248706; GFX11-NEXT:    s_waitcnt lgkmcnt(0)8707; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s28708; GFX11-NEXT:    global_atomic_dec_u32 v0, v1, s[0:1] offset:168709; GFX11-NEXT:    s_waitcnt_vscnt null, 0x08710; GFX11-NEXT:    buffer_gl1_inv8711; GFX11-NEXT:    buffer_gl0_inv8712; GFX11-NEXT:    s_endpgm8713entry:8714  %gep = getelementptr i32, ptr addrspace(1) %out, i64 48715  %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !08716  ret void8717}8718 8719define amdgpu_kernel void @atomic_dec_i32_max_neg_offset(ptr addrspace(1) %out, i32 %in) {8720; SI-LABEL: atomic_dec_i32_max_neg_offset:8721; SI:       ; %bb.0: ; %entry8722; SI-NEXT:    s_load_dword s6, s[4:5], 0xb8723; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x98724; SI-NEXT:    s_mov_b32 s3, 0xf0008725; SI-NEXT:    s_mov_b32 s2, 08726; SI-NEXT:    v_mov_b32_e32 v0, 0xfffff0008727; SI-NEXT:    v_mov_b32_e32 v1, -18728; SI-NEXT:    s_waitcnt lgkmcnt(0)8729; SI-NEXT:    v_mov_b32_e32 v2, s68730; SI-NEXT:    buffer_atomic_dec v2, v[0:1], s[0:3], 0 addr648731; SI-NEXT:    s_waitcnt vmcnt(0)8732; SI-NEXT:    buffer_wbinvl18733; SI-NEXT:    s_endpgm8734;8735; VI-LABEL: atomic_dec_i32_max_neg_offset:8736; VI:       ; %bb.0: ; %entry8737; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x248738; VI-NEXT:    s_load_dword s2, s[4:5], 0x2c8739; VI-NEXT:    s_waitcnt lgkmcnt(0)8740; VI-NEXT:    s_add_u32 s0, s0, 0xfffff0008741; VI-NEXT:    s_addc_u32 s1, s1, -18742; VI-NEXT:    v_mov_b32_e32 v0, s08743; VI-NEXT:    v_mov_b32_e32 v1, s18744; VI-NEXT:    v_mov_b32_e32 v2, s28745; VI-NEXT:    flat_atomic_dec v[0:1], v28746; VI-NEXT:    s_waitcnt vmcnt(0)8747; VI-NEXT:    buffer_wbinvl1_vol8748; VI-NEXT:    s_endpgm8749;8750; GFX9-LABEL: atomic_dec_i32_max_neg_offset:8751; GFX9:       ; %bb.0: ; %entry8752; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c8753; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x248754; GFX9-NEXT:    v_mov_b32_e32 v0, 08755; GFX9-NEXT:    s_waitcnt lgkmcnt(0)8756; GFX9-NEXT:    v_mov_b32_e32 v1, s28757; GFX9-NEXT:    global_atomic_dec v0, v1, s[0:1] offset:-40968758; GFX9-NEXT:    s_waitcnt vmcnt(0)8759; GFX9-NEXT:    buffer_wbinvl1_vol8760; GFX9-NEXT:    s_endpgm8761;8762; GFX11-LABEL: atomic_dec_i32_max_neg_offset:8763; GFX11:       ; %bb.0: ; %entry8764; GFX11-NEXT:    s_clause 0x18765; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c8766; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x248767; GFX11-NEXT:    s_waitcnt lgkmcnt(0)8768; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s28769; GFX11-NEXT:    global_atomic_dec_u32 v0, v1, s[0:1] offset:-40968770; GFX11-NEXT:    s_waitcnt_vscnt null, 0x08771; GFX11-NEXT:    buffer_gl1_inv8772; GFX11-NEXT:    buffer_gl0_inv8773; GFX11-NEXT:    s_endpgm8774entry:8775  %gep = getelementptr i32, ptr addrspace(1) %out, i64 -10248776  %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !08777  ret void8778}8779 8780define amdgpu_kernel void @atomic_dec_i32_soffset(ptr addrspace(1) %out, i32 %in) {8781; SI-LABEL: atomic_dec_i32_soffset:8782; SI:       ; %bb.0: ; %entry8783; SI-NEXT:    s_load_dword s6, s[4:5], 0xb8784; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x98785; SI-NEXT:    s_mov_b32 s3, 0xf0008786; SI-NEXT:    s_mov_b32 s2, -18787; SI-NEXT:    s_mov_b32 s4, 0x8ca08788; SI-NEXT:    s_waitcnt lgkmcnt(0)8789; SI-NEXT:    v_mov_b32_e32 v0, s68790; SI-NEXT:    buffer_atomic_dec v0, off, s[0:3], s48791; SI-NEXT:    s_waitcnt vmcnt(0)8792; SI-NEXT:    buffer_wbinvl18793; SI-NEXT:    s_endpgm8794;8795; VI-LABEL: atomic_dec_i32_soffset:8796; VI:       ; %bb.0: ; %entry8797; VI-NEXT:    s_load_dword s6, s[4:5], 0x2c8798; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x248799; VI-NEXT:    s_mov_b32 s3, 0xf0008800; VI-NEXT:    s_mov_b32 s2, -18801; VI-NEXT:    s_mov_b32 s4, 0x8ca08802; VI-NEXT:    s_waitcnt lgkmcnt(0)8803; VI-NEXT:    v_mov_b32_e32 v0, s68804; VI-NEXT:    buffer_atomic_dec v0, off, s[0:3], s48805; VI-NEXT:    s_waitcnt vmcnt(0)8806; VI-NEXT:    buffer_wbinvl1_vol8807; VI-NEXT:    s_endpgm8808;8809; GFX9-LABEL: atomic_dec_i32_soffset:8810; GFX9:       ; %bb.0: ; %entry8811; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c8812; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x248813; GFX9-NEXT:    v_mov_b32_e32 v0, 0x80008814; GFX9-NEXT:    s_waitcnt lgkmcnt(0)8815; GFX9-NEXT:    v_mov_b32_e32 v1, s28816; GFX9-NEXT:    global_atomic_dec v0, v1, s[0:1] offset:32328817; GFX9-NEXT:    s_waitcnt vmcnt(0)8818; GFX9-NEXT:    buffer_wbinvl1_vol8819; GFX9-NEXT:    s_endpgm8820;8821; GFX11-LABEL: atomic_dec_i32_soffset:8822; GFX11:       ; %bb.0: ; %entry8823; GFX11-NEXT:    s_clause 0x18824; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c8825; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x248826; GFX11-NEXT:    s_waitcnt lgkmcnt(0)8827; GFX11-NEXT:    v_dual_mov_b32 v0, 0x8000 :: v_dual_mov_b32 v1, s28828; GFX11-NEXT:    global_atomic_dec_u32 v0, v1, s[0:1] offset:32328829; GFX11-NEXT:    s_waitcnt_vscnt null, 0x08830; GFX11-NEXT:    buffer_gl1_inv8831; GFX11-NEXT:    buffer_gl0_inv8832; GFX11-NEXT:    s_endpgm8833entry:8834  %gep = getelementptr i32, ptr addrspace(1) %out, i64 90008835  %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !08836  ret void8837}8838 8839define amdgpu_kernel void @atomic_dec_i32_huge_offset(ptr addrspace(1) %out, i32 %in) {8840; SI-LABEL: atomic_dec_i32_huge_offset:8841; SI:       ; %bb.0: ; %entry8842; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x98843; SI-NEXT:    s_load_dword s4, s[4:5], 0xb8844; SI-NEXT:    s_mov_b32 s3, 0xf0008845; SI-NEXT:    s_mov_b32 s2, 08846; SI-NEXT:    v_mov_b32_e32 v0, 0xdeac8847; SI-NEXT:    v_mov_b32_e32 v1, 0xabcd8848; SI-NEXT:    s_waitcnt lgkmcnt(0)8849; SI-NEXT:    v_mov_b32_e32 v2, s48850; SI-NEXT:    buffer_atomic_dec v2, v[0:1], s[0:3], 0 addr648851; SI-NEXT:    s_waitcnt vmcnt(0)8852; SI-NEXT:    buffer_wbinvl18853; SI-NEXT:    s_endpgm8854;8855; VI-LABEL: atomic_dec_i32_huge_offset:8856; VI:       ; %bb.0: ; %entry8857; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x248858; VI-NEXT:    s_load_dword s2, s[4:5], 0x2c8859; VI-NEXT:    s_waitcnt lgkmcnt(0)8860; VI-NEXT:    s_add_u32 s0, s0, 0xdeac8861; VI-NEXT:    s_addc_u32 s1, s1, 0xabcd8862; VI-NEXT:    v_mov_b32_e32 v0, s08863; VI-NEXT:    v_mov_b32_e32 v1, s18864; VI-NEXT:    v_mov_b32_e32 v2, s28865; VI-NEXT:    flat_atomic_dec v[0:1], v28866; VI-NEXT:    s_waitcnt vmcnt(0)8867; VI-NEXT:    buffer_wbinvl1_vol8868; VI-NEXT:    s_endpgm8869;8870; GFX9-LABEL: atomic_dec_i32_huge_offset:8871; GFX9:       ; %bb.0: ; %entry8872; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x248873; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c8874; GFX9-NEXT:    v_mov_b32_e32 v0, 08875; GFX9-NEXT:    s_waitcnt lgkmcnt(0)8876; GFX9-NEXT:    s_add_u32 s0, s0, 0xdeac8877; GFX9-NEXT:    s_addc_u32 s1, s1, 0xabcd8878; GFX9-NEXT:    v_mov_b32_e32 v1, s28879; GFX9-NEXT:    global_atomic_dec v0, v1, s[0:1]8880; GFX9-NEXT:    s_waitcnt vmcnt(0)8881; GFX9-NEXT:    buffer_wbinvl1_vol8882; GFX9-NEXT:    s_endpgm8883;8884; GFX11-LABEL: atomic_dec_i32_huge_offset:8885; GFX11:       ; %bb.0: ; %entry8886; GFX11-NEXT:    s_clause 0x18887; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c8888; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x248889; GFX11-NEXT:    s_waitcnt lgkmcnt(0)8890; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s28891; GFX11-NEXT:    s_add_u32 s0, s0, 0xdeac8892; GFX11-NEXT:    s_addc_u32 s1, s1, 0xabcd8893; GFX11-NEXT:    global_atomic_dec_u32 v0, v1, s[0:1]8894; GFX11-NEXT:    s_waitcnt_vscnt null, 0x08895; GFX11-NEXT:    buffer_gl1_inv8896; GFX11-NEXT:    buffer_gl0_inv8897; GFX11-NEXT:    s_endpgm8898entry:8899  %gep = getelementptr i32, ptr addrspace(1) %out, i64 472242391755958900  %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !08901  ret void8902}8903 8904define amdgpu_kernel void @atomic_dec_i32_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) {8905; SI-LABEL: atomic_dec_i32_ret_offset:8906; SI:       ; %bb.0: ; %entry8907; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x98908; SI-NEXT:    s_load_dword s8, s[4:5], 0xd8909; SI-NEXT:    s_mov_b32 s7, 0xf0008910; SI-NEXT:    s_mov_b32 s6, -18911; SI-NEXT:    s_waitcnt lgkmcnt(0)8912; SI-NEXT:    s_mov_b32 s4, s28913; SI-NEXT:    s_mov_b32 s5, s38914; SI-NEXT:    s_mov_b32 s2, s68915; SI-NEXT:    s_mov_b32 s3, s78916; SI-NEXT:    v_mov_b32_e32 v0, s88917; SI-NEXT:    buffer_atomic_dec v0, off, s[0:3], 0 offset:16 glc8918; SI-NEXT:    s_waitcnt vmcnt(0)8919; SI-NEXT:    buffer_wbinvl18920; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 08921; SI-NEXT:    s_endpgm8922;8923; VI-LABEL: atomic_dec_i32_ret_offset:8924; VI:       ; %bb.0: ; %entry8925; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x248926; VI-NEXT:    s_load_dword s8, s[4:5], 0x348927; VI-NEXT:    s_mov_b32 s7, 0xf0008928; VI-NEXT:    s_mov_b32 s6, -18929; VI-NEXT:    s_waitcnt lgkmcnt(0)8930; VI-NEXT:    s_mov_b32 s4, s28931; VI-NEXT:    s_mov_b32 s5, s38932; VI-NEXT:    s_mov_b32 s2, s68933; VI-NEXT:    s_mov_b32 s3, s78934; VI-NEXT:    v_mov_b32_e32 v0, s88935; VI-NEXT:    buffer_atomic_dec v0, off, s[0:3], 0 offset:16 glc8936; VI-NEXT:    s_waitcnt vmcnt(0)8937; VI-NEXT:    buffer_wbinvl1_vol8938; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 08939; VI-NEXT:    s_endpgm8940;8941; GFX9-LABEL: atomic_dec_i32_ret_offset:8942; GFX9:       ; %bb.0: ; %entry8943; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x348944; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x248945; GFX9-NEXT:    v_mov_b32_e32 v0, 08946; GFX9-NEXT:    s_waitcnt lgkmcnt(0)8947; GFX9-NEXT:    v_mov_b32_e32 v1, s68948; GFX9-NEXT:    global_atomic_dec v1, v0, v1, s[0:1] offset:16 glc8949; GFX9-NEXT:    s_waitcnt vmcnt(0)8950; GFX9-NEXT:    buffer_wbinvl1_vol8951; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]8952; GFX9-NEXT:    s_endpgm8953;8954; GFX11-LABEL: atomic_dec_i32_ret_offset:8955; GFX11:       ; %bb.0: ; %entry8956; GFX11-NEXT:    s_clause 0x18957; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x348958; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x248959; GFX11-NEXT:    s_waitcnt lgkmcnt(0)8960; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s68961; GFX11-NEXT:    global_atomic_dec_u32 v1, v0, v1, s[0:1] offset:16 glc8962; GFX11-NEXT:    s_waitcnt vmcnt(0)8963; GFX11-NEXT:    buffer_gl1_inv8964; GFX11-NEXT:    buffer_gl0_inv8965; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]8966; GFX11-NEXT:    s_endpgm8967entry:8968  %gep = getelementptr i32, ptr addrspace(1) %out, i64 48969  %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !08970  store i32 %val, ptr addrspace(1) %out28971  ret void8972}8973 8974define amdgpu_kernel void @atomic_dec_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {8975; SI-LABEL: atomic_dec_i32_addr64_offset:8976; SI:       ; %bb.0: ; %entry8977; SI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd8978; SI-NEXT:    s_load_dword s8, s[4:5], 0xb8979; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x98980; SI-NEXT:    s_mov_b32 s3, 0xf0008981; SI-NEXT:    s_waitcnt lgkmcnt(0)8982; SI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 28983; SI-NEXT:    s_mov_b32 s2, 08984; SI-NEXT:    v_mov_b32_e32 v2, s88985; SI-NEXT:    v_mov_b32_e32 v0, s48986; SI-NEXT:    v_mov_b32_e32 v1, s58987; SI-NEXT:    buffer_atomic_dec v2, v[0:1], s[0:3], 0 addr64 offset:168988; SI-NEXT:    s_waitcnt vmcnt(0)8989; SI-NEXT:    buffer_wbinvl18990; SI-NEXT:    s_endpgm8991;8992; VI-LABEL: atomic_dec_i32_addr64_offset:8993; VI:       ; %bb.0: ; %entry8994; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x348995; VI-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x248996; VI-NEXT:    s_load_dword s4, s[4:5], 0x2c8997; VI-NEXT:    s_waitcnt lgkmcnt(0)8998; VI-NEXT:    s_lshl_b64 s[0:1], s[0:1], 28999; VI-NEXT:    s_add_u32 s0, s2, s09000; VI-NEXT:    s_addc_u32 s1, s3, s19001; VI-NEXT:    s_add_u32 s0, s0, 169002; VI-NEXT:    s_addc_u32 s1, s1, 09003; VI-NEXT:    v_mov_b32_e32 v0, s09004; VI-NEXT:    v_mov_b32_e32 v1, s19005; VI-NEXT:    v_mov_b32_e32 v2, s49006; VI-NEXT:    flat_atomic_dec v[0:1], v29007; VI-NEXT:    s_waitcnt vmcnt(0)9008; VI-NEXT:    buffer_wbinvl1_vol9009; VI-NEXT:    s_endpgm9010;9011; GFX9-LABEL: atomic_dec_i32_addr64_offset:9012; GFX9:       ; %bb.0: ; %entry9013; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x349014; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x249015; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c9016; GFX9-NEXT:    v_mov_b32_e32 v0, 09017; GFX9-NEXT:    s_waitcnt lgkmcnt(0)9018; GFX9-NEXT:    s_lshl_b64 s[0:1], s[0:1], 29019; GFX9-NEXT:    s_add_u32 s0, s2, s09020; GFX9-NEXT:    s_addc_u32 s1, s3, s19021; GFX9-NEXT:    v_mov_b32_e32 v1, s69022; GFX9-NEXT:    global_atomic_dec v0, v1, s[0:1] offset:169023; GFX9-NEXT:    s_waitcnt vmcnt(0)9024; GFX9-NEXT:    buffer_wbinvl1_vol9025; GFX9-NEXT:    s_endpgm9026;9027; GFX11-LABEL: atomic_dec_i32_addr64_offset:9028; GFX11:       ; %bb.0: ; %entry9029; GFX11-NEXT:    s_clause 0x29030; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x349031; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x2c9032; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x249033; GFX11-NEXT:    v_mov_b32_e32 v0, 09034; GFX11-NEXT:    s_waitcnt lgkmcnt(0)9035; GFX11-NEXT:    s_lshl_b64 s[0:1], s[0:1], 29036; GFX11-NEXT:    v_mov_b32_e32 v1, s69037; GFX11-NEXT:    s_add_u32 s0, s2, s09038; GFX11-NEXT:    s_addc_u32 s1, s3, s19039; GFX11-NEXT:    global_atomic_dec_u32 v0, v1, s[0:1] offset:169040; GFX11-NEXT:    s_waitcnt_vscnt null, 0x09041; GFX11-NEXT:    buffer_gl1_inv9042; GFX11-NEXT:    buffer_gl0_inv9043; GFX11-NEXT:    s_endpgm9044entry:9045  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index9046  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 49047  %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !09048  ret void9049}9050 9051define amdgpu_kernel void @atomic_dec_i32_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) {9052; SI-LABEL: atomic_dec_i32_ret_addr64_offset:9053; SI:       ; %bb.0: ; %entry9054; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x99055; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf9056; SI-NEXT:    s_load_dword s6, s[4:5], 0xd9057; SI-NEXT:    s_mov_b32 s7, 0xf0009058; SI-NEXT:    s_waitcnt lgkmcnt(0)9059; SI-NEXT:    s_mov_b32 s4, s29060; SI-NEXT:    s_mov_b32 s5, s39061; SI-NEXT:    s_lshl_b64 s[8:9], s[8:9], 29062; SI-NEXT:    s_mov_b32 s2, 09063; SI-NEXT:    s_mov_b32 s3, s79064; SI-NEXT:    v_mov_b32_e32 v2, s69065; SI-NEXT:    v_mov_b32_e32 v0, s89066; SI-NEXT:    v_mov_b32_e32 v1, s99067; SI-NEXT:    buffer_atomic_dec v2, v[0:1], s[0:3], 0 addr64 offset:16 glc9068; SI-NEXT:    s_waitcnt vmcnt(0)9069; SI-NEXT:    buffer_wbinvl19070; SI-NEXT:    s_mov_b32 s6, -19071; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 09072; SI-NEXT:    s_endpgm9073;9074; VI-LABEL: atomic_dec_i32_ret_addr64_offset:9075; VI:       ; %bb.0: ; %entry9076; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c9077; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x249078; VI-NEXT:    s_load_dword s8, s[4:5], 0x349079; VI-NEXT:    s_waitcnt lgkmcnt(0)9080; VI-NEXT:    s_lshl_b64 s[4:5], s[6:7], 29081; VI-NEXT:    s_add_u32 s0, s0, s49082; VI-NEXT:    s_addc_u32 s1, s1, s59083; VI-NEXT:    s_add_u32 s0, s0, 169084; VI-NEXT:    s_addc_u32 s1, s1, 09085; VI-NEXT:    v_mov_b32_e32 v0, s09086; VI-NEXT:    v_mov_b32_e32 v1, s19087; VI-NEXT:    v_mov_b32_e32 v2, s89088; VI-NEXT:    flat_atomic_dec v0, v[0:1], v2 glc9089; VI-NEXT:    s_waitcnt vmcnt(0)9090; VI-NEXT:    buffer_wbinvl1_vol9091; VI-NEXT:    s_mov_b32 s7, 0xf0009092; VI-NEXT:    s_mov_b32 s6, -19093; VI-NEXT:    s_mov_b32 s4, s29094; VI-NEXT:    s_mov_b32 s5, s39095; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 09096; VI-NEXT:    s_endpgm9097;9098; GFX9-LABEL: atomic_dec_i32_ret_addr64_offset:9099; GFX9:       ; %bb.0: ; %entry9100; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c9101; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x249102; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x349103; GFX9-NEXT:    v_mov_b32_e32 v0, 09104; GFX9-NEXT:    s_waitcnt lgkmcnt(0)9105; GFX9-NEXT:    s_lshl_b64 s[4:5], s[6:7], 29106; GFX9-NEXT:    s_add_u32 s0, s0, s49107; GFX9-NEXT:    s_addc_u32 s1, s1, s59108; GFX9-NEXT:    v_mov_b32_e32 v1, s89109; GFX9-NEXT:    global_atomic_dec v1, v0, v1, s[0:1] offset:16 glc9110; GFX9-NEXT:    s_waitcnt vmcnt(0)9111; GFX9-NEXT:    buffer_wbinvl1_vol9112; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]9113; GFX9-NEXT:    s_endpgm9114;9115; GFX11-LABEL: atomic_dec_i32_ret_addr64_offset:9116; GFX11:       ; %bb.0: ; %entry9117; GFX11-NEXT:    s_clause 0x29118; GFX11-NEXT:    s_load_b64 s[6:7], s[4:5], 0x3c9119; GFX11-NEXT:    s_load_b32 s8, s[4:5], 0x349120; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x249121; GFX11-NEXT:    v_mov_b32_e32 v0, 09122; GFX11-NEXT:    s_waitcnt lgkmcnt(0)9123; GFX11-NEXT:    s_lshl_b64 s[4:5], s[6:7], 29124; GFX11-NEXT:    v_mov_b32_e32 v1, s89125; GFX11-NEXT:    s_add_u32 s0, s0, s49126; GFX11-NEXT:    s_addc_u32 s1, s1, s59127; GFX11-NEXT:    global_atomic_dec_u32 v1, v0, v1, s[0:1] offset:16 glc9128; GFX11-NEXT:    s_waitcnt vmcnt(0)9129; GFX11-NEXT:    buffer_gl1_inv9130; GFX11-NEXT:    buffer_gl0_inv9131; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]9132; GFX11-NEXT:    s_endpgm9133entry:9134  %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index9135  %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 49136  %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !09137  store i32 %val, ptr addrspace(1) %out29138  ret void9139}9140 9141define amdgpu_kernel void @atomic_load_f16_offset(ptr addrspace(1) %in, ptr addrspace(1) %out) {9142; SI-LABEL: atomic_load_f16_offset:9143; SI:       ; %bb.0:9144; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x99145; SI-NEXT:    s_mov_b32 s7, 0xf0009146; SI-NEXT:    s_mov_b32 s6, -19147; SI-NEXT:    s_waitcnt lgkmcnt(0)9148; SI-NEXT:    s_mov_b32 s4, s29149; SI-NEXT:    s_mov_b32 s5, s39150; SI-NEXT:    s_mov_b32 s2, s69151; SI-NEXT:    s_mov_b32 s3, s79152; SI-NEXT:    buffer_load_ushort v0, off, s[0:3], 0 offset:16 glc9153; SI-NEXT:    s_waitcnt vmcnt(0)9154; SI-NEXT:    buffer_wbinvl19155; SI-NEXT:    buffer_store_short v0, off, s[4:7], 09156; SI-NEXT:    s_endpgm9157;9158; VI-LABEL: atomic_load_f16_offset:9159; VI:       ; %bb.0:9160; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x249161; VI-NEXT:    s_mov_b32 s7, 0xf0009162; VI-NEXT:    s_mov_b32 s6, -19163; VI-NEXT:    s_waitcnt lgkmcnt(0)9164; VI-NEXT:    s_mov_b32 s4, s29165; VI-NEXT:    s_mov_b32 s5, s39166; VI-NEXT:    s_mov_b32 s2, s69167; VI-NEXT:    s_mov_b32 s3, s79168; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], 0 offset:16 glc9169; VI-NEXT:    s_waitcnt vmcnt(0)9170; VI-NEXT:    buffer_wbinvl1_vol9171; VI-NEXT:    buffer_store_short v0, off, s[4:7], 09172; VI-NEXT:    s_endpgm9173;9174; GFX9-LABEL: atomic_load_f16_offset:9175; GFX9:       ; %bb.0:9176; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x249177; GFX9-NEXT:    v_mov_b32_e32 v0, 09178; GFX9-NEXT:    s_waitcnt lgkmcnt(0)9179; GFX9-NEXT:    global_load_ushort v1, v0, s[0:1] offset:16 glc9180; GFX9-NEXT:    s_waitcnt vmcnt(0)9181; GFX9-NEXT:    buffer_wbinvl1_vol9182; GFX9-NEXT:    global_store_short v0, v1, s[2:3]9183; GFX9-NEXT:    s_endpgm9184;9185; GFX11-TRUE16-LABEL: atomic_load_f16_offset:9186; GFX11-TRUE16:       ; %bb.0:9187; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x249188; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 09189; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)9190; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1] offset:16 glc9191; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)9192; GFX11-TRUE16-NEXT:    buffer_gl1_inv9193; GFX11-TRUE16-NEXT:    buffer_gl0_inv9194; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[2:3]9195; GFX11-TRUE16-NEXT:    s_endpgm9196;9197; GFX11-FAKE16-LABEL: atomic_load_f16_offset:9198; GFX11-FAKE16:       ; %bb.0:9199; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x249200; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 09201; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)9202; GFX11-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1] offset:16 glc9203; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)9204; GFX11-FAKE16-NEXT:    buffer_gl1_inv9205; GFX11-FAKE16-NEXT:    buffer_gl0_inv9206; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[2:3]9207; GFX11-FAKE16-NEXT:    s_endpgm9208  %gep = getelementptr half, ptr addrspace(1) %in, i64 89209  %val = load atomic half, ptr addrspace(1) %gep  seq_cst, align 29210  store half %val, ptr addrspace(1) %out9211  ret void9212}9213 9214define amdgpu_kernel void @atomic_load_f16_negoffset(ptr addrspace(1) %in, ptr addrspace(1) %out) {9215; SI-LABEL: atomic_load_f16_negoffset:9216; SI:       ; %bb.0:9217; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x99218; SI-NEXT:    s_mov_b32 s7, 0xf0009219; SI-NEXT:    s_waitcnt lgkmcnt(0)9220; SI-NEXT:    s_mov_b32 s4, s29221; SI-NEXT:    s_mov_b32 s5, s39222; SI-NEXT:    s_mov_b32 s2, 09223; SI-NEXT:    s_mov_b32 s3, s79224; SI-NEXT:    v_mov_b32_e32 v0, 0xfffffe009225; SI-NEXT:    v_mov_b32_e32 v1, -19226; SI-NEXT:    buffer_load_ushort v0, v[0:1], s[0:3], 0 addr64 glc9227; SI-NEXT:    s_waitcnt vmcnt(0)9228; SI-NEXT:    buffer_wbinvl19229; SI-NEXT:    s_mov_b32 s6, -19230; SI-NEXT:    buffer_store_short v0, off, s[4:7], 09231; SI-NEXT:    s_endpgm9232;9233; VI-LABEL: atomic_load_f16_negoffset:9234; VI:       ; %bb.0:9235; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x249236; VI-NEXT:    s_mov_b32 s7, 0xf0009237; VI-NEXT:    s_mov_b32 s6, -19238; VI-NEXT:    s_waitcnt lgkmcnt(0)9239; VI-NEXT:    s_add_u32 s0, s0, 0xfffffe009240; VI-NEXT:    s_addc_u32 s1, s1, -19241; VI-NEXT:    v_mov_b32_e32 v0, s09242; VI-NEXT:    v_mov_b32_e32 v1, s19243; VI-NEXT:    flat_load_ushort v0, v[0:1] glc9244; VI-NEXT:    s_waitcnt vmcnt(0)9245; VI-NEXT:    buffer_wbinvl1_vol9246; VI-NEXT:    s_mov_b32 s4, s29247; VI-NEXT:    s_mov_b32 s5, s39248; VI-NEXT:    buffer_store_short v0, off, s[4:7], 09249; VI-NEXT:    s_endpgm9250;9251; GFX9-LABEL: atomic_load_f16_negoffset:9252; GFX9:       ; %bb.0:9253; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x249254; GFX9-NEXT:    v_mov_b32_e32 v0, 09255; GFX9-NEXT:    s_waitcnt lgkmcnt(0)9256; GFX9-NEXT:    global_load_ushort v1, v0, s[0:1] offset:-512 glc9257; GFX9-NEXT:    s_waitcnt vmcnt(0)9258; GFX9-NEXT:    buffer_wbinvl1_vol9259; GFX9-NEXT:    global_store_short v0, v1, s[2:3]9260; GFX9-NEXT:    s_endpgm9261;9262; GFX11-TRUE16-LABEL: atomic_load_f16_negoffset:9263; GFX11-TRUE16:       ; %bb.0:9264; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x249265; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 09266; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)9267; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1] offset:-512 glc9268; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)9269; GFX11-TRUE16-NEXT:    buffer_gl1_inv9270; GFX11-TRUE16-NEXT:    buffer_gl0_inv9271; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[2:3]9272; GFX11-TRUE16-NEXT:    s_endpgm9273;9274; GFX11-FAKE16-LABEL: atomic_load_f16_negoffset:9275; GFX11-FAKE16:       ; %bb.0:9276; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x249277; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 09278; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)9279; GFX11-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1] offset:-512 glc9280; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)9281; GFX11-FAKE16-NEXT:    buffer_gl1_inv9282; GFX11-FAKE16-NEXT:    buffer_gl0_inv9283; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[2:3]9284; GFX11-FAKE16-NEXT:    s_endpgm9285  %gep = getelementptr half, ptr addrspace(1) %in, i64 -2569286  %val = load atomic half, ptr addrspace(1) %gep  seq_cst, align 29287  store half %val, ptr addrspace(1) %out9288  ret void9289}9290 9291define amdgpu_kernel void @atomic_load_bf16_offset(ptr addrspace(1) %in, ptr addrspace(1) %out) {9292; SI-LABEL: atomic_load_bf16_offset:9293; SI:       ; %bb.0:9294; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x99295; SI-NEXT:    s_mov_b32 s7, 0xf0009296; SI-NEXT:    s_mov_b32 s6, -19297; SI-NEXT:    s_waitcnt lgkmcnt(0)9298; SI-NEXT:    s_mov_b32 s4, s29299; SI-NEXT:    s_mov_b32 s5, s39300; SI-NEXT:    s_mov_b32 s2, s69301; SI-NEXT:    s_mov_b32 s3, s79302; SI-NEXT:    buffer_load_ushort v0, off, s[0:3], 0 offset:16 glc9303; SI-NEXT:    s_waitcnt vmcnt(0)9304; SI-NEXT:    buffer_wbinvl19305; SI-NEXT:    buffer_store_short v0, off, s[4:7], 09306; SI-NEXT:    s_endpgm9307;9308; VI-LABEL: atomic_load_bf16_offset:9309; VI:       ; %bb.0:9310; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x249311; VI-NEXT:    s_mov_b32 s7, 0xf0009312; VI-NEXT:    s_mov_b32 s6, -19313; VI-NEXT:    s_waitcnt lgkmcnt(0)9314; VI-NEXT:    s_mov_b32 s4, s29315; VI-NEXT:    s_mov_b32 s5, s39316; VI-NEXT:    s_mov_b32 s2, s69317; VI-NEXT:    s_mov_b32 s3, s79318; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], 0 offset:16 glc9319; VI-NEXT:    s_waitcnt vmcnt(0)9320; VI-NEXT:    buffer_wbinvl1_vol9321; VI-NEXT:    buffer_store_short v0, off, s[4:7], 09322; VI-NEXT:    s_endpgm9323;9324; GFX9-LABEL: atomic_load_bf16_offset:9325; GFX9:       ; %bb.0:9326; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x249327; GFX9-NEXT:    v_mov_b32_e32 v0, 09328; GFX9-NEXT:    s_waitcnt lgkmcnt(0)9329; GFX9-NEXT:    global_load_ushort v1, v0, s[0:1] offset:16 glc9330; GFX9-NEXT:    s_waitcnt vmcnt(0)9331; GFX9-NEXT:    buffer_wbinvl1_vol9332; GFX9-NEXT:    global_store_short v0, v1, s[2:3]9333; GFX9-NEXT:    s_endpgm9334;9335; GFX11-TRUE16-LABEL: atomic_load_bf16_offset:9336; GFX11-TRUE16:       ; %bb.0:9337; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x249338; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 09339; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)9340; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1] offset:16 glc9341; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)9342; GFX11-TRUE16-NEXT:    buffer_gl1_inv9343; GFX11-TRUE16-NEXT:    buffer_gl0_inv9344; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[2:3]9345; GFX11-TRUE16-NEXT:    s_endpgm9346;9347; GFX11-FAKE16-LABEL: atomic_load_bf16_offset:9348; GFX11-FAKE16:       ; %bb.0:9349; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x249350; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 09351; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)9352; GFX11-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1] offset:16 glc9353; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)9354; GFX11-FAKE16-NEXT:    buffer_gl1_inv9355; GFX11-FAKE16-NEXT:    buffer_gl0_inv9356; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[2:3]9357; GFX11-FAKE16-NEXT:    s_endpgm9358  %gep = getelementptr bfloat, ptr addrspace(1) %in, i64 89359  %val = load atomic bfloat, ptr addrspace(1) %gep  seq_cst, align 29360  store bfloat %val, ptr addrspace(1) %out9361  ret void9362}9363 9364define amdgpu_kernel void @atomic_load_bf16_negoffset(ptr addrspace(1) %in, ptr addrspace(1) %out) {9365; SI-LABEL: atomic_load_bf16_negoffset:9366; SI:       ; %bb.0:9367; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x99368; SI-NEXT:    s_mov_b32 s7, 0xf0009369; SI-NEXT:    s_waitcnt lgkmcnt(0)9370; SI-NEXT:    s_mov_b32 s4, s29371; SI-NEXT:    s_mov_b32 s5, s39372; SI-NEXT:    s_mov_b32 s2, 09373; SI-NEXT:    s_mov_b32 s3, s79374; SI-NEXT:    v_mov_b32_e32 v0, 0xfffffe009375; SI-NEXT:    v_mov_b32_e32 v1, -19376; SI-NEXT:    buffer_load_ushort v0, v[0:1], s[0:3], 0 addr64 glc9377; SI-NEXT:    s_waitcnt vmcnt(0)9378; SI-NEXT:    buffer_wbinvl19379; SI-NEXT:    s_mov_b32 s6, -19380; SI-NEXT:    buffer_store_short v0, off, s[4:7], 09381; SI-NEXT:    s_endpgm9382;9383; VI-LABEL: atomic_load_bf16_negoffset:9384; VI:       ; %bb.0:9385; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x249386; VI-NEXT:    s_mov_b32 s7, 0xf0009387; VI-NEXT:    s_mov_b32 s6, -19388; VI-NEXT:    s_waitcnt lgkmcnt(0)9389; VI-NEXT:    s_add_u32 s0, s0, 0xfffffe009390; VI-NEXT:    s_addc_u32 s1, s1, -19391; VI-NEXT:    v_mov_b32_e32 v0, s09392; VI-NEXT:    v_mov_b32_e32 v1, s19393; VI-NEXT:    flat_load_ushort v0, v[0:1] glc9394; VI-NEXT:    s_waitcnt vmcnt(0)9395; VI-NEXT:    buffer_wbinvl1_vol9396; VI-NEXT:    s_mov_b32 s4, s29397; VI-NEXT:    s_mov_b32 s5, s39398; VI-NEXT:    buffer_store_short v0, off, s[4:7], 09399; VI-NEXT:    s_endpgm9400;9401; GFX9-LABEL: atomic_load_bf16_negoffset:9402; GFX9:       ; %bb.0:9403; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x249404; GFX9-NEXT:    v_mov_b32_e32 v0, 09405; GFX9-NEXT:    s_waitcnt lgkmcnt(0)9406; GFX9-NEXT:    global_load_ushort v1, v0, s[0:1] offset:-512 glc9407; GFX9-NEXT:    s_waitcnt vmcnt(0)9408; GFX9-NEXT:    buffer_wbinvl1_vol9409; GFX9-NEXT:    global_store_short v0, v1, s[2:3]9410; GFX9-NEXT:    s_endpgm9411;9412; GFX11-TRUE16-LABEL: atomic_load_bf16_negoffset:9413; GFX11-TRUE16:       ; %bb.0:9414; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x249415; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 09416; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)9417; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1] offset:-512 glc9418; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)9419; GFX11-TRUE16-NEXT:    buffer_gl1_inv9420; GFX11-TRUE16-NEXT:    buffer_gl0_inv9421; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[2:3]9422; GFX11-TRUE16-NEXT:    s_endpgm9423;9424; GFX11-FAKE16-LABEL: atomic_load_bf16_negoffset:9425; GFX11-FAKE16:       ; %bb.0:9426; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x249427; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 09428; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)9429; GFX11-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1] offset:-512 glc9430; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)9431; GFX11-FAKE16-NEXT:    buffer_gl1_inv9432; GFX11-FAKE16-NEXT:    buffer_gl0_inv9433; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[2:3]9434; GFX11-FAKE16-NEXT:    s_endpgm9435  %gep = getelementptr bfloat, ptr addrspace(1) %in, i64 -2569436  %val = load atomic bfloat, ptr addrspace(1) %gep  seq_cst, align 29437  store bfloat %val, ptr addrspace(1) %out9438  ret void9439}9440 9441define amdgpu_kernel void @atomic_sub_i16_soffset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i16 %in) {9442; SI-LABEL: atomic_sub_i16_soffset__amdgpu_no_remote_memory:9443; SI:       ; %bb.0:9444; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x99445; SI-NEXT:    s_load_dword s2, s[4:5], 0xb9446; SI-NEXT:    s_mov_b64 s[4:5], 09447; SI-NEXT:    s_mov_b32 s3, 0xf0009448; SI-NEXT:    s_waitcnt lgkmcnt(0)9449; SI-NEXT:    s_add_u32 s6, s0, 0x46509450; SI-NEXT:    s_addc_u32 s1, s1, 09451; SI-NEXT:    s_and_b32 s0, s6, -49452; SI-NEXT:    s_and_b32 s6, s6, 39453; SI-NEXT:    s_and_b32 s2, s2, 0xffff9454; SI-NEXT:    s_load_dword s9, s[0:1], 0x09455; SI-NEXT:    s_lshl_b32 s7, s6, 39456; SI-NEXT:    s_lshl_b32 s6, 0xffff, s79457; SI-NEXT:    s_lshl_b32 s7, s2, s79458; SI-NEXT:    s_not_b32 s8, s69459; SI-NEXT:    s_waitcnt lgkmcnt(0)9460; SI-NEXT:    v_mov_b32_e32 v1, s99461; SI-NEXT:    s_mov_b32 s2, -19462; SI-NEXT:  .LBB136_1: ; %atomicrmw.start9463; SI-NEXT:    ; =>This Inner Loop Header: Depth=19464; SI-NEXT:    v_subrev_i32_e32 v0, vcc, s7, v19465; SI-NEXT:    v_and_b32_e32 v0, s6, v09466; SI-NEXT:    s_waitcnt expcnt(0)9467; SI-NEXT:    v_and_b32_e32 v2, s8, v19468; SI-NEXT:    v_or_b32_e32 v0, v2, v09469; SI-NEXT:    v_mov_b32_e32 v3, v19470; SI-NEXT:    v_mov_b32_e32 v2, v09471; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc9472; SI-NEXT:    s_waitcnt vmcnt(0)9473; SI-NEXT:    buffer_wbinvl19474; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v19475; SI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]9476; SI-NEXT:    v_mov_b32_e32 v1, v29477; SI-NEXT:    s_andn2_b64 exec, exec, s[4:5]9478; SI-NEXT:    s_cbranch_execnz .LBB136_19479; SI-NEXT:  ; %bb.2: ; %atomicrmw.end9480; SI-NEXT:    s_endpgm9481;9482; VI-LABEL: atomic_sub_i16_soffset__amdgpu_no_remote_memory:9483; VI:       ; %bb.0:9484; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x249485; VI-NEXT:    s_load_dword s2, s[4:5], 0x2c9486; VI-NEXT:    s_mov_b64 s[4:5], 09487; VI-NEXT:    s_waitcnt lgkmcnt(0)9488; VI-NEXT:    s_add_u32 s3, s0, 0x46509489; VI-NEXT:    s_addc_u32 s1, s1, 09490; VI-NEXT:    s_and_b32 s0, s3, -49491; VI-NEXT:    s_load_dword s9, s[0:1], 0x09492; VI-NEXT:    s_and_b32 s3, s3, 39493; VI-NEXT:    s_lshl_b32 s3, s3, 39494; VI-NEXT:    s_lshl_b32 s6, 0xffff, s39495; VI-NEXT:    s_and_b32 s2, s2, 0xffff9496; VI-NEXT:    s_not_b32 s7, s69497; VI-NEXT:    s_lshl_b32 s8, s2, s39498; VI-NEXT:    s_waitcnt lgkmcnt(0)9499; VI-NEXT:    v_mov_b32_e32 v1, s99500; VI-NEXT:    s_mov_b32 s3, 0xf0009501; VI-NEXT:    s_mov_b32 s2, -19502; VI-NEXT:  .LBB136_1: ; %atomicrmw.start9503; VI-NEXT:    ; =>This Inner Loop Header: Depth=19504; VI-NEXT:    v_subrev_u32_e32 v0, vcc, s8, v19505; VI-NEXT:    v_and_b32_e32 v2, s7, v19506; VI-NEXT:    v_and_b32_e32 v0, s6, v09507; VI-NEXT:    v_or_b32_e32 v0, v2, v09508; VI-NEXT:    v_mov_b32_e32 v3, v19509; VI-NEXT:    v_mov_b32_e32 v2, v09510; VI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc9511; VI-NEXT:    s_waitcnt vmcnt(0)9512; VI-NEXT:    buffer_wbinvl1_vol9513; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v19514; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]9515; VI-NEXT:    v_mov_b32_e32 v1, v29516; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]9517; VI-NEXT:    s_cbranch_execnz .LBB136_19518; VI-NEXT:  ; %bb.2: ; %atomicrmw.end9519; VI-NEXT:    s_endpgm9520;9521; GFX9-LABEL: atomic_sub_i16_soffset__amdgpu_no_remote_memory:9522; GFX9:       ; %bb.0:9523; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x249524; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c9525; GFX9-NEXT:    v_mov_b32_e32 v2, 09526; GFX9-NEXT:    s_waitcnt lgkmcnt(0)9527; GFX9-NEXT:    s_add_u32 s3, s0, 0x46509528; GFX9-NEXT:    s_addc_u32 s1, s1, 09529; GFX9-NEXT:    s_and_b32 s0, s3, -49530; GFX9-NEXT:    s_load_dword s7, s[0:1], 0x09531; GFX9-NEXT:    s_and_b32 s3, s3, 39532; GFX9-NEXT:    s_lshl_b32 s3, s3, 39533; GFX9-NEXT:    s_lshl_b32 s4, 0xffff, s39534; GFX9-NEXT:    s_and_b32 s2, s2, 0xffff9535; GFX9-NEXT:    s_not_b32 s5, s49536; GFX9-NEXT:    s_lshl_b32 s6, s2, s39537; GFX9-NEXT:    s_mov_b64 s[2:3], 09538; GFX9-NEXT:    s_waitcnt lgkmcnt(0)9539; GFX9-NEXT:    v_mov_b32_e32 v1, s79540; GFX9-NEXT:  .LBB136_1: ; %atomicrmw.start9541; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=19542; GFX9-NEXT:    v_subrev_u32_e32 v0, s6, v19543; GFX9-NEXT:    v_and_b32_e32 v0, s4, v09544; GFX9-NEXT:    v_and_or_b32 v0, v1, s5, v09545; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc9546; GFX9-NEXT:    s_waitcnt vmcnt(0)9547; GFX9-NEXT:    buffer_wbinvl1_vol9548; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v19549; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]9550; GFX9-NEXT:    v_mov_b32_e32 v1, v09551; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]9552; GFX9-NEXT:    s_cbranch_execnz .LBB136_19553; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end9554; GFX9-NEXT:    s_endpgm9555  %gep = getelementptr i16, ptr addrspace(1) %out, i64 90009556  %val = atomicrmw sub ptr addrspace(1) %gep, i16 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !09557  ret void9558}9559 9560define amdgpu_kernel void @atomic_sub_i8_soffset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i8 %in) {9561; SI-LABEL: atomic_sub_i8_soffset__amdgpu_no_remote_memory:9562; SI:       ; %bb.0:9563; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x99564; SI-NEXT:    s_load_dword s2, s[4:5], 0xb9565; SI-NEXT:    s_mov_b64 s[4:5], 09566; SI-NEXT:    s_mov_b32 s3, 0xf0009567; SI-NEXT:    s_waitcnt lgkmcnt(0)9568; SI-NEXT:    s_add_u32 s6, s0, 0x23289569; SI-NEXT:    s_addc_u32 s1, s1, 09570; SI-NEXT:    s_and_b32 s0, s6, -49571; SI-NEXT:    s_and_b32 s6, s6, 39572; SI-NEXT:    s_and_b32 s2, s2, 0xff9573; SI-NEXT:    s_load_dword s9, s[0:1], 0x09574; SI-NEXT:    s_lshl_b32 s7, s6, 39575; SI-NEXT:    s_lshl_b32 s6, 0xff, s79576; SI-NEXT:    s_lshl_b32 s7, s2, s79577; SI-NEXT:    s_not_b32 s8, s69578; SI-NEXT:    s_waitcnt lgkmcnt(0)9579; SI-NEXT:    v_mov_b32_e32 v1, s99580; SI-NEXT:    s_mov_b32 s2, -19581; SI-NEXT:  .LBB137_1: ; %atomicrmw.start9582; SI-NEXT:    ; =>This Inner Loop Header: Depth=19583; SI-NEXT:    v_subrev_i32_e32 v0, vcc, s7, v19584; SI-NEXT:    v_and_b32_e32 v0, s6, v09585; SI-NEXT:    s_waitcnt expcnt(0)9586; SI-NEXT:    v_and_b32_e32 v2, s8, v19587; SI-NEXT:    v_or_b32_e32 v0, v2, v09588; SI-NEXT:    v_mov_b32_e32 v3, v19589; SI-NEXT:    v_mov_b32_e32 v2, v09590; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc9591; SI-NEXT:    s_waitcnt vmcnt(0)9592; SI-NEXT:    buffer_wbinvl19593; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v19594; SI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]9595; SI-NEXT:    v_mov_b32_e32 v1, v29596; SI-NEXT:    s_andn2_b64 exec, exec, s[4:5]9597; SI-NEXT:    s_cbranch_execnz .LBB137_19598; SI-NEXT:  ; %bb.2: ; %atomicrmw.end9599; SI-NEXT:    s_endpgm9600;9601; VI-LABEL: atomic_sub_i8_soffset__amdgpu_no_remote_memory:9602; VI:       ; %bb.0:9603; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x249604; VI-NEXT:    s_load_dword s2, s[4:5], 0x2c9605; VI-NEXT:    s_mov_b64 s[4:5], 09606; VI-NEXT:    s_waitcnt lgkmcnt(0)9607; VI-NEXT:    s_add_u32 s3, s0, 0x23289608; VI-NEXT:    s_addc_u32 s1, s1, 09609; VI-NEXT:    s_and_b32 s0, s3, -49610; VI-NEXT:    s_load_dword s9, s[0:1], 0x09611; VI-NEXT:    s_and_b32 s3, s3, 39612; VI-NEXT:    s_lshl_b32 s3, s3, 39613; VI-NEXT:    s_lshl_b32 s6, 0xff, s39614; VI-NEXT:    s_and_b32 s2, s2, 0xff9615; VI-NEXT:    s_not_b32 s7, s69616; VI-NEXT:    s_lshl_b32 s8, s2, s39617; VI-NEXT:    s_waitcnt lgkmcnt(0)9618; VI-NEXT:    v_mov_b32_e32 v1, s99619; VI-NEXT:    s_mov_b32 s3, 0xf0009620; VI-NEXT:    s_mov_b32 s2, -19621; VI-NEXT:  .LBB137_1: ; %atomicrmw.start9622; VI-NEXT:    ; =>This Inner Loop Header: Depth=19623; VI-NEXT:    v_subrev_u32_e32 v0, vcc, s8, v19624; VI-NEXT:    v_and_b32_e32 v2, s7, v19625; VI-NEXT:    v_and_b32_e32 v0, s6, v09626; VI-NEXT:    v_or_b32_e32 v0, v2, v09627; VI-NEXT:    v_mov_b32_e32 v3, v19628; VI-NEXT:    v_mov_b32_e32 v2, v09629; VI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc9630; VI-NEXT:    s_waitcnt vmcnt(0)9631; VI-NEXT:    buffer_wbinvl1_vol9632; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v19633; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]9634; VI-NEXT:    v_mov_b32_e32 v1, v29635; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]9636; VI-NEXT:    s_cbranch_execnz .LBB137_19637; VI-NEXT:  ; %bb.2: ; %atomicrmw.end9638; VI-NEXT:    s_endpgm9639;9640; GFX9-LABEL: atomic_sub_i8_soffset__amdgpu_no_remote_memory:9641; GFX9:       ; %bb.0:9642; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x249643; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c9644; GFX9-NEXT:    v_mov_b32_e32 v2, 09645; GFX9-NEXT:    s_waitcnt lgkmcnt(0)9646; GFX9-NEXT:    s_add_u32 s3, s0, 0x23289647; GFX9-NEXT:    s_addc_u32 s1, s1, 09648; GFX9-NEXT:    s_and_b32 s0, s3, -49649; GFX9-NEXT:    s_load_dword s7, s[0:1], 0x09650; GFX9-NEXT:    s_and_b32 s3, s3, 39651; GFX9-NEXT:    s_lshl_b32 s3, s3, 39652; GFX9-NEXT:    s_lshl_b32 s4, 0xff, s39653; GFX9-NEXT:    s_and_b32 s2, s2, 0xff9654; GFX9-NEXT:    s_not_b32 s5, s49655; GFX9-NEXT:    s_lshl_b32 s6, s2, s39656; GFX9-NEXT:    s_mov_b64 s[2:3], 09657; GFX9-NEXT:    s_waitcnt lgkmcnt(0)9658; GFX9-NEXT:    v_mov_b32_e32 v1, s79659; GFX9-NEXT:  .LBB137_1: ; %atomicrmw.start9660; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=19661; GFX9-NEXT:    v_subrev_u32_e32 v0, s6, v19662; GFX9-NEXT:    v_and_b32_e32 v0, s4, v09663; GFX9-NEXT:    v_and_or_b32 v0, v1, s5, v09664; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc9665; GFX9-NEXT:    s_waitcnt vmcnt(0)9666; GFX9-NEXT:    buffer_wbinvl1_vol9667; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v19668; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]9669; GFX9-NEXT:    v_mov_b32_e32 v1, v09670; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]9671; GFX9-NEXT:    s_cbranch_execnz .LBB137_19672; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end9673; GFX9-NEXT:    s_endpgm9674  %gep = getelementptr i8, ptr addrspace(1) %out, i64 90009675  %val = atomicrmw sub ptr addrspace(1) %gep, i8 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !09676  ret void9677}9678 9679!0 = !{}9680