brintos / llvm-project-archived public Read only

1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -enable-var-scope -check-prefixes=SI %s3; RUN: llc -mtriple=amdgcn -mcpu=tonga -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -enable-var-scope -check-prefixes=VI %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s5 6; ---------------------------------------------------------------------7; atomicrmw xchg8; ---------------------------------------------------------------------9 10define void @global_atomic_xchg_i32_noret(ptr addrspace(1) %ptr, i32 %in) {11; SI-LABEL: global_atomic_xchg_i32_noret:12; SI:       ; %bb.0:13; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14; SI-NEXT:    s_mov_b32 s6, 015; SI-NEXT:    s_mov_b32 s7, 0xf00016; SI-NEXT:    s_mov_b32 s4, s617; SI-NEXT:    s_mov_b32 s5, s618; SI-NEXT:    buffer_atomic_swap v2, v[0:1], s[4:7], 0 addr6419; SI-NEXT:    s_waitcnt vmcnt(0)20; SI-NEXT:    buffer_wbinvl121; SI-NEXT:    s_waitcnt expcnt(0)22; SI-NEXT:    s_setpc_b64 s[30:31]23;24; VI-LABEL: global_atomic_xchg_i32_noret:25; VI:       ; %bb.0:26; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)27; VI-NEXT:    flat_atomic_swap v[0:1], v228; VI-NEXT:    s_waitcnt vmcnt(0)29; VI-NEXT:    buffer_wbinvl1_vol30; VI-NEXT:    s_setpc_b64 s[30:31]31;32; GFX9-LABEL: global_atomic_xchg_i32_noret:33; GFX9:       ; %bb.0:34; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)35; GFX9-NEXT:    global_atomic_swap v[0:1], v2, off36; GFX9-NEXT:    s_waitcnt vmcnt(0)37; GFX9-NEXT:    buffer_wbinvl1_vol38; GFX9-NEXT:    s_setpc_b64 s[30:31]39  %tmp0 = atomicrmw xchg ptr addrspace(1) %ptr, i32 %in seq_cst40  ret void41}42 43define void @global_atomic_xchg_i32_noret_offset(ptr addrspace(1) %out, i32 %in) {44; SI-LABEL: global_atomic_xchg_i32_noret_offset:45; SI:       ; %bb.0:46; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)47; SI-NEXT:    s_mov_b32 s6, 048; SI-NEXT:    s_mov_b32 s7, 0xf00049; SI-NEXT:    s_mov_b32 s4, s650; SI-NEXT:    s_mov_b32 s5, s651; SI-NEXT:    buffer_atomic_swap v2, v[0:1], s[4:7], 0 addr64 offset:1652; SI-NEXT:    s_waitcnt vmcnt(0)53; SI-NEXT:    buffer_wbinvl154; SI-NEXT:    s_waitcnt expcnt(0)55; SI-NEXT:    s_setpc_b64 s[30:31]56;57; VI-LABEL: global_atomic_xchg_i32_noret_offset:58; VI:       ; %bb.0:59; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)60; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v061; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc62; VI-NEXT:    flat_atomic_swap v[0:1], v263; VI-NEXT:    s_waitcnt vmcnt(0)64; VI-NEXT:    buffer_wbinvl1_vol65; VI-NEXT:    s_setpc_b64 s[30:31]66;67; GFX9-LABEL: global_atomic_xchg_i32_noret_offset:68; GFX9:       ; %bb.0:69; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)70; GFX9-NEXT:    global_atomic_swap v[0:1], v2, off offset:1671; GFX9-NEXT:    s_waitcnt vmcnt(0)72; GFX9-NEXT:    buffer_wbinvl1_vol73; GFX9-NEXT:    s_setpc_b64 s[30:31]74  %gep = getelementptr i32, ptr addrspace(1) %out, i32 475  %tmp0 = atomicrmw xchg ptr addrspace(1) %gep, i32 %in seq_cst76  ret void77}78 79define i32 @global_atomic_xchg_i32_ret(ptr addrspace(1) %ptr, i32 %in) {80; SI-LABEL: global_atomic_xchg_i32_ret:81; SI:       ; %bb.0:82; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)83; SI-NEXT:    s_mov_b32 s6, 084; SI-NEXT:    s_mov_b32 s7, 0xf00085; SI-NEXT:    s_mov_b32 s4, s686; SI-NEXT:    s_mov_b32 s5, s687; SI-NEXT:    buffer_atomic_swap v2, v[0:1], s[4:7], 0 addr64 glc88; SI-NEXT:    s_waitcnt vmcnt(0)89; SI-NEXT:    buffer_wbinvl190; SI-NEXT:    v_mov_b32_e32 v0, v291; SI-NEXT:    s_waitcnt expcnt(0)92; SI-NEXT:    s_setpc_b64 s[30:31]93;94; VI-LABEL: global_atomic_xchg_i32_ret:95; VI:       ; %bb.0:96; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)97; VI-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc98; VI-NEXT:    s_waitcnt vmcnt(0)99; VI-NEXT:    buffer_wbinvl1_vol100; VI-NEXT:    s_setpc_b64 s[30:31]101;102; GFX9-LABEL: global_atomic_xchg_i32_ret:103; GFX9:       ; %bb.0:104; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)105; GFX9-NEXT:    global_atomic_swap v0, v[0:1], v2, off glc106; GFX9-NEXT:    s_waitcnt vmcnt(0)107; GFX9-NEXT:    buffer_wbinvl1_vol108; GFX9-NEXT:    s_setpc_b64 s[30:31]109  %result = atomicrmw xchg ptr addrspace(1) %ptr, i32 %in seq_cst110  ret i32 %result111}112 113define i32 @global_atomic_xchg_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {114; SI-LABEL: global_atomic_xchg_i32_ret_offset:115; SI:       ; %bb.0:116; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)117; SI-NEXT:    s_mov_b32 s6, 0118; SI-NEXT:    s_mov_b32 s7, 0xf000119; SI-NEXT:    s_mov_b32 s4, s6120; SI-NEXT:    s_mov_b32 s5, s6121; SI-NEXT:    buffer_atomic_swap v2, v[0:1], s[4:7], 0 addr64 offset:16 glc122; SI-NEXT:    s_waitcnt vmcnt(0)123; SI-NEXT:    buffer_wbinvl1124; SI-NEXT:    v_mov_b32_e32 v0, v2125; SI-NEXT:    s_waitcnt expcnt(0)126; SI-NEXT:    s_setpc_b64 s[30:31]127;128; VI-LABEL: global_atomic_xchg_i32_ret_offset:129; VI:       ; %bb.0:130; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)131; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0132; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc133; VI-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc134; VI-NEXT:    s_waitcnt vmcnt(0)135; VI-NEXT:    buffer_wbinvl1_vol136; VI-NEXT:    s_setpc_b64 s[30:31]137;138; GFX9-LABEL: global_atomic_xchg_i32_ret_offset:139; GFX9:       ; %bb.0:140; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)141; GFX9-NEXT:    global_atomic_swap v0, v[0:1], v2, off offset:16 glc142; GFX9-NEXT:    s_waitcnt vmcnt(0)143; GFX9-NEXT:    buffer_wbinvl1_vol144; GFX9-NEXT:    s_setpc_b64 s[30:31]145  %gep = getelementptr i32, ptr addrspace(1) %out, i32 4146  %result = atomicrmw xchg ptr addrspace(1) %gep, i32 %in seq_cst147  ret i32 %result148}149 150define amdgpu_gfx void @global_atomic_xchg_i32_noret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {151; SI-LABEL: global_atomic_xchg_i32_noret_scalar:152; SI:       ; %bb.0:153; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)154; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1155; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill156; SI-NEXT:    s_mov_b64 exec, s[34:35]157; SI-NEXT:    s_waitcnt expcnt(0)158; SI-NEXT:    v_writelane_b32 v1, s6, 0159; SI-NEXT:    v_writelane_b32 v1, s7, 1160; SI-NEXT:    s_mov_b32 s34, s6161; SI-NEXT:    s_mov_b32 s7, 0xf000162; SI-NEXT:    s_mov_b32 s6, -1163; SI-NEXT:    v_mov_b32_e32 v0, s34164; SI-NEXT:    s_waitcnt vmcnt(0)165; SI-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0166; SI-NEXT:    s_waitcnt vmcnt(0)167; SI-NEXT:    buffer_wbinvl1168; SI-NEXT:    v_readlane_b32 s7, v1, 1169; SI-NEXT:    v_readlane_b32 s6, v1, 0170; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1171; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload172; SI-NEXT:    s_mov_b64 exec, s[34:35]173; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)174; SI-NEXT:    s_setpc_b64 s[30:31]175;176; VI-LABEL: global_atomic_xchg_i32_noret_scalar:177; VI:       ; %bb.0:178; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)179; VI-NEXT:    v_mov_b32_e32 v0, s4180; VI-NEXT:    v_mov_b32_e32 v1, s5181; VI-NEXT:    v_mov_b32_e32 v2, s6182; VI-NEXT:    flat_atomic_swap v[0:1], v2183; VI-NEXT:    s_waitcnt vmcnt(0)184; VI-NEXT:    buffer_wbinvl1_vol185; VI-NEXT:    s_setpc_b64 s[30:31]186;187; GFX9-LABEL: global_atomic_xchg_i32_noret_scalar:188; GFX9:       ; %bb.0:189; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)190; GFX9-NEXT:    v_mov_b32_e32 v0, 0191; GFX9-NEXT:    v_mov_b32_e32 v1, s6192; GFX9-NEXT:    global_atomic_swap v0, v1, s[4:5]193; GFX9-NEXT:    s_waitcnt vmcnt(0)194; GFX9-NEXT:    buffer_wbinvl1_vol195; GFX9-NEXT:    s_setpc_b64 s[30:31]196  %tmp0 = atomicrmw xchg ptr addrspace(1) %ptr, i32 %in seq_cst197  ret void198}199 200define amdgpu_gfx void @global_atomic_xchg_i32_noret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {201; SI-LABEL: global_atomic_xchg_i32_noret_offset_scalar:202; SI:       ; %bb.0:203; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)204; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1205; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill206; SI-NEXT:    s_mov_b64 exec, s[34:35]207; SI-NEXT:    s_waitcnt expcnt(0)208; SI-NEXT:    v_writelane_b32 v1, s6, 0209; SI-NEXT:    v_writelane_b32 v1, s7, 1210; SI-NEXT:    s_mov_b32 s34, s6211; SI-NEXT:    s_mov_b32 s7, 0xf000212; SI-NEXT:    s_mov_b32 s6, -1213; SI-NEXT:    v_mov_b32_e32 v0, s34214; SI-NEXT:    s_waitcnt vmcnt(0)215; SI-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 offset:16216; SI-NEXT:    s_waitcnt vmcnt(0)217; SI-NEXT:    buffer_wbinvl1218; SI-NEXT:    v_readlane_b32 s7, v1, 1219; SI-NEXT:    v_readlane_b32 s6, v1, 0220; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1221; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload222; SI-NEXT:    s_mov_b64 exec, s[34:35]223; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)224; SI-NEXT:    s_setpc_b64 s[30:31]225;226; VI-LABEL: global_atomic_xchg_i32_noret_offset_scalar:227; VI:       ; %bb.0:228; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)229; VI-NEXT:    s_add_u32 s34, s4, 16230; VI-NEXT:    s_addc_u32 s35, s5, 0231; VI-NEXT:    v_mov_b32_e32 v0, s34232; VI-NEXT:    v_mov_b32_e32 v1, s35233; VI-NEXT:    v_mov_b32_e32 v2, s6234; VI-NEXT:    flat_atomic_swap v[0:1], v2235; VI-NEXT:    s_waitcnt vmcnt(0)236; VI-NEXT:    buffer_wbinvl1_vol237; VI-NEXT:    s_setpc_b64 s[30:31]238;239; GFX9-LABEL: global_atomic_xchg_i32_noret_offset_scalar:240; GFX9:       ; %bb.0:241; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)242; GFX9-NEXT:    v_mov_b32_e32 v0, 0243; GFX9-NEXT:    v_mov_b32_e32 v1, s6244; GFX9-NEXT:    global_atomic_swap v0, v1, s[4:5] offset:16245; GFX9-NEXT:    s_waitcnt vmcnt(0)246; GFX9-NEXT:    buffer_wbinvl1_vol247; GFX9-NEXT:    s_setpc_b64 s[30:31]248  %gep = getelementptr i32, ptr addrspace(1) %out, i32 4249  %tmp0 = atomicrmw xchg ptr addrspace(1) %gep, i32 %in seq_cst250  ret void251}252 253define amdgpu_gfx i32 @global_atomic_xchg_i32_ret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {254; SI-LABEL: global_atomic_xchg_i32_ret_scalar:255; SI:       ; %bb.0:256; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)257; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1258; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill259; SI-NEXT:    s_mov_b64 exec, s[34:35]260; SI-NEXT:    s_waitcnt expcnt(0)261; SI-NEXT:    v_writelane_b32 v1, s6, 0262; SI-NEXT:    v_writelane_b32 v1, s7, 1263; SI-NEXT:    s_mov_b32 s34, s6264; SI-NEXT:    s_mov_b32 s7, 0xf000265; SI-NEXT:    s_mov_b32 s6, -1266; SI-NEXT:    v_mov_b32_e32 v0, s34267; SI-NEXT:    s_waitcnt vmcnt(0)268; SI-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc269; SI-NEXT:    s_waitcnt vmcnt(0)270; SI-NEXT:    buffer_wbinvl1271; SI-NEXT:    v_readlane_b32 s7, v1, 1272; SI-NEXT:    v_readlane_b32 s6, v1, 0273; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1274; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload275; SI-NEXT:    s_mov_b64 exec, s[34:35]276; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)277; SI-NEXT:    s_setpc_b64 s[30:31]278;279; VI-LABEL: global_atomic_xchg_i32_ret_scalar:280; VI:       ; %bb.0:281; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)282; VI-NEXT:    v_mov_b32_e32 v0, s4283; VI-NEXT:    v_mov_b32_e32 v1, s5284; VI-NEXT:    v_mov_b32_e32 v2, s6285; VI-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc286; VI-NEXT:    s_waitcnt vmcnt(0)287; VI-NEXT:    buffer_wbinvl1_vol288; VI-NEXT:    s_setpc_b64 s[30:31]289;290; GFX9-LABEL: global_atomic_xchg_i32_ret_scalar:291; GFX9:       ; %bb.0:292; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)293; GFX9-NEXT:    v_mov_b32_e32 v0, 0294; GFX9-NEXT:    v_mov_b32_e32 v1, s6295; GFX9-NEXT:    global_atomic_swap v0, v0, v1, s[4:5] glc296; GFX9-NEXT:    s_waitcnt vmcnt(0)297; GFX9-NEXT:    buffer_wbinvl1_vol298; GFX9-NEXT:    s_setpc_b64 s[30:31]299  %result = atomicrmw xchg ptr addrspace(1) %ptr, i32 %in seq_cst300  ret i32 %result301}302 303define amdgpu_gfx i32 @global_atomic_xchg_i32_ret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {304; SI-LABEL: global_atomic_xchg_i32_ret_offset_scalar:305; SI:       ; %bb.0:306; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)307; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1308; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill309; SI-NEXT:    s_mov_b64 exec, s[34:35]310; SI-NEXT:    s_waitcnt expcnt(0)311; SI-NEXT:    v_writelane_b32 v1, s6, 0312; SI-NEXT:    v_writelane_b32 v1, s7, 1313; SI-NEXT:    s_mov_b32 s34, s6314; SI-NEXT:    s_mov_b32 s7, 0xf000315; SI-NEXT:    s_mov_b32 s6, -1316; SI-NEXT:    v_mov_b32_e32 v0, s34317; SI-NEXT:    s_waitcnt vmcnt(0)318; SI-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 offset:16 glc319; SI-NEXT:    s_waitcnt vmcnt(0)320; SI-NEXT:    buffer_wbinvl1321; SI-NEXT:    v_readlane_b32 s7, v1, 1322; SI-NEXT:    v_readlane_b32 s6, v1, 0323; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1324; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload325; SI-NEXT:    s_mov_b64 exec, s[34:35]326; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)327; SI-NEXT:    s_setpc_b64 s[30:31]328;329; VI-LABEL: global_atomic_xchg_i32_ret_offset_scalar:330; VI:       ; %bb.0:331; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)332; VI-NEXT:    s_add_u32 s34, s4, 16333; VI-NEXT:    s_addc_u32 s35, s5, 0334; VI-NEXT:    v_mov_b32_e32 v0, s34335; VI-NEXT:    v_mov_b32_e32 v1, s35336; VI-NEXT:    v_mov_b32_e32 v2, s6337; VI-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc338; VI-NEXT:    s_waitcnt vmcnt(0)339; VI-NEXT:    buffer_wbinvl1_vol340; VI-NEXT:    s_setpc_b64 s[30:31]341;342; GFX9-LABEL: global_atomic_xchg_i32_ret_offset_scalar:343; GFX9:       ; %bb.0:344; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)345; GFX9-NEXT:    v_mov_b32_e32 v0, 0346; GFX9-NEXT:    v_mov_b32_e32 v1, s6347; GFX9-NEXT:    global_atomic_swap v0, v0, v1, s[4:5] offset:16 glc348; GFX9-NEXT:    s_waitcnt vmcnt(0)349; GFX9-NEXT:    buffer_wbinvl1_vol350; GFX9-NEXT:    s_setpc_b64 s[30:31]351  %gep = getelementptr i32, ptr addrspace(1) %out, i32 4352  %result = atomicrmw xchg ptr addrspace(1) %gep, i32 %in seq_cst353  ret i32 %result354}355 356define void @global_atomic_xchg_i32_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {357; SI-LABEL: global_atomic_xchg_i32_noret_offset__amdgpu_no_remote_memory:358; SI:       ; %bb.0:359; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)360; SI-NEXT:    s_mov_b32 s6, 0361; SI-NEXT:    s_mov_b32 s7, 0xf000362; SI-NEXT:    s_mov_b32 s4, s6363; SI-NEXT:    s_mov_b32 s5, s6364; SI-NEXT:    buffer_atomic_swap v2, v[0:1], s[4:7], 0 addr64 offset:16365; SI-NEXT:    s_waitcnt vmcnt(0)366; SI-NEXT:    buffer_wbinvl1367; SI-NEXT:    s_waitcnt expcnt(0)368; SI-NEXT:    s_setpc_b64 s[30:31]369;370; VI-LABEL: global_atomic_xchg_i32_noret_offset__amdgpu_no_remote_memory:371; VI:       ; %bb.0:372; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)373; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0374; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc375; VI-NEXT:    flat_atomic_swap v[0:1], v2376; VI-NEXT:    s_waitcnt vmcnt(0)377; VI-NEXT:    buffer_wbinvl1_vol378; VI-NEXT:    s_setpc_b64 s[30:31]379;380; GFX9-LABEL: global_atomic_xchg_i32_noret_offset__amdgpu_no_remote_memory:381; GFX9:       ; %bb.0:382; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)383; GFX9-NEXT:    global_atomic_swap v[0:1], v2, off offset:16384; GFX9-NEXT:    s_waitcnt vmcnt(0)385; GFX9-NEXT:    buffer_wbinvl1_vol386; GFX9-NEXT:    s_setpc_b64 s[30:31]387  %gep = getelementptr i32, ptr addrspace(1) %out, i64 4388  %tmp0 = atomicrmw xchg ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0389  ret void390}391 392define i32 @global_atomic_xchg_i32_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {393; SI-LABEL: global_atomic_xchg_i32_ret_offset__amdgpu_no_remote_memory:394; SI:       ; %bb.0:395; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)396; SI-NEXT:    s_mov_b32 s6, 0397; SI-NEXT:    s_mov_b32 s7, 0xf000398; SI-NEXT:    s_mov_b32 s4, s6399; SI-NEXT:    s_mov_b32 s5, s6400; SI-NEXT:    buffer_atomic_swap v2, v[0:1], s[4:7], 0 addr64 offset:16 glc401; SI-NEXT:    s_waitcnt vmcnt(0)402; SI-NEXT:    buffer_wbinvl1403; SI-NEXT:    v_mov_b32_e32 v0, v2404; SI-NEXT:    s_waitcnt expcnt(0)405; SI-NEXT:    s_setpc_b64 s[30:31]406;407; VI-LABEL: global_atomic_xchg_i32_ret_offset__amdgpu_no_remote_memory:408; VI:       ; %bb.0:409; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)410; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0411; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc412; VI-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc413; VI-NEXT:    s_waitcnt vmcnt(0)414; VI-NEXT:    buffer_wbinvl1_vol415; VI-NEXT:    s_setpc_b64 s[30:31]416;417; GFX9-LABEL: global_atomic_xchg_i32_ret_offset__amdgpu_no_remote_memory:418; GFX9:       ; %bb.0:419; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)420; GFX9-NEXT:    global_atomic_swap v0, v[0:1], v2, off offset:16 glc421; GFX9-NEXT:    s_waitcnt vmcnt(0)422; GFX9-NEXT:    buffer_wbinvl1_vol423; GFX9-NEXT:    s_setpc_b64 s[30:31]424  %gep = getelementptr i32, ptr addrspace(1) %out, i64 4425  %result = atomicrmw xchg ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0426  ret i32 %result427}428 429; ---------------------------------------------------------------------430; atomicrmw xchg f32431; ---------------------------------------------------------------------432 433define void @global_atomic_xchg_f32_noret(ptr addrspace(1) %ptr, float %in) {434; SI-LABEL: global_atomic_xchg_f32_noret:435; SI:       ; %bb.0:436; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)437; SI-NEXT:    s_mov_b32 s6, 0438; SI-NEXT:    s_mov_b32 s7, 0xf000439; SI-NEXT:    s_mov_b32 s4, s6440; SI-NEXT:    s_mov_b32 s5, s6441; SI-NEXT:    buffer_atomic_swap v2, v[0:1], s[4:7], 0 addr64442; SI-NEXT:    s_waitcnt vmcnt(0)443; SI-NEXT:    buffer_wbinvl1444; SI-NEXT:    s_waitcnt expcnt(0)445; SI-NEXT:    s_setpc_b64 s[30:31]446;447; VI-LABEL: global_atomic_xchg_f32_noret:448; VI:       ; %bb.0:449; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)450; VI-NEXT:    flat_atomic_swap v[0:1], v2451; VI-NEXT:    s_waitcnt vmcnt(0)452; VI-NEXT:    buffer_wbinvl1_vol453; VI-NEXT:    s_setpc_b64 s[30:31]454;455; GFX9-LABEL: global_atomic_xchg_f32_noret:456; GFX9:       ; %bb.0:457; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)458; GFX9-NEXT:    global_atomic_swap v[0:1], v2, off459; GFX9-NEXT:    s_waitcnt vmcnt(0)460; GFX9-NEXT:    buffer_wbinvl1_vol461; GFX9-NEXT:    s_setpc_b64 s[30:31]462  %tmp0 = atomicrmw xchg ptr addrspace(1) %ptr, float %in seq_cst463  ret void464}465 466define void @global_atomic_xchg_f32_noret_offset(ptr addrspace(1) %out, float %in) {467; SI-LABEL: global_atomic_xchg_f32_noret_offset:468; SI:       ; %bb.0:469; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)470; SI-NEXT:    s_mov_b32 s6, 0471; SI-NEXT:    s_mov_b32 s7, 0xf000472; SI-NEXT:    s_mov_b32 s4, s6473; SI-NEXT:    s_mov_b32 s5, s6474; SI-NEXT:    buffer_atomic_swap v2, v[0:1], s[4:7], 0 addr64 offset:16475; SI-NEXT:    s_waitcnt vmcnt(0)476; SI-NEXT:    buffer_wbinvl1477; SI-NEXT:    s_waitcnt expcnt(0)478; SI-NEXT:    s_setpc_b64 s[30:31]479;480; VI-LABEL: global_atomic_xchg_f32_noret_offset:481; VI:       ; %bb.0:482; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)483; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0484; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc485; VI-NEXT:    flat_atomic_swap v[0:1], v2486; VI-NEXT:    s_waitcnt vmcnt(0)487; VI-NEXT:    buffer_wbinvl1_vol488; VI-NEXT:    s_setpc_b64 s[30:31]489;490; GFX9-LABEL: global_atomic_xchg_f32_noret_offset:491; GFX9:       ; %bb.0:492; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)493; GFX9-NEXT:    global_atomic_swap v[0:1], v2, off offset:16494; GFX9-NEXT:    s_waitcnt vmcnt(0)495; GFX9-NEXT:    buffer_wbinvl1_vol496; GFX9-NEXT:    s_setpc_b64 s[30:31]497  %gep = getelementptr float, ptr addrspace(1) %out, i32 4498  %tmp0 = atomicrmw xchg ptr addrspace(1) %gep, float %in seq_cst499  ret void500}501 502define float @global_atomic_xchg_f32_ret(ptr addrspace(1) %ptr, float %in) {503; SI-LABEL: global_atomic_xchg_f32_ret:504; SI:       ; %bb.0:505; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)506; SI-NEXT:    s_mov_b32 s6, 0507; SI-NEXT:    s_mov_b32 s7, 0xf000508; SI-NEXT:    s_mov_b32 s4, s6509; SI-NEXT:    s_mov_b32 s5, s6510; SI-NEXT:    buffer_atomic_swap v2, v[0:1], s[4:7], 0 addr64 glc511; SI-NEXT:    s_waitcnt vmcnt(0)512; SI-NEXT:    buffer_wbinvl1513; SI-NEXT:    v_mov_b32_e32 v0, v2514; SI-NEXT:    s_waitcnt expcnt(0)515; SI-NEXT:    s_setpc_b64 s[30:31]516;517; VI-LABEL: global_atomic_xchg_f32_ret:518; VI:       ; %bb.0:519; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)520; VI-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc521; VI-NEXT:    s_waitcnt vmcnt(0)522; VI-NEXT:    buffer_wbinvl1_vol523; VI-NEXT:    s_setpc_b64 s[30:31]524;525; GFX9-LABEL: global_atomic_xchg_f32_ret:526; GFX9:       ; %bb.0:527; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)528; GFX9-NEXT:    global_atomic_swap v0, v[0:1], v2, off glc529; GFX9-NEXT:    s_waitcnt vmcnt(0)530; GFX9-NEXT:    buffer_wbinvl1_vol531; GFX9-NEXT:    s_setpc_b64 s[30:31]532  %result = atomicrmw xchg ptr addrspace(1) %ptr, float %in seq_cst533  ret float %result534}535 536define float @global_atomic_xchg_f32_ret_offset(ptr addrspace(1) %out, float %in) {537; SI-LABEL: global_atomic_xchg_f32_ret_offset:538; SI:       ; %bb.0:539; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)540; SI-NEXT:    s_mov_b32 s6, 0541; SI-NEXT:    s_mov_b32 s7, 0xf000542; SI-NEXT:    s_mov_b32 s4, s6543; SI-NEXT:    s_mov_b32 s5, s6544; SI-NEXT:    buffer_atomic_swap v2, v[0:1], s[4:7], 0 addr64 offset:16 glc545; SI-NEXT:    s_waitcnt vmcnt(0)546; SI-NEXT:    buffer_wbinvl1547; SI-NEXT:    v_mov_b32_e32 v0, v2548; SI-NEXT:    s_waitcnt expcnt(0)549; SI-NEXT:    s_setpc_b64 s[30:31]550;551; VI-LABEL: global_atomic_xchg_f32_ret_offset:552; VI:       ; %bb.0:553; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)554; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0555; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc556; VI-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc557; VI-NEXT:    s_waitcnt vmcnt(0)558; VI-NEXT:    buffer_wbinvl1_vol559; VI-NEXT:    s_setpc_b64 s[30:31]560;561; GFX9-LABEL: global_atomic_xchg_f32_ret_offset:562; GFX9:       ; %bb.0:563; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)564; GFX9-NEXT:    global_atomic_swap v0, v[0:1], v2, off offset:16 glc565; GFX9-NEXT:    s_waitcnt vmcnt(0)566; GFX9-NEXT:    buffer_wbinvl1_vol567; GFX9-NEXT:    s_setpc_b64 s[30:31]568  %gep = getelementptr float, ptr addrspace(1) %out, i32 4569  %result = atomicrmw xchg ptr addrspace(1) %gep, float %in seq_cst570  ret float %result571}572 573define amdgpu_gfx void @global_atomic_xchg_f32_noret_scalar(ptr addrspace(1) inreg %ptr, float inreg %in) {574; SI-LABEL: global_atomic_xchg_f32_noret_scalar:575; SI:       ; %bb.0:576; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)577; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1578; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill579; SI-NEXT:    s_mov_b64 exec, s[34:35]580; SI-NEXT:    s_waitcnt expcnt(0)581; SI-NEXT:    v_writelane_b32 v1, s6, 0582; SI-NEXT:    v_writelane_b32 v1, s7, 1583; SI-NEXT:    s_mov_b32 s34, s6584; SI-NEXT:    s_mov_b32 s7, 0xf000585; SI-NEXT:    s_mov_b32 s6, -1586; SI-NEXT:    v_mov_b32_e32 v0, s34587; SI-NEXT:    s_waitcnt vmcnt(0)588; SI-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0589; SI-NEXT:    s_waitcnt vmcnt(0)590; SI-NEXT:    buffer_wbinvl1591; SI-NEXT:    v_readlane_b32 s7, v1, 1592; SI-NEXT:    v_readlane_b32 s6, v1, 0593; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1594; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload595; SI-NEXT:    s_mov_b64 exec, s[34:35]596; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)597; SI-NEXT:    s_setpc_b64 s[30:31]598;599; VI-LABEL: global_atomic_xchg_f32_noret_scalar:600; VI:       ; %bb.0:601; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)602; VI-NEXT:    v_mov_b32_e32 v0, s4603; VI-NEXT:    v_mov_b32_e32 v1, s5604; VI-NEXT:    v_mov_b32_e32 v2, s6605; VI-NEXT:    flat_atomic_swap v[0:1], v2606; VI-NEXT:    s_waitcnt vmcnt(0)607; VI-NEXT:    buffer_wbinvl1_vol608; VI-NEXT:    s_setpc_b64 s[30:31]609;610; GFX9-LABEL: global_atomic_xchg_f32_noret_scalar:611; GFX9:       ; %bb.0:612; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)613; GFX9-NEXT:    v_mov_b32_e32 v0, 0614; GFX9-NEXT:    v_mov_b32_e32 v1, s6615; GFX9-NEXT:    global_atomic_swap v0, v1, s[4:5]616; GFX9-NEXT:    s_waitcnt vmcnt(0)617; GFX9-NEXT:    buffer_wbinvl1_vol618; GFX9-NEXT:    s_setpc_b64 s[30:31]619  %tmp0 = atomicrmw xchg ptr addrspace(1) %ptr, float %in seq_cst620  ret void621}622 623define amdgpu_gfx void @global_atomic_xchg_f32_noret_offset_scalar(ptr addrspace(1) inreg %out, float inreg %in) {624; SI-LABEL: global_atomic_xchg_f32_noret_offset_scalar:625; SI:       ; %bb.0:626; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)627; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1628; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill629; SI-NEXT:    s_mov_b64 exec, s[34:35]630; SI-NEXT:    s_waitcnt expcnt(0)631; SI-NEXT:    v_writelane_b32 v1, s6, 0632; SI-NEXT:    v_writelane_b32 v1, s7, 1633; SI-NEXT:    s_mov_b32 s34, s6634; SI-NEXT:    s_mov_b32 s7, 0xf000635; SI-NEXT:    s_mov_b32 s6, -1636; SI-NEXT:    v_mov_b32_e32 v0, s34637; SI-NEXT:    s_waitcnt vmcnt(0)638; SI-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 offset:16639; SI-NEXT:    s_waitcnt vmcnt(0)640; SI-NEXT:    buffer_wbinvl1641; SI-NEXT:    v_readlane_b32 s7, v1, 1642; SI-NEXT:    v_readlane_b32 s6, v1, 0643; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1644; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload645; SI-NEXT:    s_mov_b64 exec, s[34:35]646; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)647; SI-NEXT:    s_setpc_b64 s[30:31]648;649; VI-LABEL: global_atomic_xchg_f32_noret_offset_scalar:650; VI:       ; %bb.0:651; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)652; VI-NEXT:    s_add_u32 s34, s4, 16653; VI-NEXT:    s_addc_u32 s35, s5, 0654; VI-NEXT:    v_mov_b32_e32 v0, s34655; VI-NEXT:    v_mov_b32_e32 v1, s35656; VI-NEXT:    v_mov_b32_e32 v2, s6657; VI-NEXT:    flat_atomic_swap v[0:1], v2658; VI-NEXT:    s_waitcnt vmcnt(0)659; VI-NEXT:    buffer_wbinvl1_vol660; VI-NEXT:    s_setpc_b64 s[30:31]661;662; GFX9-LABEL: global_atomic_xchg_f32_noret_offset_scalar:663; GFX9:       ; %bb.0:664; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)665; GFX9-NEXT:    v_mov_b32_e32 v0, 0666; GFX9-NEXT:    v_mov_b32_e32 v1, s6667; GFX9-NEXT:    global_atomic_swap v0, v1, s[4:5] offset:16668; GFX9-NEXT:    s_waitcnt vmcnt(0)669; GFX9-NEXT:    buffer_wbinvl1_vol670; GFX9-NEXT:    s_setpc_b64 s[30:31]671  %gep = getelementptr float, ptr addrspace(1) %out, i32 4672  %tmp0 = atomicrmw xchg ptr addrspace(1) %gep, float %in seq_cst673  ret void674}675 676define amdgpu_gfx float @global_atomic_xchg_f32_ret_scalar(ptr addrspace(1) inreg %ptr, float inreg %in) {677; SI-LABEL: global_atomic_xchg_f32_ret_scalar:678; SI:       ; %bb.0:679; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)680; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1681; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill682; SI-NEXT:    s_mov_b64 exec, s[34:35]683; SI-NEXT:    s_waitcnt expcnt(0)684; SI-NEXT:    v_writelane_b32 v1, s6, 0685; SI-NEXT:    v_writelane_b32 v1, s7, 1686; SI-NEXT:    s_mov_b32 s34, s6687; SI-NEXT:    s_mov_b32 s7, 0xf000688; SI-NEXT:    s_mov_b32 s6, -1689; SI-NEXT:    v_mov_b32_e32 v0, s34690; SI-NEXT:    s_waitcnt vmcnt(0)691; SI-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc692; SI-NEXT:    s_waitcnt vmcnt(0)693; SI-NEXT:    buffer_wbinvl1694; SI-NEXT:    v_readlane_b32 s7, v1, 1695; SI-NEXT:    v_readlane_b32 s6, v1, 0696; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1697; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload698; SI-NEXT:    s_mov_b64 exec, s[34:35]699; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)700; SI-NEXT:    s_setpc_b64 s[30:31]701;702; VI-LABEL: global_atomic_xchg_f32_ret_scalar:703; VI:       ; %bb.0:704; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)705; VI-NEXT:    v_mov_b32_e32 v0, s4706; VI-NEXT:    v_mov_b32_e32 v1, s5707; VI-NEXT:    v_mov_b32_e32 v2, s6708; VI-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc709; VI-NEXT:    s_waitcnt vmcnt(0)710; VI-NEXT:    buffer_wbinvl1_vol711; VI-NEXT:    s_setpc_b64 s[30:31]712;713; GFX9-LABEL: global_atomic_xchg_f32_ret_scalar:714; GFX9:       ; %bb.0:715; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)716; GFX9-NEXT:    v_mov_b32_e32 v0, 0717; GFX9-NEXT:    v_mov_b32_e32 v1, s6718; GFX9-NEXT:    global_atomic_swap v0, v0, v1, s[4:5] glc719; GFX9-NEXT:    s_waitcnt vmcnt(0)720; GFX9-NEXT:    buffer_wbinvl1_vol721; GFX9-NEXT:    s_setpc_b64 s[30:31]722  %result = atomicrmw xchg ptr addrspace(1) %ptr, float %in seq_cst723  ret float %result724}725 726define amdgpu_gfx float @global_atomic_xchg_f32_ret_offset_scalar(ptr addrspace(1) inreg %out, float inreg %in) {727; SI-LABEL: global_atomic_xchg_f32_ret_offset_scalar:728; SI:       ; %bb.0:729; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)730; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1731; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill732; SI-NEXT:    s_mov_b64 exec, s[34:35]733; SI-NEXT:    s_waitcnt expcnt(0)734; SI-NEXT:    v_writelane_b32 v1, s6, 0735; SI-NEXT:    v_writelane_b32 v1, s7, 1736; SI-NEXT:    s_mov_b32 s34, s6737; SI-NEXT:    s_mov_b32 s7, 0xf000738; SI-NEXT:    s_mov_b32 s6, -1739; SI-NEXT:    v_mov_b32_e32 v0, s34740; SI-NEXT:    s_waitcnt vmcnt(0)741; SI-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 offset:16 glc742; SI-NEXT:    s_waitcnt vmcnt(0)743; SI-NEXT:    buffer_wbinvl1744; SI-NEXT:    v_readlane_b32 s7, v1, 1745; SI-NEXT:    v_readlane_b32 s6, v1, 0746; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1747; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload748; SI-NEXT:    s_mov_b64 exec, s[34:35]749; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)750; SI-NEXT:    s_setpc_b64 s[30:31]751;752; VI-LABEL: global_atomic_xchg_f32_ret_offset_scalar:753; VI:       ; %bb.0:754; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)755; VI-NEXT:    s_add_u32 s34, s4, 16756; VI-NEXT:    s_addc_u32 s35, s5, 0757; VI-NEXT:    v_mov_b32_e32 v0, s34758; VI-NEXT:    v_mov_b32_e32 v1, s35759; VI-NEXT:    v_mov_b32_e32 v2, s6760; VI-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc761; VI-NEXT:    s_waitcnt vmcnt(0)762; VI-NEXT:    buffer_wbinvl1_vol763; VI-NEXT:    s_setpc_b64 s[30:31]764;765; GFX9-LABEL: global_atomic_xchg_f32_ret_offset_scalar:766; GFX9:       ; %bb.0:767; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)768; GFX9-NEXT:    v_mov_b32_e32 v0, 0769; GFX9-NEXT:    v_mov_b32_e32 v1, s6770; GFX9-NEXT:    global_atomic_swap v0, v0, v1, s[4:5] offset:16 glc771; GFX9-NEXT:    s_waitcnt vmcnt(0)772; GFX9-NEXT:    buffer_wbinvl1_vol773; GFX9-NEXT:    s_setpc_b64 s[30:31]774  %gep = getelementptr float, ptr addrspace(1) %out, i32 4775  %result = atomicrmw xchg ptr addrspace(1) %gep, float %in seq_cst776  ret float %result777}778 779define void @global_atomic_xchg_f32_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, float %in) {780; SI-LABEL: global_atomic_xchg_f32_noret_offset__amdgpu_no_remote_memory:781; SI:       ; %bb.0:782; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)783; SI-NEXT:    s_mov_b32 s6, 0784; SI-NEXT:    s_mov_b32 s7, 0xf000785; SI-NEXT:    s_mov_b32 s4, s6786; SI-NEXT:    s_mov_b32 s5, s6787; SI-NEXT:    buffer_atomic_swap v2, v[0:1], s[4:7], 0 addr64 offset:16788; SI-NEXT:    s_waitcnt vmcnt(0)789; SI-NEXT:    buffer_wbinvl1790; SI-NEXT:    s_waitcnt expcnt(0)791; SI-NEXT:    s_setpc_b64 s[30:31]792;793; VI-LABEL: global_atomic_xchg_f32_noret_offset__amdgpu_no_remote_memory:794; VI:       ; %bb.0:795; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)796; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0797; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc798; VI-NEXT:    flat_atomic_swap v[0:1], v2799; VI-NEXT:    s_waitcnt vmcnt(0)800; VI-NEXT:    buffer_wbinvl1_vol801; VI-NEXT:    s_setpc_b64 s[30:31]802;803; GFX9-LABEL: global_atomic_xchg_f32_noret_offset__amdgpu_no_remote_memory:804; GFX9:       ; %bb.0:805; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)806; GFX9-NEXT:    global_atomic_swap v[0:1], v2, off offset:16807; GFX9-NEXT:    s_waitcnt vmcnt(0)808; GFX9-NEXT:    buffer_wbinvl1_vol809; GFX9-NEXT:    s_setpc_b64 s[30:31]810  %gep = getelementptr i32, ptr addrspace(1) %out, i64 4811  %tmp0 = atomicrmw xchg ptr addrspace(1) %gep, float %in seq_cst, !amdgpu.no.remote.memory !0812  ret void813}814 815define float @global_atomic_xchg_f32_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, float %in) {816; SI-LABEL: global_atomic_xchg_f32_ret_offset__amdgpu_no_remote_memory:817; SI:       ; %bb.0:818; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)819; SI-NEXT:    s_mov_b32 s6, 0820; SI-NEXT:    s_mov_b32 s7, 0xf000821; SI-NEXT:    s_mov_b32 s4, s6822; SI-NEXT:    s_mov_b32 s5, s6823; SI-NEXT:    buffer_atomic_swap v2, v[0:1], s[4:7], 0 addr64 offset:16 glc824; SI-NEXT:    s_waitcnt vmcnt(0)825; SI-NEXT:    buffer_wbinvl1826; SI-NEXT:    v_mov_b32_e32 v0, v2827; SI-NEXT:    s_waitcnt expcnt(0)828; SI-NEXT:    s_setpc_b64 s[30:31]829;830; VI-LABEL: global_atomic_xchg_f32_ret_offset__amdgpu_no_remote_memory:831; VI:       ; %bb.0:832; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)833; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0834; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc835; VI-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc836; VI-NEXT:    s_waitcnt vmcnt(0)837; VI-NEXT:    buffer_wbinvl1_vol838; VI-NEXT:    s_setpc_b64 s[30:31]839;840; GFX9-LABEL: global_atomic_xchg_f32_ret_offset__amdgpu_no_remote_memory:841; GFX9:       ; %bb.0:842; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)843; GFX9-NEXT:    global_atomic_swap v0, v[0:1], v2, off offset:16 glc844; GFX9-NEXT:    s_waitcnt vmcnt(0)845; GFX9-NEXT:    buffer_wbinvl1_vol846; GFX9-NEXT:    s_setpc_b64 s[30:31]847  %gep = getelementptr i32, ptr addrspace(1) %out, i64 4848  %result = atomicrmw xchg ptr addrspace(1) %gep, float %in seq_cst, !amdgpu.no.remote.memory !0849  ret float %result850}851 852; ---------------------------------------------------------------------853; atomicrmw add854; ---------------------------------------------------------------------855 856define void @global_atomic_add_i32_noret(ptr addrspace(1) %ptr, i32 %in) {857; SI-LABEL: global_atomic_add_i32_noret:858; SI:       ; %bb.0:859; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)860; SI-NEXT:    s_mov_b32 s6, 0861; SI-NEXT:    s_mov_b32 s7, 0xf000862; SI-NEXT:    s_mov_b32 s4, s6863; SI-NEXT:    s_mov_b32 s5, s6864; SI-NEXT:    buffer_atomic_add v2, v[0:1], s[4:7], 0 addr64865; SI-NEXT:    s_waitcnt vmcnt(0)866; SI-NEXT:    buffer_wbinvl1867; SI-NEXT:    s_waitcnt expcnt(0)868; SI-NEXT:    s_setpc_b64 s[30:31]869;870; VI-LABEL: global_atomic_add_i32_noret:871; VI:       ; %bb.0:872; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)873; VI-NEXT:    flat_atomic_add v[0:1], v2874; VI-NEXT:    s_waitcnt vmcnt(0)875; VI-NEXT:    buffer_wbinvl1_vol876; VI-NEXT:    s_setpc_b64 s[30:31]877;878; GFX9-LABEL: global_atomic_add_i32_noret:879; GFX9:       ; %bb.0:880; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)881; GFX9-NEXT:    global_atomic_add v[0:1], v2, off882; GFX9-NEXT:    s_waitcnt vmcnt(0)883; GFX9-NEXT:    buffer_wbinvl1_vol884; GFX9-NEXT:    s_setpc_b64 s[30:31]885  %tmp0 = atomicrmw add ptr addrspace(1) %ptr, i32 %in seq_cst886  ret void887}888 889define void @global_atomic_add_i32_noret_offset(ptr addrspace(1) %out, i32 %in) {890; SI-LABEL: global_atomic_add_i32_noret_offset:891; SI:       ; %bb.0:892; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)893; SI-NEXT:    s_mov_b32 s6, 0894; SI-NEXT:    s_mov_b32 s7, 0xf000895; SI-NEXT:    s_mov_b32 s4, s6896; SI-NEXT:    s_mov_b32 s5, s6897; SI-NEXT:    buffer_atomic_add v2, v[0:1], s[4:7], 0 addr64 offset:16898; SI-NEXT:    s_waitcnt vmcnt(0)899; SI-NEXT:    buffer_wbinvl1900; SI-NEXT:    s_waitcnt expcnt(0)901; SI-NEXT:    s_setpc_b64 s[30:31]902;903; VI-LABEL: global_atomic_add_i32_noret_offset:904; VI:       ; %bb.0:905; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)906; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0907; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc908; VI-NEXT:    flat_atomic_add v[0:1], v2909; VI-NEXT:    s_waitcnt vmcnt(0)910; VI-NEXT:    buffer_wbinvl1_vol911; VI-NEXT:    s_setpc_b64 s[30:31]912;913; GFX9-LABEL: global_atomic_add_i32_noret_offset:914; GFX9:       ; %bb.0:915; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)916; GFX9-NEXT:    global_atomic_add v[0:1], v2, off offset:16917; GFX9-NEXT:    s_waitcnt vmcnt(0)918; GFX9-NEXT:    buffer_wbinvl1_vol919; GFX9-NEXT:    s_setpc_b64 s[30:31]920  %gep = getelementptr i32, ptr addrspace(1) %out, i32 4921  %tmp0 = atomicrmw add ptr addrspace(1) %gep, i32 %in seq_cst922  ret void923}924 925define i32 @global_atomic_add_i32_ret(ptr addrspace(1) %ptr, i32 %in) {926; SI-LABEL: global_atomic_add_i32_ret:927; SI:       ; %bb.0:928; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)929; SI-NEXT:    s_mov_b32 s6, 0930; SI-NEXT:    s_mov_b32 s7, 0xf000931; SI-NEXT:    s_mov_b32 s4, s6932; SI-NEXT:    s_mov_b32 s5, s6933; SI-NEXT:    buffer_atomic_add v2, v[0:1], s[4:7], 0 addr64 glc934; SI-NEXT:    s_waitcnt vmcnt(0)935; SI-NEXT:    buffer_wbinvl1936; SI-NEXT:    v_mov_b32_e32 v0, v2937; SI-NEXT:    s_waitcnt expcnt(0)938; SI-NEXT:    s_setpc_b64 s[30:31]939;940; VI-LABEL: global_atomic_add_i32_ret:941; VI:       ; %bb.0:942; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)943; VI-NEXT:    flat_atomic_add v0, v[0:1], v2 glc944; VI-NEXT:    s_waitcnt vmcnt(0)945; VI-NEXT:    buffer_wbinvl1_vol946; VI-NEXT:    s_setpc_b64 s[30:31]947;948; GFX9-LABEL: global_atomic_add_i32_ret:949; GFX9:       ; %bb.0:950; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)951; GFX9-NEXT:    global_atomic_add v0, v[0:1], v2, off glc952; GFX9-NEXT:    s_waitcnt vmcnt(0)953; GFX9-NEXT:    buffer_wbinvl1_vol954; GFX9-NEXT:    s_setpc_b64 s[30:31]955  %result = atomicrmw add ptr addrspace(1) %ptr, i32 %in seq_cst956  ret i32 %result957}958 959define i32 @global_atomic_add_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {960; SI-LABEL: global_atomic_add_i32_ret_offset:961; SI:       ; %bb.0:962; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)963; SI-NEXT:    s_mov_b32 s6, 0964; SI-NEXT:    s_mov_b32 s7, 0xf000965; SI-NEXT:    s_mov_b32 s4, s6966; SI-NEXT:    s_mov_b32 s5, s6967; SI-NEXT:    buffer_atomic_add v2, v[0:1], s[4:7], 0 addr64 offset:16 glc968; SI-NEXT:    s_waitcnt vmcnt(0)969; SI-NEXT:    buffer_wbinvl1970; SI-NEXT:    v_mov_b32_e32 v0, v2971; SI-NEXT:    s_waitcnt expcnt(0)972; SI-NEXT:    s_setpc_b64 s[30:31]973;974; VI-LABEL: global_atomic_add_i32_ret_offset:975; VI:       ; %bb.0:976; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)977; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0978; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc979; VI-NEXT:    flat_atomic_add v0, v[0:1], v2 glc980; VI-NEXT:    s_waitcnt vmcnt(0)981; VI-NEXT:    buffer_wbinvl1_vol982; VI-NEXT:    s_setpc_b64 s[30:31]983;984; GFX9-LABEL: global_atomic_add_i32_ret_offset:985; GFX9:       ; %bb.0:986; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)987; GFX9-NEXT:    global_atomic_add v0, v[0:1], v2, off offset:16 glc988; GFX9-NEXT:    s_waitcnt vmcnt(0)989; GFX9-NEXT:    buffer_wbinvl1_vol990; GFX9-NEXT:    s_setpc_b64 s[30:31]991  %gep = getelementptr i32, ptr addrspace(1) %out, i32 4992  %result = atomicrmw add ptr addrspace(1) %gep, i32 %in seq_cst993  ret i32 %result994}995 996define amdgpu_gfx void @global_atomic_add_i32_noret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {997; SI-LABEL: global_atomic_add_i32_noret_scalar:998; SI:       ; %bb.0:999; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1000; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -11001; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill1002; SI-NEXT:    s_mov_b64 exec, s[34:35]1003; SI-NEXT:    s_waitcnt expcnt(0)1004; SI-NEXT:    v_writelane_b32 v1, s6, 01005; SI-NEXT:    v_writelane_b32 v1, s7, 11006; SI-NEXT:    s_mov_b32 s34, s61007; SI-NEXT:    s_mov_b32 s7, 0xf0001008; SI-NEXT:    s_mov_b32 s6, -11009; SI-NEXT:    v_mov_b32_e32 v0, s341010; SI-NEXT:    s_waitcnt vmcnt(0)1011; SI-NEXT:    buffer_atomic_add v0, off, s[4:7], 01012; SI-NEXT:    s_waitcnt vmcnt(0)1013; SI-NEXT:    buffer_wbinvl11014; SI-NEXT:    v_readlane_b32 s7, v1, 11015; SI-NEXT:    v_readlane_b32 s6, v1, 01016; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -11017; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload1018; SI-NEXT:    s_mov_b64 exec, s[34:35]1019; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)1020; SI-NEXT:    s_setpc_b64 s[30:31]1021;1022; VI-LABEL: global_atomic_add_i32_noret_scalar:1023; VI:       ; %bb.0:1024; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1025; VI-NEXT:    v_mov_b32_e32 v0, s41026; VI-NEXT:    v_mov_b32_e32 v1, s51027; VI-NEXT:    v_mov_b32_e32 v2, s61028; VI-NEXT:    flat_atomic_add v[0:1], v21029; VI-NEXT:    s_waitcnt vmcnt(0)1030; VI-NEXT:    buffer_wbinvl1_vol1031; VI-NEXT:    s_setpc_b64 s[30:31]1032;1033; GFX9-LABEL: global_atomic_add_i32_noret_scalar:1034; GFX9:       ; %bb.0:1035; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1036; GFX9-NEXT:    v_mov_b32_e32 v0, 01037; GFX9-NEXT:    v_mov_b32_e32 v1, s61038; GFX9-NEXT:    global_atomic_add v0, v1, s[4:5]1039; GFX9-NEXT:    s_waitcnt vmcnt(0)1040; GFX9-NEXT:    buffer_wbinvl1_vol1041; GFX9-NEXT:    s_setpc_b64 s[30:31]1042  %tmp0 = atomicrmw add ptr addrspace(1) %ptr, i32 %in seq_cst1043  ret void1044}1045 1046define amdgpu_gfx void @global_atomic_add_i32_noret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {1047; SI-LABEL: global_atomic_add_i32_noret_offset_scalar:1048; SI:       ; %bb.0:1049; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1050; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -11051; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill1052; SI-NEXT:    s_mov_b64 exec, s[34:35]1053; SI-NEXT:    s_waitcnt expcnt(0)1054; SI-NEXT:    v_writelane_b32 v1, s6, 01055; SI-NEXT:    v_writelane_b32 v1, s7, 11056; SI-NEXT:    s_mov_b32 s34, s61057; SI-NEXT:    s_mov_b32 s7, 0xf0001058; SI-NEXT:    s_mov_b32 s6, -11059; SI-NEXT:    v_mov_b32_e32 v0, s341060; SI-NEXT:    s_waitcnt vmcnt(0)1061; SI-NEXT:    buffer_atomic_add v0, off, s[4:7], 0 offset:161062; SI-NEXT:    s_waitcnt vmcnt(0)1063; SI-NEXT:    buffer_wbinvl11064; SI-NEXT:    v_readlane_b32 s7, v1, 11065; SI-NEXT:    v_readlane_b32 s6, v1, 01066; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -11067; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload1068; SI-NEXT:    s_mov_b64 exec, s[34:35]1069; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)1070; SI-NEXT:    s_setpc_b64 s[30:31]1071;1072; VI-LABEL: global_atomic_add_i32_noret_offset_scalar:1073; VI:       ; %bb.0:1074; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1075; VI-NEXT:    s_add_u32 s34, s4, 161076; VI-NEXT:    s_addc_u32 s35, s5, 01077; VI-NEXT:    v_mov_b32_e32 v0, s341078; VI-NEXT:    v_mov_b32_e32 v1, s351079; VI-NEXT:    v_mov_b32_e32 v2, s61080; VI-NEXT:    flat_atomic_add v[0:1], v21081; VI-NEXT:    s_waitcnt vmcnt(0)1082; VI-NEXT:    buffer_wbinvl1_vol1083; VI-NEXT:    s_setpc_b64 s[30:31]1084;1085; GFX9-LABEL: global_atomic_add_i32_noret_offset_scalar:1086; GFX9:       ; %bb.0:1087; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1088; GFX9-NEXT:    v_mov_b32_e32 v0, 01089; GFX9-NEXT:    v_mov_b32_e32 v1, s61090; GFX9-NEXT:    global_atomic_add v0, v1, s[4:5] offset:161091; GFX9-NEXT:    s_waitcnt vmcnt(0)1092; GFX9-NEXT:    buffer_wbinvl1_vol1093; GFX9-NEXT:    s_setpc_b64 s[30:31]1094  %gep = getelementptr i32, ptr addrspace(1) %out, i32 41095  %tmp0 = atomicrmw add ptr addrspace(1) %gep, i32 %in seq_cst1096  ret void1097}1098 1099define amdgpu_gfx i32 @global_atomic_add_i32_ret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {1100; SI-LABEL: global_atomic_add_i32_ret_scalar:1101; SI:       ; %bb.0:1102; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1103; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -11104; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill1105; SI-NEXT:    s_mov_b64 exec, s[34:35]1106; SI-NEXT:    s_waitcnt expcnt(0)1107; SI-NEXT:    v_writelane_b32 v1, s6, 01108; SI-NEXT:    v_writelane_b32 v1, s7, 11109; SI-NEXT:    s_mov_b32 s34, s61110; SI-NEXT:    s_mov_b32 s7, 0xf0001111; SI-NEXT:    s_mov_b32 s6, -11112; SI-NEXT:    v_mov_b32_e32 v0, s341113; SI-NEXT:    s_waitcnt vmcnt(0)1114; SI-NEXT:    buffer_atomic_add v0, off, s[4:7], 0 glc1115; SI-NEXT:    s_waitcnt vmcnt(0)1116; SI-NEXT:    buffer_wbinvl11117; SI-NEXT:    v_readlane_b32 s7, v1, 11118; SI-NEXT:    v_readlane_b32 s6, v1, 01119; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -11120; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload1121; SI-NEXT:    s_mov_b64 exec, s[34:35]1122; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)1123; SI-NEXT:    s_setpc_b64 s[30:31]1124;1125; VI-LABEL: global_atomic_add_i32_ret_scalar:1126; VI:       ; %bb.0:1127; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1128; VI-NEXT:    v_mov_b32_e32 v0, s41129; VI-NEXT:    v_mov_b32_e32 v1, s51130; VI-NEXT:    v_mov_b32_e32 v2, s61131; VI-NEXT:    flat_atomic_add v0, v[0:1], v2 glc1132; VI-NEXT:    s_waitcnt vmcnt(0)1133; VI-NEXT:    buffer_wbinvl1_vol1134; VI-NEXT:    s_setpc_b64 s[30:31]1135;1136; GFX9-LABEL: global_atomic_add_i32_ret_scalar:1137; GFX9:       ; %bb.0:1138; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1139; GFX9-NEXT:    v_mov_b32_e32 v0, 01140; GFX9-NEXT:    v_mov_b32_e32 v1, s61141; GFX9-NEXT:    global_atomic_add v0, v0, v1, s[4:5] glc1142; GFX9-NEXT:    s_waitcnt vmcnt(0)1143; GFX9-NEXT:    buffer_wbinvl1_vol1144; GFX9-NEXT:    s_setpc_b64 s[30:31]1145  %result = atomicrmw add ptr addrspace(1) %ptr, i32 %in seq_cst1146  ret i32 %result1147}1148 1149define amdgpu_gfx i32 @global_atomic_add_i32_ret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {1150; SI-LABEL: global_atomic_add_i32_ret_offset_scalar:1151; SI:       ; %bb.0:1152; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1153; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -11154; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill1155; SI-NEXT:    s_mov_b64 exec, s[34:35]1156; SI-NEXT:    s_waitcnt expcnt(0)1157; SI-NEXT:    v_writelane_b32 v1, s6, 01158; SI-NEXT:    v_writelane_b32 v1, s7, 11159; SI-NEXT:    s_mov_b32 s34, s61160; SI-NEXT:    s_mov_b32 s7, 0xf0001161; SI-NEXT:    s_mov_b32 s6, -11162; SI-NEXT:    v_mov_b32_e32 v0, s341163; SI-NEXT:    s_waitcnt vmcnt(0)1164; SI-NEXT:    buffer_atomic_add v0, off, s[4:7], 0 offset:16 glc1165; SI-NEXT:    s_waitcnt vmcnt(0)1166; SI-NEXT:    buffer_wbinvl11167; SI-NEXT:    v_readlane_b32 s7, v1, 11168; SI-NEXT:    v_readlane_b32 s6, v1, 01169; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -11170; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload1171; SI-NEXT:    s_mov_b64 exec, s[34:35]1172; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)1173; SI-NEXT:    s_setpc_b64 s[30:31]1174;1175; VI-LABEL: global_atomic_add_i32_ret_offset_scalar:1176; VI:       ; %bb.0:1177; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1178; VI-NEXT:    s_add_u32 s34, s4, 161179; VI-NEXT:    s_addc_u32 s35, s5, 01180; VI-NEXT:    v_mov_b32_e32 v0, s341181; VI-NEXT:    v_mov_b32_e32 v1, s351182; VI-NEXT:    v_mov_b32_e32 v2, s61183; VI-NEXT:    flat_atomic_add v0, v[0:1], v2 glc1184; VI-NEXT:    s_waitcnt vmcnt(0)1185; VI-NEXT:    buffer_wbinvl1_vol1186; VI-NEXT:    s_setpc_b64 s[30:31]1187;1188; GFX9-LABEL: global_atomic_add_i32_ret_offset_scalar:1189; GFX9:       ; %bb.0:1190; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1191; GFX9-NEXT:    v_mov_b32_e32 v0, 01192; GFX9-NEXT:    v_mov_b32_e32 v1, s61193; GFX9-NEXT:    global_atomic_add v0, v0, v1, s[4:5] offset:16 glc1194; GFX9-NEXT:    s_waitcnt vmcnt(0)1195; GFX9-NEXT:    buffer_wbinvl1_vol1196; GFX9-NEXT:    s_setpc_b64 s[30:31]1197  %gep = getelementptr i32, ptr addrspace(1) %out, i32 41198  %result = atomicrmw add ptr addrspace(1) %gep, i32 %in seq_cst1199  ret i32 %result1200}1201 1202define void @global_atomic_add_i32_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {1203; SI-LABEL: global_atomic_add_i32_noret_offset__amdgpu_no_remote_memory:1204; SI:       ; %bb.0:1205; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1206; SI-NEXT:    s_mov_b32 s6, 01207; SI-NEXT:    s_mov_b32 s7, 0xf0001208; SI-NEXT:    s_mov_b32 s4, s61209; SI-NEXT:    s_mov_b32 s5, s61210; SI-NEXT:    buffer_atomic_add v2, v[0:1], s[4:7], 0 addr64 offset:161211; SI-NEXT:    s_waitcnt vmcnt(0)1212; SI-NEXT:    buffer_wbinvl11213; SI-NEXT:    s_waitcnt expcnt(0)1214; SI-NEXT:    s_setpc_b64 s[30:31]1215;1216; VI-LABEL: global_atomic_add_i32_noret_offset__amdgpu_no_remote_memory:1217; VI:       ; %bb.0:1218; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1219; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v01220; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc1221; VI-NEXT:    flat_atomic_add v[0:1], v21222; VI-NEXT:    s_waitcnt vmcnt(0)1223; VI-NEXT:    buffer_wbinvl1_vol1224; VI-NEXT:    s_setpc_b64 s[30:31]1225;1226; GFX9-LABEL: global_atomic_add_i32_noret_offset__amdgpu_no_remote_memory:1227; GFX9:       ; %bb.0:1228; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1229; GFX9-NEXT:    global_atomic_add v[0:1], v2, off offset:161230; GFX9-NEXT:    s_waitcnt vmcnt(0)1231; GFX9-NEXT:    buffer_wbinvl1_vol1232; GFX9-NEXT:    s_setpc_b64 s[30:31]1233  %gep = getelementptr i32, ptr addrspace(1) %out, i64 41234  %tmp0 = atomicrmw add ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !01235  ret void1236}1237 1238define i32 @global_atomic_add_i32_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {1239; SI-LABEL: global_atomic_add_i32_ret_offset__amdgpu_no_remote_memory:1240; SI:       ; %bb.0:1241; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1242; SI-NEXT:    s_mov_b32 s6, 01243; SI-NEXT:    s_mov_b32 s7, 0xf0001244; SI-NEXT:    s_mov_b32 s4, s61245; SI-NEXT:    s_mov_b32 s5, s61246; SI-NEXT:    buffer_atomic_add v2, v[0:1], s[4:7], 0 addr64 offset:16 glc1247; SI-NEXT:    s_waitcnt vmcnt(0)1248; SI-NEXT:    buffer_wbinvl11249; SI-NEXT:    v_mov_b32_e32 v0, v21250; SI-NEXT:    s_waitcnt expcnt(0)1251; SI-NEXT:    s_setpc_b64 s[30:31]1252;1253; VI-LABEL: global_atomic_add_i32_ret_offset__amdgpu_no_remote_memory:1254; VI:       ; %bb.0:1255; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1256; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v01257; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc1258; VI-NEXT:    flat_atomic_add v0, v[0:1], v2 glc1259; VI-NEXT:    s_waitcnt vmcnt(0)1260; VI-NEXT:    buffer_wbinvl1_vol1261; VI-NEXT:    s_setpc_b64 s[30:31]1262;1263; GFX9-LABEL: global_atomic_add_i32_ret_offset__amdgpu_no_remote_memory:1264; GFX9:       ; %bb.0:1265; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1266; GFX9-NEXT:    global_atomic_add v0, v[0:1], v2, off offset:16 glc1267; GFX9-NEXT:    s_waitcnt vmcnt(0)1268; GFX9-NEXT:    buffer_wbinvl1_vol1269; GFX9-NEXT:    s_setpc_b64 s[30:31]1270  %gep = getelementptr i32, ptr addrspace(1) %out, i64 41271  %result = atomicrmw add ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !01272  ret i32 %result1273}1274 1275; ---------------------------------------------------------------------1276; atomicrmw sub1277; ---------------------------------------------------------------------1278 1279define void @global_atomic_sub_i32_noret(ptr addrspace(1) %ptr, i32 %in) {1280; SI-LABEL: global_atomic_sub_i32_noret:1281; SI:       ; %bb.0:1282; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1283; SI-NEXT:    s_mov_b32 s6, 01284; SI-NEXT:    s_mov_b32 s7, 0xf0001285; SI-NEXT:    s_mov_b32 s4, s61286; SI-NEXT:    s_mov_b32 s5, s61287; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr641288; SI-NEXT:    s_mov_b64 s[8:9], 01289; SI-NEXT:  .LBB30_1: ; %atomicrmw.start1290; SI-NEXT:    ; =>This Inner Loop Header: Depth=11291; SI-NEXT:    s_waitcnt vmcnt(0)1292; SI-NEXT:    v_sub_i32_e32 v3, vcc, v4, v21293; SI-NEXT:    s_waitcnt expcnt(0)1294; SI-NEXT:    v_mov_b32_e32 v6, v41295; SI-NEXT:    v_mov_b32_e32 v5, v31296; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc1297; SI-NEXT:    s_waitcnt vmcnt(0)1298; SI-NEXT:    buffer_wbinvl11299; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v41300; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]1301; SI-NEXT:    v_mov_b32_e32 v4, v51302; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]1303; SI-NEXT:    s_cbranch_execnz .LBB30_11304; SI-NEXT:  ; %bb.2: ; %atomicrmw.end1305; SI-NEXT:    s_or_b64 exec, exec, s[8:9]1306; SI-NEXT:    s_waitcnt expcnt(0)1307; SI-NEXT:    s_setpc_b64 s[30:31]1308;1309; VI-LABEL: global_atomic_sub_i32_noret:1310; VI:       ; %bb.0:1311; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1312; VI-NEXT:    flat_load_dword v4, v[0:1]1313; VI-NEXT:    s_mov_b64 s[4:5], 01314; VI-NEXT:  .LBB30_1: ; %atomicrmw.start1315; VI-NEXT:    ; =>This Inner Loop Header: Depth=11316; VI-NEXT:    s_waitcnt vmcnt(0)1317; VI-NEXT:    v_sub_u32_e32 v3, vcc, v4, v21318; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc1319; VI-NEXT:    s_waitcnt vmcnt(0)1320; VI-NEXT:    buffer_wbinvl1_vol1321; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v41322; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]1323; VI-NEXT:    v_mov_b32_e32 v4, v31324; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]1325; VI-NEXT:    s_cbranch_execnz .LBB30_11326; VI-NEXT:  ; %bb.2: ; %atomicrmw.end1327; VI-NEXT:    s_or_b64 exec, exec, s[4:5]1328; VI-NEXT:    s_setpc_b64 s[30:31]1329;1330; GFX9-LABEL: global_atomic_sub_i32_noret:1331; GFX9:       ; %bb.0:1332; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1333; GFX9-NEXT:    global_load_dword v4, v[0:1], off1334; GFX9-NEXT:    s_mov_b64 s[4:5], 01335; GFX9-NEXT:  .LBB30_1: ; %atomicrmw.start1336; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=11337; GFX9-NEXT:    s_waitcnt vmcnt(0)1338; GFX9-NEXT:    v_sub_u32_e32 v3, v4, v21339; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc1340; GFX9-NEXT:    s_waitcnt vmcnt(0)1341; GFX9-NEXT:    buffer_wbinvl1_vol1342; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v41343; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]1344; GFX9-NEXT:    v_mov_b32_e32 v4, v31345; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]1346; GFX9-NEXT:    s_cbranch_execnz .LBB30_11347; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end1348; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]1349; GFX9-NEXT:    s_setpc_b64 s[30:31]1350  %tmp0 = atomicrmw sub ptr addrspace(1) %ptr, i32 %in seq_cst1351  ret void1352}1353 1354define void @global_atomic_sub_i32_noret_offset(ptr addrspace(1) %out, i32 %in) {1355; SI-LABEL: global_atomic_sub_i32_noret_offset:1356; SI:       ; %bb.0:1357; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1358; SI-NEXT:    s_mov_b32 s6, 01359; SI-NEXT:    s_mov_b32 s7, 0xf0001360; SI-NEXT:    s_mov_b32 s4, s61361; SI-NEXT:    s_mov_b32 s5, s61362; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:161363; SI-NEXT:    s_mov_b64 s[8:9], 01364; SI-NEXT:  .LBB31_1: ; %atomicrmw.start1365; SI-NEXT:    ; =>This Inner Loop Header: Depth=11366; SI-NEXT:    s_waitcnt vmcnt(0)1367; SI-NEXT:    v_sub_i32_e32 v3, vcc, v4, v21368; SI-NEXT:    s_waitcnt expcnt(0)1369; SI-NEXT:    v_mov_b32_e32 v6, v41370; SI-NEXT:    v_mov_b32_e32 v5, v31371; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:16 glc1372; SI-NEXT:    s_waitcnt vmcnt(0)1373; SI-NEXT:    buffer_wbinvl11374; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v41375; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]1376; SI-NEXT:    v_mov_b32_e32 v4, v51377; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]1378; SI-NEXT:    s_cbranch_execnz .LBB31_11379; SI-NEXT:  ; %bb.2: ; %atomicrmw.end1380; SI-NEXT:    s_or_b64 exec, exec, s[8:9]1381; SI-NEXT:    s_waitcnt expcnt(0)1382; SI-NEXT:    s_setpc_b64 s[30:31]1383;1384; VI-LABEL: global_atomic_sub_i32_noret_offset:1385; VI:       ; %bb.0:1386; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1387; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v01388; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc1389; VI-NEXT:    flat_load_dword v4, v[0:1]1390; VI-NEXT:    s_mov_b64 s[4:5], 01391; VI-NEXT:  .LBB31_1: ; %atomicrmw.start1392; VI-NEXT:    ; =>This Inner Loop Header: Depth=11393; VI-NEXT:    s_waitcnt vmcnt(0)1394; VI-NEXT:    v_sub_u32_e32 v3, vcc, v4, v21395; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc1396; VI-NEXT:    s_waitcnt vmcnt(0)1397; VI-NEXT:    buffer_wbinvl1_vol1398; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v41399; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]1400; VI-NEXT:    v_mov_b32_e32 v4, v31401; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]1402; VI-NEXT:    s_cbranch_execnz .LBB31_11403; VI-NEXT:  ; %bb.2: ; %atomicrmw.end1404; VI-NEXT:    s_or_b64 exec, exec, s[4:5]1405; VI-NEXT:    s_setpc_b64 s[30:31]1406;1407; GFX9-LABEL: global_atomic_sub_i32_noret_offset:1408; GFX9:       ; %bb.0:1409; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1410; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:161411; GFX9-NEXT:    s_mov_b64 s[4:5], 01412; GFX9-NEXT:  .LBB31_1: ; %atomicrmw.start1413; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=11414; GFX9-NEXT:    s_waitcnt vmcnt(0)1415; GFX9-NEXT:    v_sub_u32_e32 v3, v4, v21416; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc1417; GFX9-NEXT:    s_waitcnt vmcnt(0)1418; GFX9-NEXT:    buffer_wbinvl1_vol1419; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v41420; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]1421; GFX9-NEXT:    v_mov_b32_e32 v4, v31422; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]1423; GFX9-NEXT:    s_cbranch_execnz .LBB31_11424; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end1425; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]1426; GFX9-NEXT:    s_setpc_b64 s[30:31]1427  %gep = getelementptr i32, ptr addrspace(1) %out, i32 41428  %tmp0 = atomicrmw sub ptr addrspace(1) %gep, i32 %in seq_cst1429  ret void1430}1431 1432define i32 @global_atomic_sub_i32_ret(ptr addrspace(1) %ptr, i32 %in) {1433; SI-LABEL: global_atomic_sub_i32_ret:1434; SI:       ; %bb.0:1435; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1436; SI-NEXT:    s_mov_b32 s6, 01437; SI-NEXT:    s_mov_b32 s7, 0xf0001438; SI-NEXT:    s_mov_b32 s4, s61439; SI-NEXT:    s_mov_b32 s5, s61440; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr641441; SI-NEXT:    s_mov_b64 s[8:9], 01442; SI-NEXT:  .LBB32_1: ; %atomicrmw.start1443; SI-NEXT:    ; =>This Inner Loop Header: Depth=11444; SI-NEXT:    s_waitcnt vmcnt(0)1445; SI-NEXT:    v_mov_b32_e32 v5, v31446; SI-NEXT:    s_waitcnt expcnt(0)1447; SI-NEXT:    v_sub_i32_e32 v4, vcc, v5, v21448; SI-NEXT:    v_mov_b32_e32 v3, v41449; SI-NEXT:    v_mov_b32_e32 v4, v51450; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc1451; SI-NEXT:    s_waitcnt vmcnt(0)1452; SI-NEXT:    buffer_wbinvl11453; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v51454; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]1455; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]1456; SI-NEXT:    s_cbranch_execnz .LBB32_11457; SI-NEXT:  ; %bb.2: ; %atomicrmw.end1458; SI-NEXT:    s_or_b64 exec, exec, s[8:9]1459; SI-NEXT:    v_mov_b32_e32 v0, v31460; SI-NEXT:    s_waitcnt expcnt(0)1461; SI-NEXT:    s_setpc_b64 s[30:31]1462;1463; VI-LABEL: global_atomic_sub_i32_ret:1464; VI:       ; %bb.0:1465; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1466; VI-NEXT:    flat_load_dword v3, v[0:1]1467; VI-NEXT:    s_mov_b64 s[4:5], 01468; VI-NEXT:  .LBB32_1: ; %atomicrmw.start1469; VI-NEXT:    ; =>This Inner Loop Header: Depth=11470; VI-NEXT:    s_waitcnt vmcnt(0)1471; VI-NEXT:    v_mov_b32_e32 v4, v31472; VI-NEXT:    v_sub_u32_e32 v3, vcc, v4, v21473; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc1474; VI-NEXT:    s_waitcnt vmcnt(0)1475; VI-NEXT:    buffer_wbinvl1_vol1476; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v41477; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]1478; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]1479; VI-NEXT:    s_cbranch_execnz .LBB32_11480; VI-NEXT:  ; %bb.2: ; %atomicrmw.end1481; VI-NEXT:    s_or_b64 exec, exec, s[4:5]1482; VI-NEXT:    v_mov_b32_e32 v0, v31483; VI-NEXT:    s_setpc_b64 s[30:31]1484;1485; GFX9-LABEL: global_atomic_sub_i32_ret:1486; GFX9:       ; %bb.0:1487; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1488; GFX9-NEXT:    global_load_dword v3, v[0:1], off1489; GFX9-NEXT:    s_mov_b64 s[4:5], 01490; GFX9-NEXT:  .LBB32_1: ; %atomicrmw.start1491; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=11492; GFX9-NEXT:    s_waitcnt vmcnt(0)1493; GFX9-NEXT:    v_mov_b32_e32 v4, v31494; GFX9-NEXT:    v_sub_u32_e32 v3, v4, v21495; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc1496; GFX9-NEXT:    s_waitcnt vmcnt(0)1497; GFX9-NEXT:    buffer_wbinvl1_vol1498; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v41499; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]1500; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]1501; GFX9-NEXT:    s_cbranch_execnz .LBB32_11502; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end1503; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]1504; GFX9-NEXT:    v_mov_b32_e32 v0, v31505; GFX9-NEXT:    s_setpc_b64 s[30:31]1506  %result = atomicrmw sub ptr addrspace(1) %ptr, i32 %in seq_cst1507  ret i32 %result1508}1509 1510define i32 @global_atomic_sub_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {1511; SI-LABEL: global_atomic_sub_i32_ret_offset:1512; SI:       ; %bb.0:1513; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1514; SI-NEXT:    s_mov_b32 s6, 01515; SI-NEXT:    s_mov_b32 s7, 0xf0001516; SI-NEXT:    s_mov_b32 s4, s61517; SI-NEXT:    s_mov_b32 s5, s61518; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:161519; SI-NEXT:    s_mov_b64 s[8:9], 01520; SI-NEXT:  .LBB33_1: ; %atomicrmw.start1521; SI-NEXT:    ; =>This Inner Loop Header: Depth=11522; SI-NEXT:    s_waitcnt vmcnt(0)1523; SI-NEXT:    v_mov_b32_e32 v5, v31524; SI-NEXT:    s_waitcnt expcnt(0)1525; SI-NEXT:    v_sub_i32_e32 v4, vcc, v5, v21526; SI-NEXT:    v_mov_b32_e32 v3, v41527; SI-NEXT:    v_mov_b32_e32 v4, v51528; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:16 glc1529; SI-NEXT:    s_waitcnt vmcnt(0)1530; SI-NEXT:    buffer_wbinvl11531; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v51532; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]1533; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]1534; SI-NEXT:    s_cbranch_execnz .LBB33_11535; SI-NEXT:  ; %bb.2: ; %atomicrmw.end1536; SI-NEXT:    s_or_b64 exec, exec, s[8:9]1537; SI-NEXT:    v_mov_b32_e32 v0, v31538; SI-NEXT:    s_waitcnt expcnt(0)1539; SI-NEXT:    s_setpc_b64 s[30:31]1540;1541; VI-LABEL: global_atomic_sub_i32_ret_offset:1542; VI:       ; %bb.0:1543; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1544; VI-NEXT:    v_add_u32_e32 v3, vcc, 16, v01545; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc1546; VI-NEXT:    flat_load_dword v0, v[3:4]1547; VI-NEXT:    s_mov_b64 s[4:5], 01548; VI-NEXT:  .LBB33_1: ; %atomicrmw.start1549; VI-NEXT:    ; =>This Inner Loop Header: Depth=11550; VI-NEXT:    s_waitcnt vmcnt(0)1551; VI-NEXT:    v_mov_b32_e32 v1, v01552; VI-NEXT:    v_sub_u32_e32 v0, vcc, v1, v21553; VI-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc1554; VI-NEXT:    s_waitcnt vmcnt(0)1555; VI-NEXT:    buffer_wbinvl1_vol1556; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v11557; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]1558; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]1559; VI-NEXT:    s_cbranch_execnz .LBB33_11560; VI-NEXT:  ; %bb.2: ; %atomicrmw.end1561; VI-NEXT:    s_or_b64 exec, exec, s[4:5]1562; VI-NEXT:    s_setpc_b64 s[30:31]1563;1564; GFX9-LABEL: global_atomic_sub_i32_ret_offset:1565; GFX9:       ; %bb.0:1566; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1567; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:161568; GFX9-NEXT:    s_mov_b64 s[4:5], 01569; GFX9-NEXT:  .LBB33_1: ; %atomicrmw.start1570; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=11571; GFX9-NEXT:    s_waitcnt vmcnt(0)1572; GFX9-NEXT:    v_mov_b32_e32 v4, v31573; GFX9-NEXT:    v_sub_u32_e32 v3, v4, v21574; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc1575; GFX9-NEXT:    s_waitcnt vmcnt(0)1576; GFX9-NEXT:    buffer_wbinvl1_vol1577; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v41578; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]1579; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]1580; GFX9-NEXT:    s_cbranch_execnz .LBB33_11581; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end1582; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]1583; GFX9-NEXT:    v_mov_b32_e32 v0, v31584; GFX9-NEXT:    s_setpc_b64 s[30:31]1585  %gep = getelementptr i32, ptr addrspace(1) %out, i32 41586  %result = atomicrmw sub ptr addrspace(1) %gep, i32 %in seq_cst1587  ret i32 %result1588}1589 1590define amdgpu_gfx void @global_atomic_sub_i32_noret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {1591; SI-LABEL: global_atomic_sub_i32_noret_scalar:1592; SI:       ; %bb.0:1593; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1594; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -11595; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill1596; SI-NEXT:    s_mov_b64 exec, s[34:35]1597; SI-NEXT:    s_waitcnt expcnt(0)1598; SI-NEXT:    v_writelane_b32 v4, s6, 01599; SI-NEXT:    v_writelane_b32 v4, s7, 11600; SI-NEXT:    s_mov_b32 s34, s61601; SI-NEXT:    s_mov_b32 s7, 0xf0001602; SI-NEXT:    s_mov_b32 s6, -11603; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 01604; SI-NEXT:    s_mov_b64 s[36:37], 01605; SI-NEXT:  .LBB34_1: ; %atomicrmw.start1606; SI-NEXT:    ; =>This Inner Loop Header: Depth=11607; SI-NEXT:    s_waitcnt vmcnt(0)1608; SI-NEXT:    v_subrev_i32_e32 v0, vcc, s34, v11609; SI-NEXT:    s_waitcnt expcnt(0)1610; SI-NEXT:    v_mov_b32_e32 v3, v11611; SI-NEXT:    v_mov_b32_e32 v2, v01612; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc1613; SI-NEXT:    s_waitcnt vmcnt(0)1614; SI-NEXT:    buffer_wbinvl11615; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v11616; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]1617; SI-NEXT:    v_mov_b32_e32 v1, v21618; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]1619; SI-NEXT:    s_cbranch_execnz .LBB34_11620; SI-NEXT:  ; %bb.2: ; %atomicrmw.end1621; SI-NEXT:    s_or_b64 exec, exec, s[36:37]1622; SI-NEXT:    v_readlane_b32 s7, v4, 11623; SI-NEXT:    v_readlane_b32 s6, v4, 01624; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -11625; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload1626; SI-NEXT:    s_mov_b64 exec, s[34:35]1627; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)1628; SI-NEXT:    s_setpc_b64 s[30:31]1629;1630; VI-LABEL: global_atomic_sub_i32_noret_scalar:1631; VI:       ; %bb.0:1632; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1633; VI-NEXT:    v_mov_b32_e32 v0, s41634; VI-NEXT:    v_mov_b32_e32 v1, s51635; VI-NEXT:    flat_load_dword v3, v[0:1]1636; VI-NEXT:    s_mov_b64 s[34:35], 01637; VI-NEXT:  .LBB34_1: ; %atomicrmw.start1638; VI-NEXT:    ; =>This Inner Loop Header: Depth=11639; VI-NEXT:    s_waitcnt vmcnt(0)1640; VI-NEXT:    v_subrev_u32_e32 v2, vcc, s6, v31641; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc1642; VI-NEXT:    s_waitcnt vmcnt(0)1643; VI-NEXT:    buffer_wbinvl1_vol1644; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v31645; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]1646; VI-NEXT:    v_mov_b32_e32 v3, v21647; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]1648; VI-NEXT:    s_cbranch_execnz .LBB34_11649; VI-NEXT:  ; %bb.2: ; %atomicrmw.end1650; VI-NEXT:    s_or_b64 exec, exec, s[34:35]1651; VI-NEXT:    s_setpc_b64 s[30:31]1652;1653; GFX9-LABEL: global_atomic_sub_i32_noret_scalar:1654; GFX9:       ; %bb.0:1655; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1656; GFX9-NEXT:    v_mov_b32_e32 v2, 01657; GFX9-NEXT:    global_load_dword v1, v2, s[4:5]1658; GFX9-NEXT:    s_mov_b64 s[34:35], 01659; GFX9-NEXT:  .LBB34_1: ; %atomicrmw.start1660; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=11661; GFX9-NEXT:    s_waitcnt vmcnt(0)1662; GFX9-NEXT:    v_subrev_u32_e32 v0, s6, v11663; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] glc1664; GFX9-NEXT:    s_waitcnt vmcnt(0)1665; GFX9-NEXT:    buffer_wbinvl1_vol1666; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v11667; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]1668; GFX9-NEXT:    v_mov_b32_e32 v1, v01669; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]1670; GFX9-NEXT:    s_cbranch_execnz .LBB34_11671; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end1672; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]1673; GFX9-NEXT:    s_setpc_b64 s[30:31]1674  %tmp0 = atomicrmw sub ptr addrspace(1) %ptr, i32 %in seq_cst1675  ret void1676}1677 1678define amdgpu_gfx void @global_atomic_sub_i32_noret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {1679; SI-LABEL: global_atomic_sub_i32_noret_offset_scalar:1680; SI:       ; %bb.0:1681; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1682; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -11683; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill1684; SI-NEXT:    s_mov_b64 exec, s[34:35]1685; SI-NEXT:    s_waitcnt expcnt(0)1686; SI-NEXT:    v_writelane_b32 v4, s6, 01687; SI-NEXT:    v_writelane_b32 v4, s7, 11688; SI-NEXT:    s_mov_b32 s34, s61689; SI-NEXT:    s_mov_b32 s7, 0xf0001690; SI-NEXT:    s_mov_b32 s6, -11691; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:161692; SI-NEXT:    s_mov_b64 s[36:37], 01693; SI-NEXT:  .LBB35_1: ; %atomicrmw.start1694; SI-NEXT:    ; =>This Inner Loop Header: Depth=11695; SI-NEXT:    s_waitcnt vmcnt(0)1696; SI-NEXT:    v_subrev_i32_e32 v0, vcc, s34, v11697; SI-NEXT:    s_waitcnt expcnt(0)1698; SI-NEXT:    v_mov_b32_e32 v3, v11699; SI-NEXT:    v_mov_b32_e32 v2, v01700; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 offset:16 glc1701; SI-NEXT:    s_waitcnt vmcnt(0)1702; SI-NEXT:    buffer_wbinvl11703; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v11704; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]1705; SI-NEXT:    v_mov_b32_e32 v1, v21706; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]1707; SI-NEXT:    s_cbranch_execnz .LBB35_11708; SI-NEXT:  ; %bb.2: ; %atomicrmw.end1709; SI-NEXT:    s_or_b64 exec, exec, s[36:37]1710; SI-NEXT:    v_readlane_b32 s7, v4, 11711; SI-NEXT:    v_readlane_b32 s6, v4, 01712; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -11713; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload1714; SI-NEXT:    s_mov_b64 exec, s[34:35]1715; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)1716; SI-NEXT:    s_setpc_b64 s[30:31]1717;1718; VI-LABEL: global_atomic_sub_i32_noret_offset_scalar:1719; VI:       ; %bb.0:1720; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1721; VI-NEXT:    s_add_u32 s34, s4, 161722; VI-NEXT:    s_addc_u32 s35, s5, 01723; VI-NEXT:    v_mov_b32_e32 v0, s341724; VI-NEXT:    v_mov_b32_e32 v1, s351725; VI-NEXT:    flat_load_dword v3, v[0:1]1726; VI-NEXT:    s_mov_b64 s[34:35], 01727; VI-NEXT:  .LBB35_1: ; %atomicrmw.start1728; VI-NEXT:    ; =>This Inner Loop Header: Depth=11729; VI-NEXT:    s_waitcnt vmcnt(0)1730; VI-NEXT:    v_subrev_u32_e32 v2, vcc, s6, v31731; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc1732; VI-NEXT:    s_waitcnt vmcnt(0)1733; VI-NEXT:    buffer_wbinvl1_vol1734; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v31735; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]1736; VI-NEXT:    v_mov_b32_e32 v3, v21737; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]1738; VI-NEXT:    s_cbranch_execnz .LBB35_11739; VI-NEXT:  ; %bb.2: ; %atomicrmw.end1740; VI-NEXT:    s_or_b64 exec, exec, s[34:35]1741; VI-NEXT:    s_setpc_b64 s[30:31]1742;1743; GFX9-LABEL: global_atomic_sub_i32_noret_offset_scalar:1744; GFX9:       ; %bb.0:1745; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1746; GFX9-NEXT:    v_mov_b32_e32 v2, 01747; GFX9-NEXT:    global_load_dword v1, v2, s[4:5] offset:161748; GFX9-NEXT:    s_mov_b64 s[34:35], 01749; GFX9-NEXT:  .LBB35_1: ; %atomicrmw.start1750; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=11751; GFX9-NEXT:    s_waitcnt vmcnt(0)1752; GFX9-NEXT:    v_subrev_u32_e32 v0, s6, v11753; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 glc1754; GFX9-NEXT:    s_waitcnt vmcnt(0)1755; GFX9-NEXT:    buffer_wbinvl1_vol1756; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v11757; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]1758; GFX9-NEXT:    v_mov_b32_e32 v1, v01759; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]1760; GFX9-NEXT:    s_cbranch_execnz .LBB35_11761; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end1762; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]1763; GFX9-NEXT:    s_setpc_b64 s[30:31]1764  %gep = getelementptr i32, ptr addrspace(1) %out, i32 41765  %tmp0 = atomicrmw sub ptr addrspace(1) %gep, i32 %in seq_cst1766  ret void1767}1768 1769define amdgpu_gfx i32 @global_atomic_sub_i32_ret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {1770; SI-LABEL: global_atomic_sub_i32_ret_scalar:1771; SI:       ; %bb.0:1772; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1773; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -11774; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill1775; SI-NEXT:    s_mov_b64 exec, s[34:35]1776; SI-NEXT:    s_waitcnt expcnt(0)1777; SI-NEXT:    v_writelane_b32 v3, s6, 01778; SI-NEXT:    v_writelane_b32 v3, s7, 11779; SI-NEXT:    s_mov_b32 s34, s61780; SI-NEXT:    s_mov_b32 s7, 0xf0001781; SI-NEXT:    s_mov_b32 s6, -11782; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 01783; SI-NEXT:    s_mov_b64 s[36:37], 01784; SI-NEXT:  .LBB36_1: ; %atomicrmw.start1785; SI-NEXT:    ; =>This Inner Loop Header: Depth=11786; SI-NEXT:    s_waitcnt vmcnt(0)1787; SI-NEXT:    v_mov_b32_e32 v2, v01788; SI-NEXT:    s_waitcnt expcnt(0)1789; SI-NEXT:    v_subrev_i32_e32 v1, vcc, s34, v21790; SI-NEXT:    v_mov_b32_e32 v0, v11791; SI-NEXT:    v_mov_b32_e32 v1, v21792; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc1793; SI-NEXT:    s_waitcnt vmcnt(0)1794; SI-NEXT:    buffer_wbinvl11795; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v21796; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]1797; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]1798; SI-NEXT:    s_cbranch_execnz .LBB36_11799; SI-NEXT:  ; %bb.2: ; %atomicrmw.end1800; SI-NEXT:    s_or_b64 exec, exec, s[36:37]1801; SI-NEXT:    v_readlane_b32 s7, v3, 11802; SI-NEXT:    v_readlane_b32 s6, v3, 01803; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -11804; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload1805; SI-NEXT:    s_mov_b64 exec, s[34:35]1806; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)1807; SI-NEXT:    s_setpc_b64 s[30:31]1808;1809; VI-LABEL: global_atomic_sub_i32_ret_scalar:1810; VI:       ; %bb.0:1811; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1812; VI-NEXT:    v_mov_b32_e32 v0, s41813; VI-NEXT:    v_mov_b32_e32 v1, s51814; VI-NEXT:    flat_load_dword v0, v[0:1]1815; VI-NEXT:    v_mov_b32_e32 v1, s41816; VI-NEXT:    s_mov_b64 s[34:35], 01817; VI-NEXT:    v_mov_b32_e32 v2, s51818; VI-NEXT:  .LBB36_1: ; %atomicrmw.start1819; VI-NEXT:    ; =>This Inner Loop Header: Depth=11820; VI-NEXT:    s_waitcnt vmcnt(0)1821; VI-NEXT:    v_mov_b32_e32 v4, v01822; VI-NEXT:    v_subrev_u32_e32 v3, vcc, s6, v41823; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc1824; VI-NEXT:    s_waitcnt vmcnt(0)1825; VI-NEXT:    buffer_wbinvl1_vol1826; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v41827; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]1828; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]1829; VI-NEXT:    s_cbranch_execnz .LBB36_11830; VI-NEXT:  ; %bb.2: ; %atomicrmw.end1831; VI-NEXT:    s_or_b64 exec, exec, s[34:35]1832; VI-NEXT:    s_setpc_b64 s[30:31]1833;1834; GFX9-LABEL: global_atomic_sub_i32_ret_scalar:1835; GFX9:       ; %bb.0:1836; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1837; GFX9-NEXT:    v_mov_b32_e32 v1, 01838; GFX9-NEXT:    global_load_dword v0, v1, s[4:5]1839; GFX9-NEXT:    s_mov_b64 s[34:35], 01840; GFX9-NEXT:  .LBB36_1: ; %atomicrmw.start1841; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=11842; GFX9-NEXT:    s_waitcnt vmcnt(0)1843; GFX9-NEXT:    v_mov_b32_e32 v3, v01844; GFX9-NEXT:    v_subrev_u32_e32 v2, s6, v31845; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] glc1846; GFX9-NEXT:    s_waitcnt vmcnt(0)1847; GFX9-NEXT:    buffer_wbinvl1_vol1848; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v31849; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]1850; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]1851; GFX9-NEXT:    s_cbranch_execnz .LBB36_11852; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end1853; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]1854; GFX9-NEXT:    s_setpc_b64 s[30:31]1855  %result = atomicrmw sub ptr addrspace(1) %ptr, i32 %in seq_cst1856  ret i32 %result1857}1858 1859define amdgpu_gfx i32 @global_atomic_sub_i32_ret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {1860; SI-LABEL: global_atomic_sub_i32_ret_offset_scalar:1861; SI:       ; %bb.0:1862; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1863; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -11864; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill1865; SI-NEXT:    s_mov_b64 exec, s[34:35]1866; SI-NEXT:    s_waitcnt expcnt(0)1867; SI-NEXT:    v_writelane_b32 v3, s6, 01868; SI-NEXT:    v_writelane_b32 v3, s7, 11869; SI-NEXT:    s_mov_b32 s34, s61870; SI-NEXT:    s_mov_b32 s7, 0xf0001871; SI-NEXT:    s_mov_b32 s6, -11872; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:161873; SI-NEXT:    s_mov_b64 s[36:37], 01874; SI-NEXT:  .LBB37_1: ; %atomicrmw.start1875; SI-NEXT:    ; =>This Inner Loop Header: Depth=11876; SI-NEXT:    s_waitcnt vmcnt(0)1877; SI-NEXT:    v_mov_b32_e32 v2, v01878; SI-NEXT:    s_waitcnt expcnt(0)1879; SI-NEXT:    v_subrev_i32_e32 v1, vcc, s34, v21880; SI-NEXT:    v_mov_b32_e32 v0, v11881; SI-NEXT:    v_mov_b32_e32 v1, v21882; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc1883; SI-NEXT:    s_waitcnt vmcnt(0)1884; SI-NEXT:    buffer_wbinvl11885; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v21886; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]1887; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]1888; SI-NEXT:    s_cbranch_execnz .LBB37_11889; SI-NEXT:  ; %bb.2: ; %atomicrmw.end1890; SI-NEXT:    s_or_b64 exec, exec, s[36:37]1891; SI-NEXT:    v_readlane_b32 s7, v3, 11892; SI-NEXT:    v_readlane_b32 s6, v3, 01893; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -11894; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload1895; SI-NEXT:    s_mov_b64 exec, s[34:35]1896; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)1897; SI-NEXT:    s_setpc_b64 s[30:31]1898;1899; VI-LABEL: global_atomic_sub_i32_ret_offset_scalar:1900; VI:       ; %bb.0:1901; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1902; VI-NEXT:    s_add_u32 s34, s4, 161903; VI-NEXT:    s_addc_u32 s35, s5, 01904; VI-NEXT:    v_mov_b32_e32 v1, s341905; VI-NEXT:    v_mov_b32_e32 v2, s351906; VI-NEXT:    flat_load_dword v0, v[1:2]1907; VI-NEXT:    s_mov_b64 s[34:35], 01908; VI-NEXT:  .LBB37_1: ; %atomicrmw.start1909; VI-NEXT:    ; =>This Inner Loop Header: Depth=11910; VI-NEXT:    s_waitcnt vmcnt(0)1911; VI-NEXT:    v_mov_b32_e32 v4, v01912; VI-NEXT:    v_subrev_u32_e32 v3, vcc, s6, v41913; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc1914; VI-NEXT:    s_waitcnt vmcnt(0)1915; VI-NEXT:    buffer_wbinvl1_vol1916; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v41917; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]1918; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]1919; VI-NEXT:    s_cbranch_execnz .LBB37_11920; VI-NEXT:  ; %bb.2: ; %atomicrmw.end1921; VI-NEXT:    s_or_b64 exec, exec, s[34:35]1922; VI-NEXT:    s_setpc_b64 s[30:31]1923;1924; GFX9-LABEL: global_atomic_sub_i32_ret_offset_scalar:1925; GFX9:       ; %bb.0:1926; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1927; GFX9-NEXT:    v_mov_b32_e32 v1, 01928; GFX9-NEXT:    global_load_dword v0, v1, s[4:5] offset:161929; GFX9-NEXT:    s_mov_b64 s[34:35], 01930; GFX9-NEXT:  .LBB37_1: ; %atomicrmw.start1931; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=11932; GFX9-NEXT:    s_waitcnt vmcnt(0)1933; GFX9-NEXT:    v_mov_b32_e32 v3, v01934; GFX9-NEXT:    v_subrev_u32_e32 v2, s6, v31935; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] offset:16 glc1936; GFX9-NEXT:    s_waitcnt vmcnt(0)1937; GFX9-NEXT:    buffer_wbinvl1_vol1938; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v31939; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]1940; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]1941; GFX9-NEXT:    s_cbranch_execnz .LBB37_11942; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end1943; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]1944; GFX9-NEXT:    s_setpc_b64 s[30:31]1945  %gep = getelementptr i32, ptr addrspace(1) %out, i32 41946  %result = atomicrmw sub ptr addrspace(1) %gep, i32 %in seq_cst1947  ret i32 %result1948}1949 1950define i32 @global_atomic_sub_0_i32_ret(ptr addrspace(1) %ptr) {1951; SI-LABEL: global_atomic_sub_0_i32_ret:1952; SI:       ; %bb.0:1953; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1954; SI-NEXT:    s_mov_b32 s7, 0xf0001955; SI-NEXT:    s_mov_b32 s6, 01956; SI-NEXT:    v_mov_b32_e32 v2, 01957; SI-NEXT:    s_mov_b32 s4, s61958; SI-NEXT:    s_mov_b32 s5, s61959; SI-NEXT:    buffer_atomic_add v2, v[0:1], s[4:7], 0 addr64 glc1960; SI-NEXT:    s_waitcnt vmcnt(0)1961; SI-NEXT:    buffer_wbinvl11962; SI-NEXT:    v_mov_b32_e32 v0, v21963; SI-NEXT:    s_waitcnt expcnt(0)1964; SI-NEXT:    s_setpc_b64 s[30:31]1965;1966; VI-LABEL: global_atomic_sub_0_i32_ret:1967; VI:       ; %bb.0:1968; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1969; VI-NEXT:    v_mov_b32_e32 v2, 01970; VI-NEXT:    flat_atomic_add v0, v[0:1], v2 glc1971; VI-NEXT:    s_waitcnt vmcnt(0)1972; VI-NEXT:    buffer_wbinvl1_vol1973; VI-NEXT:    s_setpc_b64 s[30:31]1974;1975; GFX9-LABEL: global_atomic_sub_0_i32_ret:1976; GFX9:       ; %bb.0:1977; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1978; GFX9-NEXT:    v_mov_b32_e32 v2, 01979; GFX9-NEXT:    global_atomic_add v0, v[0:1], v2, off glc1980; GFX9-NEXT:    s_waitcnt vmcnt(0)1981; GFX9-NEXT:    buffer_wbinvl1_vol1982; GFX9-NEXT:    s_setpc_b64 s[30:31]1983  %result = atomicrmw sub ptr addrspace(1) %ptr, i32 0 seq_cst1984  ret i32 %result1985}1986 1987define void @global_atomic_sub_i32_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {1988; SI-LABEL: global_atomic_sub_i32_noret_offset__amdgpu_no_remote_memory:1989; SI:       ; %bb.0:1990; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1991; SI-NEXT:    s_mov_b32 s6, 01992; SI-NEXT:    s_mov_b32 s7, 0xf0001993; SI-NEXT:    s_mov_b32 s4, s61994; SI-NEXT:    s_mov_b32 s5, s61995; SI-NEXT:    buffer_atomic_sub v2, v[0:1], s[4:7], 0 addr64 offset:161996; SI-NEXT:    s_waitcnt vmcnt(0)1997; SI-NEXT:    buffer_wbinvl11998; SI-NEXT:    s_waitcnt expcnt(0)1999; SI-NEXT:    s_setpc_b64 s[30:31]2000;2001; VI-LABEL: global_atomic_sub_i32_noret_offset__amdgpu_no_remote_memory:2002; VI:       ; %bb.0:2003; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2004; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v02005; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc2006; VI-NEXT:    flat_atomic_sub v[0:1], v22007; VI-NEXT:    s_waitcnt vmcnt(0)2008; VI-NEXT:    buffer_wbinvl1_vol2009; VI-NEXT:    s_setpc_b64 s[30:31]2010;2011; GFX9-LABEL: global_atomic_sub_i32_noret_offset__amdgpu_no_remote_memory:2012; GFX9:       ; %bb.0:2013; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2014; GFX9-NEXT:    global_atomic_sub v[0:1], v2, off offset:162015; GFX9-NEXT:    s_waitcnt vmcnt(0)2016; GFX9-NEXT:    buffer_wbinvl1_vol2017; GFX9-NEXT:    s_setpc_b64 s[30:31]2018  %gep = getelementptr i32, ptr addrspace(1) %out, i64 42019  %tmp0 = atomicrmw sub ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !02020  ret void2021}2022 2023define i32 @global_atomic_sub_i32_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {2024; SI-LABEL: global_atomic_sub_i32_ret_offset__amdgpu_no_remote_memory:2025; SI:       ; %bb.0:2026; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2027; SI-NEXT:    s_mov_b32 s6, 02028; SI-NEXT:    s_mov_b32 s7, 0xf0002029; SI-NEXT:    s_mov_b32 s4, s62030; SI-NEXT:    s_mov_b32 s5, s62031; SI-NEXT:    buffer_atomic_sub v2, v[0:1], s[4:7], 0 addr64 offset:16 glc2032; SI-NEXT:    s_waitcnt vmcnt(0)2033; SI-NEXT:    buffer_wbinvl12034; SI-NEXT:    v_mov_b32_e32 v0, v22035; SI-NEXT:    s_waitcnt expcnt(0)2036; SI-NEXT:    s_setpc_b64 s[30:31]2037;2038; VI-LABEL: global_atomic_sub_i32_ret_offset__amdgpu_no_remote_memory:2039; VI:       ; %bb.0:2040; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2041; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v02042; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc2043; VI-NEXT:    flat_atomic_sub v0, v[0:1], v2 glc2044; VI-NEXT:    s_waitcnt vmcnt(0)2045; VI-NEXT:    buffer_wbinvl1_vol2046; VI-NEXT:    s_setpc_b64 s[30:31]2047;2048; GFX9-LABEL: global_atomic_sub_i32_ret_offset__amdgpu_no_remote_memory:2049; GFX9:       ; %bb.0:2050; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2051; GFX9-NEXT:    global_atomic_sub v0, v[0:1], v2, off offset:16 glc2052; GFX9-NEXT:    s_waitcnt vmcnt(0)2053; GFX9-NEXT:    buffer_wbinvl1_vol2054; GFX9-NEXT:    s_setpc_b64 s[30:31]2055  %gep = getelementptr i32, ptr addrspace(1) %out, i64 42056  %result = atomicrmw sub ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !02057  ret i32 %result2058}2059 2060; ---------------------------------------------------------------------2061; atomicrmw and2062; ---------------------------------------------------------------------2063 2064define void @global_atomic_and_i32_noret(ptr addrspace(1) %ptr, i32 %in) {2065; SI-LABEL: global_atomic_and_i32_noret:2066; SI:       ; %bb.0:2067; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2068; SI-NEXT:    s_mov_b32 s6, 02069; SI-NEXT:    s_mov_b32 s7, 0xf0002070; SI-NEXT:    s_mov_b32 s4, s62071; SI-NEXT:    s_mov_b32 s5, s62072; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr642073; SI-NEXT:    s_mov_b64 s[8:9], 02074; SI-NEXT:  .LBB41_1: ; %atomicrmw.start2075; SI-NEXT:    ; =>This Inner Loop Header: Depth=12076; SI-NEXT:    s_waitcnt vmcnt(0)2077; SI-NEXT:    v_and_b32_e32 v3, v4, v22078; SI-NEXT:    s_waitcnt expcnt(0)2079; SI-NEXT:    v_mov_b32_e32 v6, v42080; SI-NEXT:    v_mov_b32_e32 v5, v32081; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc2082; SI-NEXT:    s_waitcnt vmcnt(0)2083; SI-NEXT:    buffer_wbinvl12084; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v42085; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]2086; SI-NEXT:    v_mov_b32_e32 v4, v52087; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]2088; SI-NEXT:    s_cbranch_execnz .LBB41_12089; SI-NEXT:  ; %bb.2: ; %atomicrmw.end2090; SI-NEXT:    s_or_b64 exec, exec, s[8:9]2091; SI-NEXT:    s_waitcnt expcnt(0)2092; SI-NEXT:    s_setpc_b64 s[30:31]2093;2094; VI-LABEL: global_atomic_and_i32_noret:2095; VI:       ; %bb.0:2096; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2097; VI-NEXT:    flat_load_dword v4, v[0:1]2098; VI-NEXT:    s_mov_b64 s[4:5], 02099; VI-NEXT:  .LBB41_1: ; %atomicrmw.start2100; VI-NEXT:    ; =>This Inner Loop Header: Depth=12101; VI-NEXT:    s_waitcnt vmcnt(0)2102; VI-NEXT:    v_and_b32_e32 v3, v4, v22103; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc2104; VI-NEXT:    s_waitcnt vmcnt(0)2105; VI-NEXT:    buffer_wbinvl1_vol2106; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v42107; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]2108; VI-NEXT:    v_mov_b32_e32 v4, v32109; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]2110; VI-NEXT:    s_cbranch_execnz .LBB41_12111; VI-NEXT:  ; %bb.2: ; %atomicrmw.end2112; VI-NEXT:    s_or_b64 exec, exec, s[4:5]2113; VI-NEXT:    s_setpc_b64 s[30:31]2114;2115; GFX9-LABEL: global_atomic_and_i32_noret:2116; GFX9:       ; %bb.0:2117; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2118; GFX9-NEXT:    global_load_dword v4, v[0:1], off2119; GFX9-NEXT:    s_mov_b64 s[4:5], 02120; GFX9-NEXT:  .LBB41_1: ; %atomicrmw.start2121; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=12122; GFX9-NEXT:    s_waitcnt vmcnt(0)2123; GFX9-NEXT:    v_and_b32_e32 v3, v4, v22124; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc2125; GFX9-NEXT:    s_waitcnt vmcnt(0)2126; GFX9-NEXT:    buffer_wbinvl1_vol2127; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v42128; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]2129; GFX9-NEXT:    v_mov_b32_e32 v4, v32130; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]2131; GFX9-NEXT:    s_cbranch_execnz .LBB41_12132; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end2133; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]2134; GFX9-NEXT:    s_setpc_b64 s[30:31]2135  %tmp0 = atomicrmw and ptr addrspace(1) %ptr, i32 %in seq_cst2136  ret void2137}2138 2139define void @global_atomic_and_i32_noret_offset(ptr addrspace(1) %out, i32 %in) {2140; SI-LABEL: global_atomic_and_i32_noret_offset:2141; SI:       ; %bb.0:2142; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2143; SI-NEXT:    s_mov_b32 s6, 02144; SI-NEXT:    s_mov_b32 s7, 0xf0002145; SI-NEXT:    s_mov_b32 s4, s62146; SI-NEXT:    s_mov_b32 s5, s62147; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:162148; SI-NEXT:    s_mov_b64 s[8:9], 02149; SI-NEXT:  .LBB42_1: ; %atomicrmw.start2150; SI-NEXT:    ; =>This Inner Loop Header: Depth=12151; SI-NEXT:    s_waitcnt vmcnt(0)2152; SI-NEXT:    v_and_b32_e32 v3, v4, v22153; SI-NEXT:    s_waitcnt expcnt(0)2154; SI-NEXT:    v_mov_b32_e32 v6, v42155; SI-NEXT:    v_mov_b32_e32 v5, v32156; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:16 glc2157; SI-NEXT:    s_waitcnt vmcnt(0)2158; SI-NEXT:    buffer_wbinvl12159; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v42160; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]2161; SI-NEXT:    v_mov_b32_e32 v4, v52162; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]2163; SI-NEXT:    s_cbranch_execnz .LBB42_12164; SI-NEXT:  ; %bb.2: ; %atomicrmw.end2165; SI-NEXT:    s_or_b64 exec, exec, s[8:9]2166; SI-NEXT:    s_waitcnt expcnt(0)2167; SI-NEXT:    s_setpc_b64 s[30:31]2168;2169; VI-LABEL: global_atomic_and_i32_noret_offset:2170; VI:       ; %bb.0:2171; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2172; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v02173; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc2174; VI-NEXT:    flat_load_dword v4, v[0:1]2175; VI-NEXT:    s_mov_b64 s[4:5], 02176; VI-NEXT:  .LBB42_1: ; %atomicrmw.start2177; VI-NEXT:    ; =>This Inner Loop Header: Depth=12178; VI-NEXT:    s_waitcnt vmcnt(0)2179; VI-NEXT:    v_and_b32_e32 v3, v4, v22180; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc2181; VI-NEXT:    s_waitcnt vmcnt(0)2182; VI-NEXT:    buffer_wbinvl1_vol2183; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v42184; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]2185; VI-NEXT:    v_mov_b32_e32 v4, v32186; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]2187; VI-NEXT:    s_cbranch_execnz .LBB42_12188; VI-NEXT:  ; %bb.2: ; %atomicrmw.end2189; VI-NEXT:    s_or_b64 exec, exec, s[4:5]2190; VI-NEXT:    s_setpc_b64 s[30:31]2191;2192; GFX9-LABEL: global_atomic_and_i32_noret_offset:2193; GFX9:       ; %bb.0:2194; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2195; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:162196; GFX9-NEXT:    s_mov_b64 s[4:5], 02197; GFX9-NEXT:  .LBB42_1: ; %atomicrmw.start2198; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=12199; GFX9-NEXT:    s_waitcnt vmcnt(0)2200; GFX9-NEXT:    v_and_b32_e32 v3, v4, v22201; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc2202; GFX9-NEXT:    s_waitcnt vmcnt(0)2203; GFX9-NEXT:    buffer_wbinvl1_vol2204; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v42205; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]2206; GFX9-NEXT:    v_mov_b32_e32 v4, v32207; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]2208; GFX9-NEXT:    s_cbranch_execnz .LBB42_12209; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end2210; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]2211; GFX9-NEXT:    s_setpc_b64 s[30:31]2212  %gep = getelementptr i32, ptr addrspace(1) %out, i32 42213  %tmp0 = atomicrmw and ptr addrspace(1) %gep, i32 %in seq_cst2214  ret void2215}2216 2217define i32 @global_atomic_and_i32_ret(ptr addrspace(1) %ptr, i32 %in) {2218; SI-LABEL: global_atomic_and_i32_ret:2219; SI:       ; %bb.0:2220; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2221; SI-NEXT:    s_mov_b32 s6, 02222; SI-NEXT:    s_mov_b32 s7, 0xf0002223; SI-NEXT:    s_mov_b32 s4, s62224; SI-NEXT:    s_mov_b32 s5, s62225; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr642226; SI-NEXT:    s_mov_b64 s[8:9], 02227; SI-NEXT:  .LBB43_1: ; %atomicrmw.start2228; SI-NEXT:    ; =>This Inner Loop Header: Depth=12229; SI-NEXT:    s_waitcnt vmcnt(0)2230; SI-NEXT:    v_mov_b32_e32 v5, v32231; SI-NEXT:    s_waitcnt expcnt(0)2232; SI-NEXT:    v_and_b32_e32 v4, v5, v22233; SI-NEXT:    v_mov_b32_e32 v3, v42234; SI-NEXT:    v_mov_b32_e32 v4, v52235; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc2236; SI-NEXT:    s_waitcnt vmcnt(0)2237; SI-NEXT:    buffer_wbinvl12238; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v52239; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]2240; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]2241; SI-NEXT:    s_cbranch_execnz .LBB43_12242; SI-NEXT:  ; %bb.2: ; %atomicrmw.end2243; SI-NEXT:    s_or_b64 exec, exec, s[8:9]2244; SI-NEXT:    v_mov_b32_e32 v0, v32245; SI-NEXT:    s_waitcnt expcnt(0)2246; SI-NEXT:    s_setpc_b64 s[30:31]2247;2248; VI-LABEL: global_atomic_and_i32_ret:2249; VI:       ; %bb.0:2250; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2251; VI-NEXT:    flat_load_dword v3, v[0:1]2252; VI-NEXT:    s_mov_b64 s[4:5], 02253; VI-NEXT:  .LBB43_1: ; %atomicrmw.start2254; VI-NEXT:    ; =>This Inner Loop Header: Depth=12255; VI-NEXT:    s_waitcnt vmcnt(0)2256; VI-NEXT:    v_mov_b32_e32 v4, v32257; VI-NEXT:    v_and_b32_e32 v3, v4, v22258; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc2259; VI-NEXT:    s_waitcnt vmcnt(0)2260; VI-NEXT:    buffer_wbinvl1_vol2261; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v42262; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]2263; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]2264; VI-NEXT:    s_cbranch_execnz .LBB43_12265; VI-NEXT:  ; %bb.2: ; %atomicrmw.end2266; VI-NEXT:    s_or_b64 exec, exec, s[4:5]2267; VI-NEXT:    v_mov_b32_e32 v0, v32268; VI-NEXT:    s_setpc_b64 s[30:31]2269;2270; GFX9-LABEL: global_atomic_and_i32_ret:2271; GFX9:       ; %bb.0:2272; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2273; GFX9-NEXT:    global_load_dword v3, v[0:1], off2274; GFX9-NEXT:    s_mov_b64 s[4:5], 02275; GFX9-NEXT:  .LBB43_1: ; %atomicrmw.start2276; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=12277; GFX9-NEXT:    s_waitcnt vmcnt(0)2278; GFX9-NEXT:    v_mov_b32_e32 v4, v32279; GFX9-NEXT:    v_and_b32_e32 v3, v4, v22280; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc2281; GFX9-NEXT:    s_waitcnt vmcnt(0)2282; GFX9-NEXT:    buffer_wbinvl1_vol2283; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v42284; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]2285; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]2286; GFX9-NEXT:    s_cbranch_execnz .LBB43_12287; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end2288; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]2289; GFX9-NEXT:    v_mov_b32_e32 v0, v32290; GFX9-NEXT:    s_setpc_b64 s[30:31]2291  %result = atomicrmw and ptr addrspace(1) %ptr, i32 %in seq_cst2292  ret i32 %result2293}2294 2295define i32 @global_atomic_and_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {2296; SI-LABEL: global_atomic_and_i32_ret_offset:2297; SI:       ; %bb.0:2298; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2299; SI-NEXT:    s_mov_b32 s6, 02300; SI-NEXT:    s_mov_b32 s7, 0xf0002301; SI-NEXT:    s_mov_b32 s4, s62302; SI-NEXT:    s_mov_b32 s5, s62303; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:162304; SI-NEXT:    s_mov_b64 s[8:9], 02305; SI-NEXT:  .LBB44_1: ; %atomicrmw.start2306; SI-NEXT:    ; =>This Inner Loop Header: Depth=12307; SI-NEXT:    s_waitcnt vmcnt(0)2308; SI-NEXT:    v_mov_b32_e32 v5, v32309; SI-NEXT:    s_waitcnt expcnt(0)2310; SI-NEXT:    v_and_b32_e32 v4, v5, v22311; SI-NEXT:    v_mov_b32_e32 v3, v42312; SI-NEXT:    v_mov_b32_e32 v4, v52313; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:16 glc2314; SI-NEXT:    s_waitcnt vmcnt(0)2315; SI-NEXT:    buffer_wbinvl12316; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v52317; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]2318; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]2319; SI-NEXT:    s_cbranch_execnz .LBB44_12320; SI-NEXT:  ; %bb.2: ; %atomicrmw.end2321; SI-NEXT:    s_or_b64 exec, exec, s[8:9]2322; SI-NEXT:    v_mov_b32_e32 v0, v32323; SI-NEXT:    s_waitcnt expcnt(0)2324; SI-NEXT:    s_setpc_b64 s[30:31]2325;2326; VI-LABEL: global_atomic_and_i32_ret_offset:2327; VI:       ; %bb.0:2328; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2329; VI-NEXT:    v_add_u32_e32 v3, vcc, 16, v02330; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc2331; VI-NEXT:    flat_load_dword v0, v[3:4]2332; VI-NEXT:    s_mov_b64 s[4:5], 02333; VI-NEXT:  .LBB44_1: ; %atomicrmw.start2334; VI-NEXT:    ; =>This Inner Loop Header: Depth=12335; VI-NEXT:    s_waitcnt vmcnt(0)2336; VI-NEXT:    v_mov_b32_e32 v1, v02337; VI-NEXT:    v_and_b32_e32 v0, v1, v22338; VI-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc2339; VI-NEXT:    s_waitcnt vmcnt(0)2340; VI-NEXT:    buffer_wbinvl1_vol2341; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v12342; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]2343; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]2344; VI-NEXT:    s_cbranch_execnz .LBB44_12345; VI-NEXT:  ; %bb.2: ; %atomicrmw.end2346; VI-NEXT:    s_or_b64 exec, exec, s[4:5]2347; VI-NEXT:    s_setpc_b64 s[30:31]2348;2349; GFX9-LABEL: global_atomic_and_i32_ret_offset:2350; GFX9:       ; %bb.0:2351; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2352; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:162353; GFX9-NEXT:    s_mov_b64 s[4:5], 02354; GFX9-NEXT:  .LBB44_1: ; %atomicrmw.start2355; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=12356; GFX9-NEXT:    s_waitcnt vmcnt(0)2357; GFX9-NEXT:    v_mov_b32_e32 v4, v32358; GFX9-NEXT:    v_and_b32_e32 v3, v4, v22359; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc2360; GFX9-NEXT:    s_waitcnt vmcnt(0)2361; GFX9-NEXT:    buffer_wbinvl1_vol2362; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v42363; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]2364; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]2365; GFX9-NEXT:    s_cbranch_execnz .LBB44_12366; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end2367; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]2368; GFX9-NEXT:    v_mov_b32_e32 v0, v32369; GFX9-NEXT:    s_setpc_b64 s[30:31]2370  %gep = getelementptr i32, ptr addrspace(1) %out, i32 42371  %result = atomicrmw and ptr addrspace(1) %gep, i32 %in seq_cst2372  ret i32 %result2373}2374 2375define amdgpu_gfx void @global_atomic_and_i32_noret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {2376; SI-LABEL: global_atomic_and_i32_noret_scalar:2377; SI:       ; %bb.0:2378; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2379; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -12380; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill2381; SI-NEXT:    s_mov_b64 exec, s[34:35]2382; SI-NEXT:    s_waitcnt expcnt(0)2383; SI-NEXT:    v_writelane_b32 v4, s6, 02384; SI-NEXT:    v_writelane_b32 v4, s7, 12385; SI-NEXT:    s_mov_b32 s34, s62386; SI-NEXT:    s_mov_b32 s7, 0xf0002387; SI-NEXT:    s_mov_b32 s6, -12388; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 02389; SI-NEXT:    s_mov_b64 s[36:37], 02390; SI-NEXT:  .LBB45_1: ; %atomicrmw.start2391; SI-NEXT:    ; =>This Inner Loop Header: Depth=12392; SI-NEXT:    s_waitcnt vmcnt(0)2393; SI-NEXT:    v_and_b32_e32 v0, s34, v12394; SI-NEXT:    s_waitcnt expcnt(0)2395; SI-NEXT:    v_mov_b32_e32 v3, v12396; SI-NEXT:    v_mov_b32_e32 v2, v02397; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc2398; SI-NEXT:    s_waitcnt vmcnt(0)2399; SI-NEXT:    buffer_wbinvl12400; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v12401; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]2402; SI-NEXT:    v_mov_b32_e32 v1, v22403; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]2404; SI-NEXT:    s_cbranch_execnz .LBB45_12405; SI-NEXT:  ; %bb.2: ; %atomicrmw.end2406; SI-NEXT:    s_or_b64 exec, exec, s[36:37]2407; SI-NEXT:    v_readlane_b32 s7, v4, 12408; SI-NEXT:    v_readlane_b32 s6, v4, 02409; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -12410; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload2411; SI-NEXT:    s_mov_b64 exec, s[34:35]2412; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)2413; SI-NEXT:    s_setpc_b64 s[30:31]2414;2415; VI-LABEL: global_atomic_and_i32_noret_scalar:2416; VI:       ; %bb.0:2417; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2418; VI-NEXT:    v_mov_b32_e32 v0, s42419; VI-NEXT:    v_mov_b32_e32 v1, s52420; VI-NEXT:    flat_load_dword v3, v[0:1]2421; VI-NEXT:    s_mov_b64 s[34:35], 02422; VI-NEXT:  .LBB45_1: ; %atomicrmw.start2423; VI-NEXT:    ; =>This Inner Loop Header: Depth=12424; VI-NEXT:    s_waitcnt vmcnt(0)2425; VI-NEXT:    v_and_b32_e32 v2, s6, v32426; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc2427; VI-NEXT:    s_waitcnt vmcnt(0)2428; VI-NEXT:    buffer_wbinvl1_vol2429; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v32430; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]2431; VI-NEXT:    v_mov_b32_e32 v3, v22432; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]2433; VI-NEXT:    s_cbranch_execnz .LBB45_12434; VI-NEXT:  ; %bb.2: ; %atomicrmw.end2435; VI-NEXT:    s_or_b64 exec, exec, s[34:35]2436; VI-NEXT:    s_setpc_b64 s[30:31]2437;2438; GFX9-LABEL: global_atomic_and_i32_noret_scalar:2439; GFX9:       ; %bb.0:2440; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2441; GFX9-NEXT:    v_mov_b32_e32 v2, 02442; GFX9-NEXT:    global_load_dword v1, v2, s[4:5]2443; GFX9-NEXT:    s_mov_b64 s[34:35], 02444; GFX9-NEXT:  .LBB45_1: ; %atomicrmw.start2445; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=12446; GFX9-NEXT:    s_waitcnt vmcnt(0)2447; GFX9-NEXT:    v_and_b32_e32 v0, s6, v12448; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] glc2449; GFX9-NEXT:    s_waitcnt vmcnt(0)2450; GFX9-NEXT:    buffer_wbinvl1_vol2451; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v12452; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]2453; GFX9-NEXT:    v_mov_b32_e32 v1, v02454; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]2455; GFX9-NEXT:    s_cbranch_execnz .LBB45_12456; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end2457; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]2458; GFX9-NEXT:    s_setpc_b64 s[30:31]2459  %tmp0 = atomicrmw and ptr addrspace(1) %ptr, i32 %in seq_cst2460  ret void2461}2462 2463define amdgpu_gfx void @global_atomic_and_i32_noret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {2464; SI-LABEL: global_atomic_and_i32_noret_offset_scalar:2465; SI:       ; %bb.0:2466; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2467; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -12468; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill2469; SI-NEXT:    s_mov_b64 exec, s[34:35]2470; SI-NEXT:    s_waitcnt expcnt(0)2471; SI-NEXT:    v_writelane_b32 v4, s6, 02472; SI-NEXT:    v_writelane_b32 v4, s7, 12473; SI-NEXT:    s_mov_b32 s34, s62474; SI-NEXT:    s_mov_b32 s7, 0xf0002475; SI-NEXT:    s_mov_b32 s6, -12476; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:162477; SI-NEXT:    s_mov_b64 s[36:37], 02478; SI-NEXT:  .LBB46_1: ; %atomicrmw.start2479; SI-NEXT:    ; =>This Inner Loop Header: Depth=12480; SI-NEXT:    s_waitcnt vmcnt(0)2481; SI-NEXT:    v_and_b32_e32 v0, s34, v12482; SI-NEXT:    s_waitcnt expcnt(0)2483; SI-NEXT:    v_mov_b32_e32 v3, v12484; SI-NEXT:    v_mov_b32_e32 v2, v02485; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 offset:16 glc2486; SI-NEXT:    s_waitcnt vmcnt(0)2487; SI-NEXT:    buffer_wbinvl12488; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v12489; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]2490; SI-NEXT:    v_mov_b32_e32 v1, v22491; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]2492; SI-NEXT:    s_cbranch_execnz .LBB46_12493; SI-NEXT:  ; %bb.2: ; %atomicrmw.end2494; SI-NEXT:    s_or_b64 exec, exec, s[36:37]2495; SI-NEXT:    v_readlane_b32 s7, v4, 12496; SI-NEXT:    v_readlane_b32 s6, v4, 02497; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -12498; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload2499; SI-NEXT:    s_mov_b64 exec, s[34:35]2500; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)2501; SI-NEXT:    s_setpc_b64 s[30:31]2502;2503; VI-LABEL: global_atomic_and_i32_noret_offset_scalar:2504; VI:       ; %bb.0:2505; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2506; VI-NEXT:    s_add_u32 s34, s4, 162507; VI-NEXT:    s_addc_u32 s35, s5, 02508; VI-NEXT:    v_mov_b32_e32 v0, s342509; VI-NEXT:    v_mov_b32_e32 v1, s352510; VI-NEXT:    flat_load_dword v3, v[0:1]2511; VI-NEXT:    s_mov_b64 s[34:35], 02512; VI-NEXT:  .LBB46_1: ; %atomicrmw.start2513; VI-NEXT:    ; =>This Inner Loop Header: Depth=12514; VI-NEXT:    s_waitcnt vmcnt(0)2515; VI-NEXT:    v_and_b32_e32 v2, s6, v32516; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc2517; VI-NEXT:    s_waitcnt vmcnt(0)2518; VI-NEXT:    buffer_wbinvl1_vol2519; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v32520; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]2521; VI-NEXT:    v_mov_b32_e32 v3, v22522; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]2523; VI-NEXT:    s_cbranch_execnz .LBB46_12524; VI-NEXT:  ; %bb.2: ; %atomicrmw.end2525; VI-NEXT:    s_or_b64 exec, exec, s[34:35]2526; VI-NEXT:    s_setpc_b64 s[30:31]2527;2528; GFX9-LABEL: global_atomic_and_i32_noret_offset_scalar:2529; GFX9:       ; %bb.0:2530; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2531; GFX9-NEXT:    v_mov_b32_e32 v2, 02532; GFX9-NEXT:    global_load_dword v1, v2, s[4:5] offset:162533; GFX9-NEXT:    s_mov_b64 s[34:35], 02534; GFX9-NEXT:  .LBB46_1: ; %atomicrmw.start2535; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=12536; GFX9-NEXT:    s_waitcnt vmcnt(0)2537; GFX9-NEXT:    v_and_b32_e32 v0, s6, v12538; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 glc2539; GFX9-NEXT:    s_waitcnt vmcnt(0)2540; GFX9-NEXT:    buffer_wbinvl1_vol2541; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v12542; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]2543; GFX9-NEXT:    v_mov_b32_e32 v1, v02544; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]2545; GFX9-NEXT:    s_cbranch_execnz .LBB46_12546; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end2547; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]2548; GFX9-NEXT:    s_setpc_b64 s[30:31]2549  %gep = getelementptr i32, ptr addrspace(1) %out, i32 42550  %tmp0 = atomicrmw and ptr addrspace(1) %gep, i32 %in seq_cst2551  ret void2552}2553 2554define amdgpu_gfx i32 @global_atomic_and_i32_ret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {2555; SI-LABEL: global_atomic_and_i32_ret_scalar:2556; SI:       ; %bb.0:2557; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2558; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -12559; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill2560; SI-NEXT:    s_mov_b64 exec, s[34:35]2561; SI-NEXT:    s_waitcnt expcnt(0)2562; SI-NEXT:    v_writelane_b32 v3, s6, 02563; SI-NEXT:    v_writelane_b32 v3, s7, 12564; SI-NEXT:    s_mov_b32 s34, s62565; SI-NEXT:    s_mov_b32 s7, 0xf0002566; SI-NEXT:    s_mov_b32 s6, -12567; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 02568; SI-NEXT:    s_mov_b64 s[36:37], 02569; SI-NEXT:  .LBB47_1: ; %atomicrmw.start2570; SI-NEXT:    ; =>This Inner Loop Header: Depth=12571; SI-NEXT:    s_waitcnt vmcnt(0)2572; SI-NEXT:    v_mov_b32_e32 v2, v02573; SI-NEXT:    s_waitcnt expcnt(0)2574; SI-NEXT:    v_and_b32_e32 v1, s34, v22575; SI-NEXT:    v_mov_b32_e32 v0, v12576; SI-NEXT:    v_mov_b32_e32 v1, v22577; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc2578; SI-NEXT:    s_waitcnt vmcnt(0)2579; SI-NEXT:    buffer_wbinvl12580; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v22581; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]2582; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]2583; SI-NEXT:    s_cbranch_execnz .LBB47_12584; SI-NEXT:  ; %bb.2: ; %atomicrmw.end2585; SI-NEXT:    s_or_b64 exec, exec, s[36:37]2586; SI-NEXT:    v_readlane_b32 s7, v3, 12587; SI-NEXT:    v_readlane_b32 s6, v3, 02588; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -12589; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload2590; SI-NEXT:    s_mov_b64 exec, s[34:35]2591; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)2592; SI-NEXT:    s_setpc_b64 s[30:31]2593;2594; VI-LABEL: global_atomic_and_i32_ret_scalar:2595; VI:       ; %bb.0:2596; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2597; VI-NEXT:    v_mov_b32_e32 v0, s42598; VI-NEXT:    v_mov_b32_e32 v1, s52599; VI-NEXT:    flat_load_dword v0, v[0:1]2600; VI-NEXT:    v_mov_b32_e32 v1, s42601; VI-NEXT:    s_mov_b64 s[34:35], 02602; VI-NEXT:    v_mov_b32_e32 v2, s52603; VI-NEXT:  .LBB47_1: ; %atomicrmw.start2604; VI-NEXT:    ; =>This Inner Loop Header: Depth=12605; VI-NEXT:    s_waitcnt vmcnt(0)2606; VI-NEXT:    v_mov_b32_e32 v4, v02607; VI-NEXT:    v_and_b32_e32 v3, s6, v42608; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc2609; VI-NEXT:    s_waitcnt vmcnt(0)2610; VI-NEXT:    buffer_wbinvl1_vol2611; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v42612; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]2613; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]2614; VI-NEXT:    s_cbranch_execnz .LBB47_12615; VI-NEXT:  ; %bb.2: ; %atomicrmw.end2616; VI-NEXT:    s_or_b64 exec, exec, s[34:35]2617; VI-NEXT:    s_setpc_b64 s[30:31]2618;2619; GFX9-LABEL: global_atomic_and_i32_ret_scalar:2620; GFX9:       ; %bb.0:2621; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2622; GFX9-NEXT:    v_mov_b32_e32 v1, 02623; GFX9-NEXT:    global_load_dword v0, v1, s[4:5]2624; GFX9-NEXT:    s_mov_b64 s[34:35], 02625; GFX9-NEXT:  .LBB47_1: ; %atomicrmw.start2626; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=12627; GFX9-NEXT:    s_waitcnt vmcnt(0)2628; GFX9-NEXT:    v_mov_b32_e32 v3, v02629; GFX9-NEXT:    v_and_b32_e32 v2, s6, v32630; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] glc2631; GFX9-NEXT:    s_waitcnt vmcnt(0)2632; GFX9-NEXT:    buffer_wbinvl1_vol2633; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v32634; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]2635; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]2636; GFX9-NEXT:    s_cbranch_execnz .LBB47_12637; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end2638; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]2639; GFX9-NEXT:    s_setpc_b64 s[30:31]2640  %result = atomicrmw and ptr addrspace(1) %ptr, i32 %in seq_cst2641  ret i32 %result2642}2643 2644define amdgpu_gfx i32 @global_atomic_and_i32_ret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {2645; SI-LABEL: global_atomic_and_i32_ret_offset_scalar:2646; SI:       ; %bb.0:2647; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2648; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -12649; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill2650; SI-NEXT:    s_mov_b64 exec, s[34:35]2651; SI-NEXT:    s_waitcnt expcnt(0)2652; SI-NEXT:    v_writelane_b32 v3, s6, 02653; SI-NEXT:    v_writelane_b32 v3, s7, 12654; SI-NEXT:    s_mov_b32 s34, s62655; SI-NEXT:    s_mov_b32 s7, 0xf0002656; SI-NEXT:    s_mov_b32 s6, -12657; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:162658; SI-NEXT:    s_mov_b64 s[36:37], 02659; SI-NEXT:  .LBB48_1: ; %atomicrmw.start2660; SI-NEXT:    ; =>This Inner Loop Header: Depth=12661; SI-NEXT:    s_waitcnt vmcnt(0)2662; SI-NEXT:    v_mov_b32_e32 v2, v02663; SI-NEXT:    s_waitcnt expcnt(0)2664; SI-NEXT:    v_and_b32_e32 v1, s34, v22665; SI-NEXT:    v_mov_b32_e32 v0, v12666; SI-NEXT:    v_mov_b32_e32 v1, v22667; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc2668; SI-NEXT:    s_waitcnt vmcnt(0)2669; SI-NEXT:    buffer_wbinvl12670; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v22671; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]2672; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]2673; SI-NEXT:    s_cbranch_execnz .LBB48_12674; SI-NEXT:  ; %bb.2: ; %atomicrmw.end2675; SI-NEXT:    s_or_b64 exec, exec, s[36:37]2676; SI-NEXT:    v_readlane_b32 s7, v3, 12677; SI-NEXT:    v_readlane_b32 s6, v3, 02678; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -12679; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload2680; SI-NEXT:    s_mov_b64 exec, s[34:35]2681; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)2682; SI-NEXT:    s_setpc_b64 s[30:31]2683;2684; VI-LABEL: global_atomic_and_i32_ret_offset_scalar:2685; VI:       ; %bb.0:2686; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2687; VI-NEXT:    s_add_u32 s34, s4, 162688; VI-NEXT:    s_addc_u32 s35, s5, 02689; VI-NEXT:    v_mov_b32_e32 v1, s342690; VI-NEXT:    v_mov_b32_e32 v2, s352691; VI-NEXT:    flat_load_dword v0, v[1:2]2692; VI-NEXT:    s_mov_b64 s[34:35], 02693; VI-NEXT:  .LBB48_1: ; %atomicrmw.start2694; VI-NEXT:    ; =>This Inner Loop Header: Depth=12695; VI-NEXT:    s_waitcnt vmcnt(0)2696; VI-NEXT:    v_mov_b32_e32 v4, v02697; VI-NEXT:    v_and_b32_e32 v3, s6, v42698; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc2699; VI-NEXT:    s_waitcnt vmcnt(0)2700; VI-NEXT:    buffer_wbinvl1_vol2701; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v42702; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]2703; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]2704; VI-NEXT:    s_cbranch_execnz .LBB48_12705; VI-NEXT:  ; %bb.2: ; %atomicrmw.end2706; VI-NEXT:    s_or_b64 exec, exec, s[34:35]2707; VI-NEXT:    s_setpc_b64 s[30:31]2708;2709; GFX9-LABEL: global_atomic_and_i32_ret_offset_scalar:2710; GFX9:       ; %bb.0:2711; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2712; GFX9-NEXT:    v_mov_b32_e32 v1, 02713; GFX9-NEXT:    global_load_dword v0, v1, s[4:5] offset:162714; GFX9-NEXT:    s_mov_b64 s[34:35], 02715; GFX9-NEXT:  .LBB48_1: ; %atomicrmw.start2716; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=12717; GFX9-NEXT:    s_waitcnt vmcnt(0)2718; GFX9-NEXT:    v_mov_b32_e32 v3, v02719; GFX9-NEXT:    v_and_b32_e32 v2, s6, v32720; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] offset:16 glc2721; GFX9-NEXT:    s_waitcnt vmcnt(0)2722; GFX9-NEXT:    buffer_wbinvl1_vol2723; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v32724; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]2725; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]2726; GFX9-NEXT:    s_cbranch_execnz .LBB48_12727; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end2728; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]2729; GFX9-NEXT:    s_setpc_b64 s[30:31]2730  %gep = getelementptr i32, ptr addrspace(1) %out, i32 42731  %result = atomicrmw and ptr addrspace(1) %gep, i32 %in seq_cst2732  ret i32 %result2733}2734 2735define void @global_atomic_and_i32_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {2736; SI-LABEL: global_atomic_and_i32_noret_offset__amdgpu_no_remote_memory:2737; SI:       ; %bb.0:2738; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2739; SI-NEXT:    s_mov_b32 s6, 02740; SI-NEXT:    s_mov_b32 s7, 0xf0002741; SI-NEXT:    s_mov_b32 s4, s62742; SI-NEXT:    s_mov_b32 s5, s62743; SI-NEXT:    buffer_atomic_and v2, v[0:1], s[4:7], 0 addr64 offset:162744; SI-NEXT:    s_waitcnt vmcnt(0)2745; SI-NEXT:    buffer_wbinvl12746; SI-NEXT:    s_waitcnt expcnt(0)2747; SI-NEXT:    s_setpc_b64 s[30:31]2748;2749; VI-LABEL: global_atomic_and_i32_noret_offset__amdgpu_no_remote_memory:2750; VI:       ; %bb.0:2751; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2752; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v02753; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc2754; VI-NEXT:    flat_atomic_and v[0:1], v22755; VI-NEXT:    s_waitcnt vmcnt(0)2756; VI-NEXT:    buffer_wbinvl1_vol2757; VI-NEXT:    s_setpc_b64 s[30:31]2758;2759; GFX9-LABEL: global_atomic_and_i32_noret_offset__amdgpu_no_remote_memory:2760; GFX9:       ; %bb.0:2761; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2762; GFX9-NEXT:    global_atomic_and v[0:1], v2, off offset:162763; GFX9-NEXT:    s_waitcnt vmcnt(0)2764; GFX9-NEXT:    buffer_wbinvl1_vol2765; GFX9-NEXT:    s_setpc_b64 s[30:31]2766  %gep = getelementptr i32, ptr addrspace(1) %out, i64 42767  %tmp0 = atomicrmw and ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !02768  ret void2769}2770 2771define i32 @global_atomic_and_i32_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {2772; SI-LABEL: global_atomic_and_i32_ret_offset__amdgpu_no_remote_memory:2773; SI:       ; %bb.0:2774; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2775; SI-NEXT:    s_mov_b32 s6, 02776; SI-NEXT:    s_mov_b32 s7, 0xf0002777; SI-NEXT:    s_mov_b32 s4, s62778; SI-NEXT:    s_mov_b32 s5, s62779; SI-NEXT:    buffer_atomic_and v2, v[0:1], s[4:7], 0 addr64 offset:16 glc2780; SI-NEXT:    s_waitcnt vmcnt(0)2781; SI-NEXT:    buffer_wbinvl12782; SI-NEXT:    v_mov_b32_e32 v0, v22783; SI-NEXT:    s_waitcnt expcnt(0)2784; SI-NEXT:    s_setpc_b64 s[30:31]2785;2786; VI-LABEL: global_atomic_and_i32_ret_offset__amdgpu_no_remote_memory:2787; VI:       ; %bb.0:2788; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2789; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v02790; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc2791; VI-NEXT:    flat_atomic_and v0, v[0:1], v2 glc2792; VI-NEXT:    s_waitcnt vmcnt(0)2793; VI-NEXT:    buffer_wbinvl1_vol2794; VI-NEXT:    s_setpc_b64 s[30:31]2795;2796; GFX9-LABEL: global_atomic_and_i32_ret_offset__amdgpu_no_remote_memory:2797; GFX9:       ; %bb.0:2798; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2799; GFX9-NEXT:    global_atomic_and v0, v[0:1], v2, off offset:16 glc2800; GFX9-NEXT:    s_waitcnt vmcnt(0)2801; GFX9-NEXT:    buffer_wbinvl1_vol2802; GFX9-NEXT:    s_setpc_b64 s[30:31]2803  %gep = getelementptr i32, ptr addrspace(1) %out, i64 42804  %result = atomicrmw and ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !02805  ret i32 %result2806}2807 2808; ---------------------------------------------------------------------2809; atomicrmw nand2810; ---------------------------------------------------------------------2811 2812define void @global_atomic_nand_i32_noret(ptr addrspace(1) %ptr, i32 %in) {2813; SI-LABEL: global_atomic_nand_i32_noret:2814; SI:       ; %bb.0:2815; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2816; SI-NEXT:    s_mov_b32 s6, 02817; SI-NEXT:    s_mov_b32 s7, 0xf0002818; SI-NEXT:    s_mov_b32 s4, s62819; SI-NEXT:    s_mov_b32 s5, s62820; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr642821; SI-NEXT:    s_mov_b64 s[8:9], 02822; SI-NEXT:  .LBB51_1: ; %atomicrmw.start2823; SI-NEXT:    ; =>This Inner Loop Header: Depth=12824; SI-NEXT:    s_waitcnt vmcnt(0)2825; SI-NEXT:    v_and_b32_e32 v3, v4, v22826; SI-NEXT:    v_not_b32_e32 v3, v32827; SI-NEXT:    s_waitcnt expcnt(0)2828; SI-NEXT:    v_mov_b32_e32 v6, v42829; SI-NEXT:    v_mov_b32_e32 v5, v32830; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc2831; SI-NEXT:    s_waitcnt vmcnt(0)2832; SI-NEXT:    buffer_wbinvl12833; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v42834; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]2835; SI-NEXT:    v_mov_b32_e32 v4, v52836; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]2837; SI-NEXT:    s_cbranch_execnz .LBB51_12838; SI-NEXT:  ; %bb.2: ; %atomicrmw.end2839; SI-NEXT:    s_or_b64 exec, exec, s[8:9]2840; SI-NEXT:    s_waitcnt expcnt(0)2841; SI-NEXT:    s_setpc_b64 s[30:31]2842;2843; VI-LABEL: global_atomic_nand_i32_noret:2844; VI:       ; %bb.0:2845; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2846; VI-NEXT:    flat_load_dword v4, v[0:1]2847; VI-NEXT:    s_mov_b64 s[4:5], 02848; VI-NEXT:  .LBB51_1: ; %atomicrmw.start2849; VI-NEXT:    ; =>This Inner Loop Header: Depth=12850; VI-NEXT:    s_waitcnt vmcnt(0)2851; VI-NEXT:    v_and_b32_e32 v3, v4, v22852; VI-NEXT:    v_not_b32_e32 v3, v32853; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc2854; VI-NEXT:    s_waitcnt vmcnt(0)2855; VI-NEXT:    buffer_wbinvl1_vol2856; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v42857; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]2858; VI-NEXT:    v_mov_b32_e32 v4, v32859; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]2860; VI-NEXT:    s_cbranch_execnz .LBB51_12861; VI-NEXT:  ; %bb.2: ; %atomicrmw.end2862; VI-NEXT:    s_or_b64 exec, exec, s[4:5]2863; VI-NEXT:    s_setpc_b64 s[30:31]2864;2865; GFX9-LABEL: global_atomic_nand_i32_noret:2866; GFX9:       ; %bb.0:2867; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2868; GFX9-NEXT:    global_load_dword v4, v[0:1], off2869; GFX9-NEXT:    s_mov_b64 s[4:5], 02870; GFX9-NEXT:  .LBB51_1: ; %atomicrmw.start2871; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=12872; GFX9-NEXT:    s_waitcnt vmcnt(0)2873; GFX9-NEXT:    v_and_b32_e32 v3, v4, v22874; GFX9-NEXT:    v_not_b32_e32 v3, v32875; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc2876; GFX9-NEXT:    s_waitcnt vmcnt(0)2877; GFX9-NEXT:    buffer_wbinvl1_vol2878; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v42879; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]2880; GFX9-NEXT:    v_mov_b32_e32 v4, v32881; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]2882; GFX9-NEXT:    s_cbranch_execnz .LBB51_12883; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end2884; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]2885; GFX9-NEXT:    s_setpc_b64 s[30:31]2886  %tmp0 = atomicrmw nand ptr addrspace(1) %ptr, i32 %in seq_cst2887  ret void2888}2889 2890define void @global_atomic_nand_i32_noret_offset(ptr addrspace(1) %out, i32 %in) {2891; SI-LABEL: global_atomic_nand_i32_noret_offset:2892; SI:       ; %bb.0:2893; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2894; SI-NEXT:    s_mov_b32 s6, 02895; SI-NEXT:    s_mov_b32 s7, 0xf0002896; SI-NEXT:    s_mov_b32 s4, s62897; SI-NEXT:    s_mov_b32 s5, s62898; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:162899; SI-NEXT:    s_mov_b64 s[8:9], 02900; SI-NEXT:  .LBB52_1: ; %atomicrmw.start2901; SI-NEXT:    ; =>This Inner Loop Header: Depth=12902; SI-NEXT:    s_waitcnt vmcnt(0)2903; SI-NEXT:    v_and_b32_e32 v3, v4, v22904; SI-NEXT:    v_not_b32_e32 v3, v32905; SI-NEXT:    s_waitcnt expcnt(0)2906; SI-NEXT:    v_mov_b32_e32 v6, v42907; SI-NEXT:    v_mov_b32_e32 v5, v32908; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:16 glc2909; SI-NEXT:    s_waitcnt vmcnt(0)2910; SI-NEXT:    buffer_wbinvl12911; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v42912; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]2913; SI-NEXT:    v_mov_b32_e32 v4, v52914; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]2915; SI-NEXT:    s_cbranch_execnz .LBB52_12916; SI-NEXT:  ; %bb.2: ; %atomicrmw.end2917; SI-NEXT:    s_or_b64 exec, exec, s[8:9]2918; SI-NEXT:    s_waitcnt expcnt(0)2919; SI-NEXT:    s_setpc_b64 s[30:31]2920;2921; VI-LABEL: global_atomic_nand_i32_noret_offset:2922; VI:       ; %bb.0:2923; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2924; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v02925; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc2926; VI-NEXT:    flat_load_dword v4, v[0:1]2927; VI-NEXT:    s_mov_b64 s[4:5], 02928; VI-NEXT:  .LBB52_1: ; %atomicrmw.start2929; VI-NEXT:    ; =>This Inner Loop Header: Depth=12930; VI-NEXT:    s_waitcnt vmcnt(0)2931; VI-NEXT:    v_and_b32_e32 v3, v4, v22932; VI-NEXT:    v_not_b32_e32 v3, v32933; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc2934; VI-NEXT:    s_waitcnt vmcnt(0)2935; VI-NEXT:    buffer_wbinvl1_vol2936; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v42937; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]2938; VI-NEXT:    v_mov_b32_e32 v4, v32939; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]2940; VI-NEXT:    s_cbranch_execnz .LBB52_12941; VI-NEXT:  ; %bb.2: ; %atomicrmw.end2942; VI-NEXT:    s_or_b64 exec, exec, s[4:5]2943; VI-NEXT:    s_setpc_b64 s[30:31]2944;2945; GFX9-LABEL: global_atomic_nand_i32_noret_offset:2946; GFX9:       ; %bb.0:2947; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2948; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:162949; GFX9-NEXT:    s_mov_b64 s[4:5], 02950; GFX9-NEXT:  .LBB52_1: ; %atomicrmw.start2951; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=12952; GFX9-NEXT:    s_waitcnt vmcnt(0)2953; GFX9-NEXT:    v_and_b32_e32 v3, v4, v22954; GFX9-NEXT:    v_not_b32_e32 v3, v32955; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc2956; GFX9-NEXT:    s_waitcnt vmcnt(0)2957; GFX9-NEXT:    buffer_wbinvl1_vol2958; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v42959; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]2960; GFX9-NEXT:    v_mov_b32_e32 v4, v32961; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]2962; GFX9-NEXT:    s_cbranch_execnz .LBB52_12963; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end2964; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]2965; GFX9-NEXT:    s_setpc_b64 s[30:31]2966  %gep = getelementptr i32, ptr addrspace(1) %out, i32 42967  %tmp0 = atomicrmw nand ptr addrspace(1) %gep, i32 %in seq_cst2968  ret void2969}2970 2971define i32 @global_atomic_nand_i32_ret(ptr addrspace(1) %ptr, i32 %in) {2972; SI-LABEL: global_atomic_nand_i32_ret:2973; SI:       ; %bb.0:2974; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2975; SI-NEXT:    s_mov_b32 s6, 02976; SI-NEXT:    s_mov_b32 s7, 0xf0002977; SI-NEXT:    s_mov_b32 s4, s62978; SI-NEXT:    s_mov_b32 s5, s62979; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr642980; SI-NEXT:    s_mov_b64 s[8:9], 02981; SI-NEXT:  .LBB53_1: ; %atomicrmw.start2982; SI-NEXT:    ; =>This Inner Loop Header: Depth=12983; SI-NEXT:    s_waitcnt vmcnt(0)2984; SI-NEXT:    v_mov_b32_e32 v5, v32985; SI-NEXT:    s_waitcnt expcnt(0)2986; SI-NEXT:    v_and_b32_e32 v3, v5, v22987; SI-NEXT:    v_not_b32_e32 v4, v32988; SI-NEXT:    v_mov_b32_e32 v3, v42989; SI-NEXT:    v_mov_b32_e32 v4, v52990; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc2991; SI-NEXT:    s_waitcnt vmcnt(0)2992; SI-NEXT:    buffer_wbinvl12993; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v52994; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]2995; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]2996; SI-NEXT:    s_cbranch_execnz .LBB53_12997; SI-NEXT:  ; %bb.2: ; %atomicrmw.end2998; SI-NEXT:    s_or_b64 exec, exec, s[8:9]2999; SI-NEXT:    v_mov_b32_e32 v0, v33000; SI-NEXT:    s_waitcnt expcnt(0)3001; SI-NEXT:    s_setpc_b64 s[30:31]3002;3003; VI-LABEL: global_atomic_nand_i32_ret:3004; VI:       ; %bb.0:3005; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3006; VI-NEXT:    flat_load_dword v3, v[0:1]3007; VI-NEXT:    s_mov_b64 s[4:5], 03008; VI-NEXT:  .LBB53_1: ; %atomicrmw.start3009; VI-NEXT:    ; =>This Inner Loop Header: Depth=13010; VI-NEXT:    s_waitcnt vmcnt(0)3011; VI-NEXT:    v_mov_b32_e32 v4, v33012; VI-NEXT:    v_and_b32_e32 v3, v4, v23013; VI-NEXT:    v_not_b32_e32 v3, v33014; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc3015; VI-NEXT:    s_waitcnt vmcnt(0)3016; VI-NEXT:    buffer_wbinvl1_vol3017; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v43018; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]3019; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]3020; VI-NEXT:    s_cbranch_execnz .LBB53_13021; VI-NEXT:  ; %bb.2: ; %atomicrmw.end3022; VI-NEXT:    s_or_b64 exec, exec, s[4:5]3023; VI-NEXT:    v_mov_b32_e32 v0, v33024; VI-NEXT:    s_setpc_b64 s[30:31]3025;3026; GFX9-LABEL: global_atomic_nand_i32_ret:3027; GFX9:       ; %bb.0:3028; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3029; GFX9-NEXT:    global_load_dword v3, v[0:1], off3030; GFX9-NEXT:    s_mov_b64 s[4:5], 03031; GFX9-NEXT:  .LBB53_1: ; %atomicrmw.start3032; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=13033; GFX9-NEXT:    s_waitcnt vmcnt(0)3034; GFX9-NEXT:    v_mov_b32_e32 v4, v33035; GFX9-NEXT:    v_and_b32_e32 v3, v4, v23036; GFX9-NEXT:    v_not_b32_e32 v3, v33037; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc3038; GFX9-NEXT:    s_waitcnt vmcnt(0)3039; GFX9-NEXT:    buffer_wbinvl1_vol3040; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v43041; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]3042; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]3043; GFX9-NEXT:    s_cbranch_execnz .LBB53_13044; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end3045; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]3046; GFX9-NEXT:    v_mov_b32_e32 v0, v33047; GFX9-NEXT:    s_setpc_b64 s[30:31]3048  %result = atomicrmw nand ptr addrspace(1) %ptr, i32 %in seq_cst3049  ret i32 %result3050}3051 3052define i32 @global_atomic_nand_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {3053; SI-LABEL: global_atomic_nand_i32_ret_offset:3054; SI:       ; %bb.0:3055; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3056; SI-NEXT:    s_mov_b32 s6, 03057; SI-NEXT:    s_mov_b32 s7, 0xf0003058; SI-NEXT:    s_mov_b32 s4, s63059; SI-NEXT:    s_mov_b32 s5, s63060; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:163061; SI-NEXT:    s_mov_b64 s[8:9], 03062; SI-NEXT:  .LBB54_1: ; %atomicrmw.start3063; SI-NEXT:    ; =>This Inner Loop Header: Depth=13064; SI-NEXT:    s_waitcnt vmcnt(0)3065; SI-NEXT:    v_mov_b32_e32 v5, v33066; SI-NEXT:    s_waitcnt expcnt(0)3067; SI-NEXT:    v_and_b32_e32 v3, v5, v23068; SI-NEXT:    v_not_b32_e32 v4, v33069; SI-NEXT:    v_mov_b32_e32 v3, v43070; SI-NEXT:    v_mov_b32_e32 v4, v53071; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:16 glc3072; SI-NEXT:    s_waitcnt vmcnt(0)3073; SI-NEXT:    buffer_wbinvl13074; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v53075; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]3076; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]3077; SI-NEXT:    s_cbranch_execnz .LBB54_13078; SI-NEXT:  ; %bb.2: ; %atomicrmw.end3079; SI-NEXT:    s_or_b64 exec, exec, s[8:9]3080; SI-NEXT:    v_mov_b32_e32 v0, v33081; SI-NEXT:    s_waitcnt expcnt(0)3082; SI-NEXT:    s_setpc_b64 s[30:31]3083;3084; VI-LABEL: global_atomic_nand_i32_ret_offset:3085; VI:       ; %bb.0:3086; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3087; VI-NEXT:    v_add_u32_e32 v3, vcc, 16, v03088; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc3089; VI-NEXT:    flat_load_dword v0, v[3:4]3090; VI-NEXT:    s_mov_b64 s[4:5], 03091; VI-NEXT:  .LBB54_1: ; %atomicrmw.start3092; VI-NEXT:    ; =>This Inner Loop Header: Depth=13093; VI-NEXT:    s_waitcnt vmcnt(0)3094; VI-NEXT:    v_mov_b32_e32 v1, v03095; VI-NEXT:    v_and_b32_e32 v0, v1, v23096; VI-NEXT:    v_not_b32_e32 v0, v03097; VI-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc3098; VI-NEXT:    s_waitcnt vmcnt(0)3099; VI-NEXT:    buffer_wbinvl1_vol3100; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v13101; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]3102; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]3103; VI-NEXT:    s_cbranch_execnz .LBB54_13104; VI-NEXT:  ; %bb.2: ; %atomicrmw.end3105; VI-NEXT:    s_or_b64 exec, exec, s[4:5]3106; VI-NEXT:    s_setpc_b64 s[30:31]3107;3108; GFX9-LABEL: global_atomic_nand_i32_ret_offset:3109; GFX9:       ; %bb.0:3110; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3111; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:163112; GFX9-NEXT:    s_mov_b64 s[4:5], 03113; GFX9-NEXT:  .LBB54_1: ; %atomicrmw.start3114; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=13115; GFX9-NEXT:    s_waitcnt vmcnt(0)3116; GFX9-NEXT:    v_mov_b32_e32 v4, v33117; GFX9-NEXT:    v_and_b32_e32 v3, v4, v23118; GFX9-NEXT:    v_not_b32_e32 v3, v33119; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc3120; GFX9-NEXT:    s_waitcnt vmcnt(0)3121; GFX9-NEXT:    buffer_wbinvl1_vol3122; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v43123; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]3124; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]3125; GFX9-NEXT:    s_cbranch_execnz .LBB54_13126; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end3127; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]3128; GFX9-NEXT:    v_mov_b32_e32 v0, v33129; GFX9-NEXT:    s_setpc_b64 s[30:31]3130  %gep = getelementptr i32, ptr addrspace(1) %out, i32 43131  %result = atomicrmw nand ptr addrspace(1) %gep, i32 %in seq_cst3132  ret i32 %result3133}3134 3135define amdgpu_gfx void @global_atomic_nand_i32_noret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {3136; SI-LABEL: global_atomic_nand_i32_noret_scalar:3137; SI:       ; %bb.0:3138; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3139; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -13140; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill3141; SI-NEXT:    s_mov_b64 exec, s[34:35]3142; SI-NEXT:    s_waitcnt expcnt(0)3143; SI-NEXT:    v_writelane_b32 v4, s6, 03144; SI-NEXT:    v_writelane_b32 v4, s7, 13145; SI-NEXT:    s_mov_b32 s34, s63146; SI-NEXT:    s_mov_b32 s7, 0xf0003147; SI-NEXT:    s_mov_b32 s6, -13148; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 03149; SI-NEXT:    s_mov_b64 s[36:37], 03150; SI-NEXT:  .LBB55_1: ; %atomicrmw.start3151; SI-NEXT:    ; =>This Inner Loop Header: Depth=13152; SI-NEXT:    s_waitcnt vmcnt(0)3153; SI-NEXT:    v_and_b32_e32 v0, s34, v13154; SI-NEXT:    v_not_b32_e32 v0, v03155; SI-NEXT:    s_waitcnt expcnt(0)3156; SI-NEXT:    v_mov_b32_e32 v3, v13157; SI-NEXT:    v_mov_b32_e32 v2, v03158; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc3159; SI-NEXT:    s_waitcnt vmcnt(0)3160; SI-NEXT:    buffer_wbinvl13161; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v13162; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]3163; SI-NEXT:    v_mov_b32_e32 v1, v23164; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]3165; SI-NEXT:    s_cbranch_execnz .LBB55_13166; SI-NEXT:  ; %bb.2: ; %atomicrmw.end3167; SI-NEXT:    s_or_b64 exec, exec, s[36:37]3168; SI-NEXT:    v_readlane_b32 s7, v4, 13169; SI-NEXT:    v_readlane_b32 s6, v4, 03170; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -13171; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload3172; SI-NEXT:    s_mov_b64 exec, s[34:35]3173; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)3174; SI-NEXT:    s_setpc_b64 s[30:31]3175;3176; VI-LABEL: global_atomic_nand_i32_noret_scalar:3177; VI:       ; %bb.0:3178; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3179; VI-NEXT:    v_mov_b32_e32 v0, s43180; VI-NEXT:    v_mov_b32_e32 v1, s53181; VI-NEXT:    flat_load_dword v3, v[0:1]3182; VI-NEXT:    s_mov_b64 s[34:35], 03183; VI-NEXT:  .LBB55_1: ; %atomicrmw.start3184; VI-NEXT:    ; =>This Inner Loop Header: Depth=13185; VI-NEXT:    s_waitcnt vmcnt(0)3186; VI-NEXT:    v_and_b32_e32 v2, s6, v33187; VI-NEXT:    v_not_b32_e32 v2, v23188; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc3189; VI-NEXT:    s_waitcnt vmcnt(0)3190; VI-NEXT:    buffer_wbinvl1_vol3191; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v33192; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]3193; VI-NEXT:    v_mov_b32_e32 v3, v23194; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]3195; VI-NEXT:    s_cbranch_execnz .LBB55_13196; VI-NEXT:  ; %bb.2: ; %atomicrmw.end3197; VI-NEXT:    s_or_b64 exec, exec, s[34:35]3198; VI-NEXT:    s_setpc_b64 s[30:31]3199;3200; GFX9-LABEL: global_atomic_nand_i32_noret_scalar:3201; GFX9:       ; %bb.0:3202; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3203; GFX9-NEXT:    v_mov_b32_e32 v2, 03204; GFX9-NEXT:    global_load_dword v1, v2, s[4:5]3205; GFX9-NEXT:    s_mov_b64 s[34:35], 03206; GFX9-NEXT:  .LBB55_1: ; %atomicrmw.start3207; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=13208; GFX9-NEXT:    s_waitcnt vmcnt(0)3209; GFX9-NEXT:    v_and_b32_e32 v0, s6, v13210; GFX9-NEXT:    v_not_b32_e32 v0, v03211; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] glc3212; GFX9-NEXT:    s_waitcnt vmcnt(0)3213; GFX9-NEXT:    buffer_wbinvl1_vol3214; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v13215; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]3216; GFX9-NEXT:    v_mov_b32_e32 v1, v03217; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]3218; GFX9-NEXT:    s_cbranch_execnz .LBB55_13219; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end3220; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]3221; GFX9-NEXT:    s_setpc_b64 s[30:31]3222  %tmp0 = atomicrmw nand ptr addrspace(1) %ptr, i32 %in seq_cst3223  ret void3224}3225 3226define amdgpu_gfx void @global_atomic_nand_i32_noret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {3227; SI-LABEL: global_atomic_nand_i32_noret_offset_scalar:3228; SI:       ; %bb.0:3229; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3230; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -13231; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill3232; SI-NEXT:    s_mov_b64 exec, s[34:35]3233; SI-NEXT:    s_waitcnt expcnt(0)3234; SI-NEXT:    v_writelane_b32 v4, s6, 03235; SI-NEXT:    v_writelane_b32 v4, s7, 13236; SI-NEXT:    s_mov_b32 s34, s63237; SI-NEXT:    s_mov_b32 s7, 0xf0003238; SI-NEXT:    s_mov_b32 s6, -13239; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:163240; SI-NEXT:    s_mov_b64 s[36:37], 03241; SI-NEXT:  .LBB56_1: ; %atomicrmw.start3242; SI-NEXT:    ; =>This Inner Loop Header: Depth=13243; SI-NEXT:    s_waitcnt vmcnt(0)3244; SI-NEXT:    v_and_b32_e32 v0, s34, v13245; SI-NEXT:    v_not_b32_e32 v0, v03246; SI-NEXT:    s_waitcnt expcnt(0)3247; SI-NEXT:    v_mov_b32_e32 v3, v13248; SI-NEXT:    v_mov_b32_e32 v2, v03249; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 offset:16 glc3250; SI-NEXT:    s_waitcnt vmcnt(0)3251; SI-NEXT:    buffer_wbinvl13252; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v13253; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]3254; SI-NEXT:    v_mov_b32_e32 v1, v23255; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]3256; SI-NEXT:    s_cbranch_execnz .LBB56_13257; SI-NEXT:  ; %bb.2: ; %atomicrmw.end3258; SI-NEXT:    s_or_b64 exec, exec, s[36:37]3259; SI-NEXT:    v_readlane_b32 s7, v4, 13260; SI-NEXT:    v_readlane_b32 s6, v4, 03261; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -13262; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload3263; SI-NEXT:    s_mov_b64 exec, s[34:35]3264; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)3265; SI-NEXT:    s_setpc_b64 s[30:31]3266;3267; VI-LABEL: global_atomic_nand_i32_noret_offset_scalar:3268; VI:       ; %bb.0:3269; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3270; VI-NEXT:    s_add_u32 s34, s4, 163271; VI-NEXT:    s_addc_u32 s35, s5, 03272; VI-NEXT:    v_mov_b32_e32 v0, s343273; VI-NEXT:    v_mov_b32_e32 v1, s353274; VI-NEXT:    flat_load_dword v3, v[0:1]3275; VI-NEXT:    s_mov_b64 s[34:35], 03276; VI-NEXT:  .LBB56_1: ; %atomicrmw.start3277; VI-NEXT:    ; =>This Inner Loop Header: Depth=13278; VI-NEXT:    s_waitcnt vmcnt(0)3279; VI-NEXT:    v_and_b32_e32 v2, s6, v33280; VI-NEXT:    v_not_b32_e32 v2, v23281; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc3282; VI-NEXT:    s_waitcnt vmcnt(0)3283; VI-NEXT:    buffer_wbinvl1_vol3284; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v33285; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]3286; VI-NEXT:    v_mov_b32_e32 v3, v23287; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]3288; VI-NEXT:    s_cbranch_execnz .LBB56_13289; VI-NEXT:  ; %bb.2: ; %atomicrmw.end3290; VI-NEXT:    s_or_b64 exec, exec, s[34:35]3291; VI-NEXT:    s_setpc_b64 s[30:31]3292;3293; GFX9-LABEL: global_atomic_nand_i32_noret_offset_scalar:3294; GFX9:       ; %bb.0:3295; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3296; GFX9-NEXT:    v_mov_b32_e32 v2, 03297; GFX9-NEXT:    global_load_dword v1, v2, s[4:5] offset:163298; GFX9-NEXT:    s_mov_b64 s[34:35], 03299; GFX9-NEXT:  .LBB56_1: ; %atomicrmw.start3300; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=13301; GFX9-NEXT:    s_waitcnt vmcnt(0)3302; GFX9-NEXT:    v_and_b32_e32 v0, s6, v13303; GFX9-NEXT:    v_not_b32_e32 v0, v03304; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 glc3305; GFX9-NEXT:    s_waitcnt vmcnt(0)3306; GFX9-NEXT:    buffer_wbinvl1_vol3307; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v13308; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]3309; GFX9-NEXT:    v_mov_b32_e32 v1, v03310; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]3311; GFX9-NEXT:    s_cbranch_execnz .LBB56_13312; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end3313; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]3314; GFX9-NEXT:    s_setpc_b64 s[30:31]3315  %gep = getelementptr i32, ptr addrspace(1) %out, i32 43316  %tmp0 = atomicrmw nand ptr addrspace(1) %gep, i32 %in seq_cst3317  ret void3318}3319 3320define amdgpu_gfx i32 @global_atomic_nand_i32_ret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {3321; SI-LABEL: global_atomic_nand_i32_ret_scalar:3322; SI:       ; %bb.0:3323; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3324; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -13325; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill3326; SI-NEXT:    s_mov_b64 exec, s[34:35]3327; SI-NEXT:    s_waitcnt expcnt(0)3328; SI-NEXT:    v_writelane_b32 v3, s6, 03329; SI-NEXT:    v_writelane_b32 v3, s7, 13330; SI-NEXT:    s_mov_b32 s34, s63331; SI-NEXT:    s_mov_b32 s7, 0xf0003332; SI-NEXT:    s_mov_b32 s6, -13333; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 03334; SI-NEXT:    s_mov_b64 s[36:37], 03335; SI-NEXT:  .LBB57_1: ; %atomicrmw.start3336; SI-NEXT:    ; =>This Inner Loop Header: Depth=13337; SI-NEXT:    s_waitcnt vmcnt(0)3338; SI-NEXT:    v_mov_b32_e32 v2, v03339; SI-NEXT:    s_waitcnt expcnt(0)3340; SI-NEXT:    v_and_b32_e32 v0, s34, v23341; SI-NEXT:    v_not_b32_e32 v1, v03342; SI-NEXT:    v_mov_b32_e32 v0, v13343; SI-NEXT:    v_mov_b32_e32 v1, v23344; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc3345; SI-NEXT:    s_waitcnt vmcnt(0)3346; SI-NEXT:    buffer_wbinvl13347; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v23348; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]3349; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]3350; SI-NEXT:    s_cbranch_execnz .LBB57_13351; SI-NEXT:  ; %bb.2: ; %atomicrmw.end3352; SI-NEXT:    s_or_b64 exec, exec, s[36:37]3353; SI-NEXT:    v_readlane_b32 s7, v3, 13354; SI-NEXT:    v_readlane_b32 s6, v3, 03355; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -13356; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload3357; SI-NEXT:    s_mov_b64 exec, s[34:35]3358; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)3359; SI-NEXT:    s_setpc_b64 s[30:31]3360;3361; VI-LABEL: global_atomic_nand_i32_ret_scalar:3362; VI:       ; %bb.0:3363; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3364; VI-NEXT:    v_mov_b32_e32 v0, s43365; VI-NEXT:    v_mov_b32_e32 v1, s53366; VI-NEXT:    flat_load_dword v0, v[0:1]3367; VI-NEXT:    v_mov_b32_e32 v1, s43368; VI-NEXT:    s_mov_b64 s[34:35], 03369; VI-NEXT:    v_mov_b32_e32 v2, s53370; VI-NEXT:  .LBB57_1: ; %atomicrmw.start3371; VI-NEXT:    ; =>This Inner Loop Header: Depth=13372; VI-NEXT:    s_waitcnt vmcnt(0)3373; VI-NEXT:    v_mov_b32_e32 v4, v03374; VI-NEXT:    v_and_b32_e32 v0, s6, v43375; VI-NEXT:    v_not_b32_e32 v3, v03376; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc3377; VI-NEXT:    s_waitcnt vmcnt(0)3378; VI-NEXT:    buffer_wbinvl1_vol3379; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v43380; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]3381; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]3382; VI-NEXT:    s_cbranch_execnz .LBB57_13383; VI-NEXT:  ; %bb.2: ; %atomicrmw.end3384; VI-NEXT:    s_or_b64 exec, exec, s[34:35]3385; VI-NEXT:    s_setpc_b64 s[30:31]3386;3387; GFX9-LABEL: global_atomic_nand_i32_ret_scalar:3388; GFX9:       ; %bb.0:3389; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3390; GFX9-NEXT:    v_mov_b32_e32 v1, 03391; GFX9-NEXT:    global_load_dword v0, v1, s[4:5]3392; GFX9-NEXT:    s_mov_b64 s[34:35], 03393; GFX9-NEXT:  .LBB57_1: ; %atomicrmw.start3394; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=13395; GFX9-NEXT:    s_waitcnt vmcnt(0)3396; GFX9-NEXT:    v_mov_b32_e32 v3, v03397; GFX9-NEXT:    v_and_b32_e32 v0, s6, v33398; GFX9-NEXT:    v_not_b32_e32 v2, v03399; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] glc3400; GFX9-NEXT:    s_waitcnt vmcnt(0)3401; GFX9-NEXT:    buffer_wbinvl1_vol3402; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v33403; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]3404; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]3405; GFX9-NEXT:    s_cbranch_execnz .LBB57_13406; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end3407; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]3408; GFX9-NEXT:    s_setpc_b64 s[30:31]3409  %result = atomicrmw nand ptr addrspace(1) %ptr, i32 %in seq_cst3410  ret i32 %result3411}3412 3413define amdgpu_gfx i32 @global_atomic_nand_i32_ret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {3414; SI-LABEL: global_atomic_nand_i32_ret_offset_scalar:3415; SI:       ; %bb.0:3416; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3417; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -13418; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill3419; SI-NEXT:    s_mov_b64 exec, s[34:35]3420; SI-NEXT:    s_waitcnt expcnt(0)3421; SI-NEXT:    v_writelane_b32 v3, s6, 03422; SI-NEXT:    v_writelane_b32 v3, s7, 13423; SI-NEXT:    s_mov_b32 s34, s63424; SI-NEXT:    s_mov_b32 s7, 0xf0003425; SI-NEXT:    s_mov_b32 s6, -13426; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:163427; SI-NEXT:    s_mov_b64 s[36:37], 03428; SI-NEXT:  .LBB58_1: ; %atomicrmw.start3429; SI-NEXT:    ; =>This Inner Loop Header: Depth=13430; SI-NEXT:    s_waitcnt vmcnt(0)3431; SI-NEXT:    v_mov_b32_e32 v2, v03432; SI-NEXT:    s_waitcnt expcnt(0)3433; SI-NEXT:    v_and_b32_e32 v0, s34, v23434; SI-NEXT:    v_not_b32_e32 v1, v03435; SI-NEXT:    v_mov_b32_e32 v0, v13436; SI-NEXT:    v_mov_b32_e32 v1, v23437; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc3438; SI-NEXT:    s_waitcnt vmcnt(0)3439; SI-NEXT:    buffer_wbinvl13440; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v23441; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]3442; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]3443; SI-NEXT:    s_cbranch_execnz .LBB58_13444; SI-NEXT:  ; %bb.2: ; %atomicrmw.end3445; SI-NEXT:    s_or_b64 exec, exec, s[36:37]3446; SI-NEXT:    v_readlane_b32 s7, v3, 13447; SI-NEXT:    v_readlane_b32 s6, v3, 03448; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -13449; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload3450; SI-NEXT:    s_mov_b64 exec, s[34:35]3451; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)3452; SI-NEXT:    s_setpc_b64 s[30:31]3453;3454; VI-LABEL: global_atomic_nand_i32_ret_offset_scalar:3455; VI:       ; %bb.0:3456; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3457; VI-NEXT:    s_add_u32 s34, s4, 163458; VI-NEXT:    s_addc_u32 s35, s5, 03459; VI-NEXT:    v_mov_b32_e32 v1, s343460; VI-NEXT:    v_mov_b32_e32 v2, s353461; VI-NEXT:    flat_load_dword v0, v[1:2]3462; VI-NEXT:    s_mov_b64 s[34:35], 03463; VI-NEXT:  .LBB58_1: ; %atomicrmw.start3464; VI-NEXT:    ; =>This Inner Loop Header: Depth=13465; VI-NEXT:    s_waitcnt vmcnt(0)3466; VI-NEXT:    v_mov_b32_e32 v4, v03467; VI-NEXT:    v_and_b32_e32 v0, s6, v43468; VI-NEXT:    v_not_b32_e32 v3, v03469; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc3470; VI-NEXT:    s_waitcnt vmcnt(0)3471; VI-NEXT:    buffer_wbinvl1_vol3472; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v43473; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]3474; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]3475; VI-NEXT:    s_cbranch_execnz .LBB58_13476; VI-NEXT:  ; %bb.2: ; %atomicrmw.end3477; VI-NEXT:    s_or_b64 exec, exec, s[34:35]3478; VI-NEXT:    s_setpc_b64 s[30:31]3479;3480; GFX9-LABEL: global_atomic_nand_i32_ret_offset_scalar:3481; GFX9:       ; %bb.0:3482; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3483; GFX9-NEXT:    v_mov_b32_e32 v1, 03484; GFX9-NEXT:    global_load_dword v0, v1, s[4:5] offset:163485; GFX9-NEXT:    s_mov_b64 s[34:35], 03486; GFX9-NEXT:  .LBB58_1: ; %atomicrmw.start3487; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=13488; GFX9-NEXT:    s_waitcnt vmcnt(0)3489; GFX9-NEXT:    v_mov_b32_e32 v3, v03490; GFX9-NEXT:    v_and_b32_e32 v0, s6, v33491; GFX9-NEXT:    v_not_b32_e32 v2, v03492; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] offset:16 glc3493; GFX9-NEXT:    s_waitcnt vmcnt(0)3494; GFX9-NEXT:    buffer_wbinvl1_vol3495; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v33496; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]3497; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]3498; GFX9-NEXT:    s_cbranch_execnz .LBB58_13499; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end3500; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]3501; GFX9-NEXT:    s_setpc_b64 s[30:31]3502  %gep = getelementptr i32, ptr addrspace(1) %out, i32 43503  %result = atomicrmw nand ptr addrspace(1) %gep, i32 %in seq_cst3504  ret i32 %result3505}3506 3507define void @global_atomic_nand_i32_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {3508; SI-LABEL: global_atomic_nand_i32_noret_offset__amdgpu_no_remote_memory:3509; SI:       ; %bb.0:3510; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3511; SI-NEXT:    s_mov_b32 s6, 03512; SI-NEXT:    s_mov_b32 s7, 0xf0003513; SI-NEXT:    s_mov_b32 s4, s63514; SI-NEXT:    s_mov_b32 s5, s63515; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:163516; SI-NEXT:    s_mov_b64 s[8:9], 03517; SI-NEXT:  .LBB59_1: ; %atomicrmw.start3518; SI-NEXT:    ; =>This Inner Loop Header: Depth=13519; SI-NEXT:    s_waitcnt vmcnt(0)3520; SI-NEXT:    v_and_b32_e32 v3, v4, v23521; SI-NEXT:    v_not_b32_e32 v3, v33522; SI-NEXT:    s_waitcnt expcnt(0)3523; SI-NEXT:    v_mov_b32_e32 v6, v43524; SI-NEXT:    v_mov_b32_e32 v5, v33525; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:16 glc3526; SI-NEXT:    s_waitcnt vmcnt(0)3527; SI-NEXT:    buffer_wbinvl13528; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v43529; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]3530; SI-NEXT:    v_mov_b32_e32 v4, v53531; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]3532; SI-NEXT:    s_cbranch_execnz .LBB59_13533; SI-NEXT:  ; %bb.2: ; %atomicrmw.end3534; SI-NEXT:    s_or_b64 exec, exec, s[8:9]3535; SI-NEXT:    s_waitcnt expcnt(0)3536; SI-NEXT:    s_setpc_b64 s[30:31]3537;3538; VI-LABEL: global_atomic_nand_i32_noret_offset__amdgpu_no_remote_memory:3539; VI:       ; %bb.0:3540; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3541; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v03542; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc3543; VI-NEXT:    flat_load_dword v4, v[0:1]3544; VI-NEXT:    s_mov_b64 s[4:5], 03545; VI-NEXT:  .LBB59_1: ; %atomicrmw.start3546; VI-NEXT:    ; =>This Inner Loop Header: Depth=13547; VI-NEXT:    s_waitcnt vmcnt(0)3548; VI-NEXT:    v_and_b32_e32 v3, v4, v23549; VI-NEXT:    v_not_b32_e32 v3, v33550; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc3551; VI-NEXT:    s_waitcnt vmcnt(0)3552; VI-NEXT:    buffer_wbinvl1_vol3553; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v43554; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]3555; VI-NEXT:    v_mov_b32_e32 v4, v33556; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]3557; VI-NEXT:    s_cbranch_execnz .LBB59_13558; VI-NEXT:  ; %bb.2: ; %atomicrmw.end3559; VI-NEXT:    s_or_b64 exec, exec, s[4:5]3560; VI-NEXT:    s_setpc_b64 s[30:31]3561;3562; GFX9-LABEL: global_atomic_nand_i32_noret_offset__amdgpu_no_remote_memory:3563; GFX9:       ; %bb.0:3564; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3565; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:163566; GFX9-NEXT:    s_mov_b64 s[4:5], 03567; GFX9-NEXT:  .LBB59_1: ; %atomicrmw.start3568; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=13569; GFX9-NEXT:    s_waitcnt vmcnt(0)3570; GFX9-NEXT:    v_and_b32_e32 v3, v4, v23571; GFX9-NEXT:    v_not_b32_e32 v3, v33572; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc3573; GFX9-NEXT:    s_waitcnt vmcnt(0)3574; GFX9-NEXT:    buffer_wbinvl1_vol3575; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v43576; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]3577; GFX9-NEXT:    v_mov_b32_e32 v4, v33578; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]3579; GFX9-NEXT:    s_cbranch_execnz .LBB59_13580; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end3581; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]3582; GFX9-NEXT:    s_setpc_b64 s[30:31]3583  %gep = getelementptr i32, ptr addrspace(1) %out, i64 43584  %tmp0 = atomicrmw nand ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !03585  ret void3586}3587 3588define i32 @global_atomic_nand_i32_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {3589; SI-LABEL: global_atomic_nand_i32_ret_offset__amdgpu_no_remote_memory:3590; SI:       ; %bb.0:3591; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3592; SI-NEXT:    s_mov_b32 s6, 03593; SI-NEXT:    s_mov_b32 s7, 0xf0003594; SI-NEXT:    s_mov_b32 s4, s63595; SI-NEXT:    s_mov_b32 s5, s63596; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:163597; SI-NEXT:    s_mov_b64 s[8:9], 03598; SI-NEXT:  .LBB60_1: ; %atomicrmw.start3599; SI-NEXT:    ; =>This Inner Loop Header: Depth=13600; SI-NEXT:    s_waitcnt vmcnt(0)3601; SI-NEXT:    v_mov_b32_e32 v5, v33602; SI-NEXT:    s_waitcnt expcnt(0)3603; SI-NEXT:    v_and_b32_e32 v3, v5, v23604; SI-NEXT:    v_not_b32_e32 v4, v33605; SI-NEXT:    v_mov_b32_e32 v3, v43606; SI-NEXT:    v_mov_b32_e32 v4, v53607; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:16 glc3608; SI-NEXT:    s_waitcnt vmcnt(0)3609; SI-NEXT:    buffer_wbinvl13610; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v53611; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]3612; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]3613; SI-NEXT:    s_cbranch_execnz .LBB60_13614; SI-NEXT:  ; %bb.2: ; %atomicrmw.end3615; SI-NEXT:    s_or_b64 exec, exec, s[8:9]3616; SI-NEXT:    v_mov_b32_e32 v0, v33617; SI-NEXT:    s_waitcnt expcnt(0)3618; SI-NEXT:    s_setpc_b64 s[30:31]3619;3620; VI-LABEL: global_atomic_nand_i32_ret_offset__amdgpu_no_remote_memory:3621; VI:       ; %bb.0:3622; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3623; VI-NEXT:    v_add_u32_e32 v3, vcc, 16, v03624; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc3625; VI-NEXT:    flat_load_dword v0, v[3:4]3626; VI-NEXT:    s_mov_b64 s[4:5], 03627; VI-NEXT:  .LBB60_1: ; %atomicrmw.start3628; VI-NEXT:    ; =>This Inner Loop Header: Depth=13629; VI-NEXT:    s_waitcnt vmcnt(0)3630; VI-NEXT:    v_mov_b32_e32 v1, v03631; VI-NEXT:    v_and_b32_e32 v0, v1, v23632; VI-NEXT:    v_not_b32_e32 v0, v03633; VI-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc3634; VI-NEXT:    s_waitcnt vmcnt(0)3635; VI-NEXT:    buffer_wbinvl1_vol3636; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v13637; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]3638; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]3639; VI-NEXT:    s_cbranch_execnz .LBB60_13640; VI-NEXT:  ; %bb.2: ; %atomicrmw.end3641; VI-NEXT:    s_or_b64 exec, exec, s[4:5]3642; VI-NEXT:    s_setpc_b64 s[30:31]3643;3644; GFX9-LABEL: global_atomic_nand_i32_ret_offset__amdgpu_no_remote_memory:3645; GFX9:       ; %bb.0:3646; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3647; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:163648; GFX9-NEXT:    s_mov_b64 s[4:5], 03649; GFX9-NEXT:  .LBB60_1: ; %atomicrmw.start3650; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=13651; GFX9-NEXT:    s_waitcnt vmcnt(0)3652; GFX9-NEXT:    v_mov_b32_e32 v4, v33653; GFX9-NEXT:    v_and_b32_e32 v3, v4, v23654; GFX9-NEXT:    v_not_b32_e32 v3, v33655; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc3656; GFX9-NEXT:    s_waitcnt vmcnt(0)3657; GFX9-NEXT:    buffer_wbinvl1_vol3658; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v43659; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]3660; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]3661; GFX9-NEXT:    s_cbranch_execnz .LBB60_13662; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end3663; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]3664; GFX9-NEXT:    v_mov_b32_e32 v0, v33665; GFX9-NEXT:    s_setpc_b64 s[30:31]3666  %gep = getelementptr i32, ptr addrspace(1) %out, i64 43667  %result = atomicrmw nand ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !03668  ret i32 %result3669}3670 3671; ---------------------------------------------------------------------3672; atomicrmw or3673; ---------------------------------------------------------------------3674 3675define void @global_atomic_or_i32_noret(ptr addrspace(1) %ptr, i32 %in) {3676; SI-LABEL: global_atomic_or_i32_noret:3677; SI:       ; %bb.0:3678; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3679; SI-NEXT:    s_mov_b32 s6, 03680; SI-NEXT:    s_mov_b32 s7, 0xf0003681; SI-NEXT:    s_mov_b32 s4, s63682; SI-NEXT:    s_mov_b32 s5, s63683; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr643684; SI-NEXT:    s_mov_b64 s[8:9], 03685; SI-NEXT:  .LBB61_1: ; %atomicrmw.start3686; SI-NEXT:    ; =>This Inner Loop Header: Depth=13687; SI-NEXT:    s_waitcnt vmcnt(0)3688; SI-NEXT:    v_or_b32_e32 v3, v4, v23689; SI-NEXT:    s_waitcnt expcnt(0)3690; SI-NEXT:    v_mov_b32_e32 v6, v43691; SI-NEXT:    v_mov_b32_e32 v5, v33692; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc3693; SI-NEXT:    s_waitcnt vmcnt(0)3694; SI-NEXT:    buffer_wbinvl13695; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v43696; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]3697; SI-NEXT:    v_mov_b32_e32 v4, v53698; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]3699; SI-NEXT:    s_cbranch_execnz .LBB61_13700; SI-NEXT:  ; %bb.2: ; %atomicrmw.end3701; SI-NEXT:    s_or_b64 exec, exec, s[8:9]3702; SI-NEXT:    s_waitcnt expcnt(0)3703; SI-NEXT:    s_setpc_b64 s[30:31]3704;3705; VI-LABEL: global_atomic_or_i32_noret:3706; VI:       ; %bb.0:3707; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3708; VI-NEXT:    flat_load_dword v4, v[0:1]3709; VI-NEXT:    s_mov_b64 s[4:5], 03710; VI-NEXT:  .LBB61_1: ; %atomicrmw.start3711; VI-NEXT:    ; =>This Inner Loop Header: Depth=13712; VI-NEXT:    s_waitcnt vmcnt(0)3713; VI-NEXT:    v_or_b32_e32 v3, v4, v23714; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc3715; VI-NEXT:    s_waitcnt vmcnt(0)3716; VI-NEXT:    buffer_wbinvl1_vol3717; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v43718; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]3719; VI-NEXT:    v_mov_b32_e32 v4, v33720; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]3721; VI-NEXT:    s_cbranch_execnz .LBB61_13722; VI-NEXT:  ; %bb.2: ; %atomicrmw.end3723; VI-NEXT:    s_or_b64 exec, exec, s[4:5]3724; VI-NEXT:    s_setpc_b64 s[30:31]3725;3726; GFX9-LABEL: global_atomic_or_i32_noret:3727; GFX9:       ; %bb.0:3728; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3729; GFX9-NEXT:    global_load_dword v4, v[0:1], off3730; GFX9-NEXT:    s_mov_b64 s[4:5], 03731; GFX9-NEXT:  .LBB61_1: ; %atomicrmw.start3732; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=13733; GFX9-NEXT:    s_waitcnt vmcnt(0)3734; GFX9-NEXT:    v_or_b32_e32 v3, v4, v23735; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc3736; GFX9-NEXT:    s_waitcnt vmcnt(0)3737; GFX9-NEXT:    buffer_wbinvl1_vol3738; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v43739; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]3740; GFX9-NEXT:    v_mov_b32_e32 v4, v33741; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]3742; GFX9-NEXT:    s_cbranch_execnz .LBB61_13743; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end3744; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]3745; GFX9-NEXT:    s_setpc_b64 s[30:31]3746  %tmp0 = atomicrmw or ptr addrspace(1) %ptr, i32 %in seq_cst3747  ret void3748}3749 3750define void @global_atomic_or_i32_noret_offset(ptr addrspace(1) %out, i32 %in) {3751; SI-LABEL: global_atomic_or_i32_noret_offset:3752; SI:       ; %bb.0:3753; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3754; SI-NEXT:    s_mov_b32 s6, 03755; SI-NEXT:    s_mov_b32 s7, 0xf0003756; SI-NEXT:    s_mov_b32 s4, s63757; SI-NEXT:    s_mov_b32 s5, s63758; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:163759; SI-NEXT:    s_mov_b64 s[8:9], 03760; SI-NEXT:  .LBB62_1: ; %atomicrmw.start3761; SI-NEXT:    ; =>This Inner Loop Header: Depth=13762; SI-NEXT:    s_waitcnt vmcnt(0)3763; SI-NEXT:    v_or_b32_e32 v3, v4, v23764; SI-NEXT:    s_waitcnt expcnt(0)3765; SI-NEXT:    v_mov_b32_e32 v6, v43766; SI-NEXT:    v_mov_b32_e32 v5, v33767; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:16 glc3768; SI-NEXT:    s_waitcnt vmcnt(0)3769; SI-NEXT:    buffer_wbinvl13770; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v43771; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]3772; SI-NEXT:    v_mov_b32_e32 v4, v53773; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]3774; SI-NEXT:    s_cbranch_execnz .LBB62_13775; SI-NEXT:  ; %bb.2: ; %atomicrmw.end3776; SI-NEXT:    s_or_b64 exec, exec, s[8:9]3777; SI-NEXT:    s_waitcnt expcnt(0)3778; SI-NEXT:    s_setpc_b64 s[30:31]3779;3780; VI-LABEL: global_atomic_or_i32_noret_offset:3781; VI:       ; %bb.0:3782; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3783; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v03784; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc3785; VI-NEXT:    flat_load_dword v4, v[0:1]3786; VI-NEXT:    s_mov_b64 s[4:5], 03787; VI-NEXT:  .LBB62_1: ; %atomicrmw.start3788; VI-NEXT:    ; =>This Inner Loop Header: Depth=13789; VI-NEXT:    s_waitcnt vmcnt(0)3790; VI-NEXT:    v_or_b32_e32 v3, v4, v23791; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc3792; VI-NEXT:    s_waitcnt vmcnt(0)3793; VI-NEXT:    buffer_wbinvl1_vol3794; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v43795; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]3796; VI-NEXT:    v_mov_b32_e32 v4, v33797; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]3798; VI-NEXT:    s_cbranch_execnz .LBB62_13799; VI-NEXT:  ; %bb.2: ; %atomicrmw.end3800; VI-NEXT:    s_or_b64 exec, exec, s[4:5]3801; VI-NEXT:    s_setpc_b64 s[30:31]3802;3803; GFX9-LABEL: global_atomic_or_i32_noret_offset:3804; GFX9:       ; %bb.0:3805; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3806; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:163807; GFX9-NEXT:    s_mov_b64 s[4:5], 03808; GFX9-NEXT:  .LBB62_1: ; %atomicrmw.start3809; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=13810; GFX9-NEXT:    s_waitcnt vmcnt(0)3811; GFX9-NEXT:    v_or_b32_e32 v3, v4, v23812; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc3813; GFX9-NEXT:    s_waitcnt vmcnt(0)3814; GFX9-NEXT:    buffer_wbinvl1_vol3815; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v43816; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]3817; GFX9-NEXT:    v_mov_b32_e32 v4, v33818; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]3819; GFX9-NEXT:    s_cbranch_execnz .LBB62_13820; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end3821; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]3822; GFX9-NEXT:    s_setpc_b64 s[30:31]3823  %gep = getelementptr i32, ptr addrspace(1) %out, i32 43824  %tmp0 = atomicrmw or ptr addrspace(1) %gep, i32 %in seq_cst3825  ret void3826}3827 3828define i32 @global_atomic_or_i32_ret(ptr addrspace(1) %ptr, i32 %in) {3829; SI-LABEL: global_atomic_or_i32_ret:3830; SI:       ; %bb.0:3831; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3832; SI-NEXT:    s_mov_b32 s6, 03833; SI-NEXT:    s_mov_b32 s7, 0xf0003834; SI-NEXT:    s_mov_b32 s4, s63835; SI-NEXT:    s_mov_b32 s5, s63836; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr643837; SI-NEXT:    s_mov_b64 s[8:9], 03838; SI-NEXT:  .LBB63_1: ; %atomicrmw.start3839; SI-NEXT:    ; =>This Inner Loop Header: Depth=13840; SI-NEXT:    s_waitcnt vmcnt(0)3841; SI-NEXT:    v_mov_b32_e32 v5, v33842; SI-NEXT:    s_waitcnt expcnt(0)3843; SI-NEXT:    v_or_b32_e32 v4, v5, v23844; SI-NEXT:    v_mov_b32_e32 v3, v43845; SI-NEXT:    v_mov_b32_e32 v4, v53846; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc3847; SI-NEXT:    s_waitcnt vmcnt(0)3848; SI-NEXT:    buffer_wbinvl13849; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v53850; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]3851; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]3852; SI-NEXT:    s_cbranch_execnz .LBB63_13853; SI-NEXT:  ; %bb.2: ; %atomicrmw.end3854; SI-NEXT:    s_or_b64 exec, exec, s[8:9]3855; SI-NEXT:    v_mov_b32_e32 v0, v33856; SI-NEXT:    s_waitcnt expcnt(0)3857; SI-NEXT:    s_setpc_b64 s[30:31]3858;3859; VI-LABEL: global_atomic_or_i32_ret:3860; VI:       ; %bb.0:3861; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3862; VI-NEXT:    flat_load_dword v3, v[0:1]3863; VI-NEXT:    s_mov_b64 s[4:5], 03864; VI-NEXT:  .LBB63_1: ; %atomicrmw.start3865; VI-NEXT:    ; =>This Inner Loop Header: Depth=13866; VI-NEXT:    s_waitcnt vmcnt(0)3867; VI-NEXT:    v_mov_b32_e32 v4, v33868; VI-NEXT:    v_or_b32_e32 v3, v4, v23869; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc3870; VI-NEXT:    s_waitcnt vmcnt(0)3871; VI-NEXT:    buffer_wbinvl1_vol3872; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v43873; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]3874; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]3875; VI-NEXT:    s_cbranch_execnz .LBB63_13876; VI-NEXT:  ; %bb.2: ; %atomicrmw.end3877; VI-NEXT:    s_or_b64 exec, exec, s[4:5]3878; VI-NEXT:    v_mov_b32_e32 v0, v33879; VI-NEXT:    s_setpc_b64 s[30:31]3880;3881; GFX9-LABEL: global_atomic_or_i32_ret:3882; GFX9:       ; %bb.0:3883; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3884; GFX9-NEXT:    global_load_dword v3, v[0:1], off3885; GFX9-NEXT:    s_mov_b64 s[4:5], 03886; GFX9-NEXT:  .LBB63_1: ; %atomicrmw.start3887; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=13888; GFX9-NEXT:    s_waitcnt vmcnt(0)3889; GFX9-NEXT:    v_mov_b32_e32 v4, v33890; GFX9-NEXT:    v_or_b32_e32 v3, v4, v23891; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc3892; GFX9-NEXT:    s_waitcnt vmcnt(0)3893; GFX9-NEXT:    buffer_wbinvl1_vol3894; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v43895; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]3896; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]3897; GFX9-NEXT:    s_cbranch_execnz .LBB63_13898; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end3899; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]3900; GFX9-NEXT:    v_mov_b32_e32 v0, v33901; GFX9-NEXT:    s_setpc_b64 s[30:31]3902  %result = atomicrmw or ptr addrspace(1) %ptr, i32 %in seq_cst3903  ret i32 %result3904}3905 3906define i32 @global_atomic_or_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {3907; SI-LABEL: global_atomic_or_i32_ret_offset:3908; SI:       ; %bb.0:3909; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3910; SI-NEXT:    s_mov_b32 s6, 03911; SI-NEXT:    s_mov_b32 s7, 0xf0003912; SI-NEXT:    s_mov_b32 s4, s63913; SI-NEXT:    s_mov_b32 s5, s63914; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:163915; SI-NEXT:    s_mov_b64 s[8:9], 03916; SI-NEXT:  .LBB64_1: ; %atomicrmw.start3917; SI-NEXT:    ; =>This Inner Loop Header: Depth=13918; SI-NEXT:    s_waitcnt vmcnt(0)3919; SI-NEXT:    v_mov_b32_e32 v5, v33920; SI-NEXT:    s_waitcnt expcnt(0)3921; SI-NEXT:    v_or_b32_e32 v4, v5, v23922; SI-NEXT:    v_mov_b32_e32 v3, v43923; SI-NEXT:    v_mov_b32_e32 v4, v53924; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:16 glc3925; SI-NEXT:    s_waitcnt vmcnt(0)3926; SI-NEXT:    buffer_wbinvl13927; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v53928; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]3929; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]3930; SI-NEXT:    s_cbranch_execnz .LBB64_13931; SI-NEXT:  ; %bb.2: ; %atomicrmw.end3932; SI-NEXT:    s_or_b64 exec, exec, s[8:9]3933; SI-NEXT:    v_mov_b32_e32 v0, v33934; SI-NEXT:    s_waitcnt expcnt(0)3935; SI-NEXT:    s_setpc_b64 s[30:31]3936;3937; VI-LABEL: global_atomic_or_i32_ret_offset:3938; VI:       ; %bb.0:3939; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3940; VI-NEXT:    v_add_u32_e32 v3, vcc, 16, v03941; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc3942; VI-NEXT:    flat_load_dword v0, v[3:4]3943; VI-NEXT:    s_mov_b64 s[4:5], 03944; VI-NEXT:  .LBB64_1: ; %atomicrmw.start3945; VI-NEXT:    ; =>This Inner Loop Header: Depth=13946; VI-NEXT:    s_waitcnt vmcnt(0)3947; VI-NEXT:    v_mov_b32_e32 v1, v03948; VI-NEXT:    v_or_b32_e32 v0, v1, v23949; VI-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc3950; VI-NEXT:    s_waitcnt vmcnt(0)3951; VI-NEXT:    buffer_wbinvl1_vol3952; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v13953; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]3954; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]3955; VI-NEXT:    s_cbranch_execnz .LBB64_13956; VI-NEXT:  ; %bb.2: ; %atomicrmw.end3957; VI-NEXT:    s_or_b64 exec, exec, s[4:5]3958; VI-NEXT:    s_setpc_b64 s[30:31]3959;3960; GFX9-LABEL: global_atomic_or_i32_ret_offset:3961; GFX9:       ; %bb.0:3962; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3963; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:163964; GFX9-NEXT:    s_mov_b64 s[4:5], 03965; GFX9-NEXT:  .LBB64_1: ; %atomicrmw.start3966; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=13967; GFX9-NEXT:    s_waitcnt vmcnt(0)3968; GFX9-NEXT:    v_mov_b32_e32 v4, v33969; GFX9-NEXT:    v_or_b32_e32 v3, v4, v23970; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc3971; GFX9-NEXT:    s_waitcnt vmcnt(0)3972; GFX9-NEXT:    buffer_wbinvl1_vol3973; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v43974; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]3975; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]3976; GFX9-NEXT:    s_cbranch_execnz .LBB64_13977; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end3978; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]3979; GFX9-NEXT:    v_mov_b32_e32 v0, v33980; GFX9-NEXT:    s_setpc_b64 s[30:31]3981  %gep = getelementptr i32, ptr addrspace(1) %out, i32 43982  %result = atomicrmw or ptr addrspace(1) %gep, i32 %in seq_cst3983  ret i32 %result3984}3985 3986define amdgpu_gfx void @global_atomic_or_i32_noret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {3987; SI-LABEL: global_atomic_or_i32_noret_scalar:3988; SI:       ; %bb.0:3989; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3990; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -13991; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill3992; SI-NEXT:    s_mov_b64 exec, s[34:35]3993; SI-NEXT:    s_waitcnt expcnt(0)3994; SI-NEXT:    v_writelane_b32 v4, s6, 03995; SI-NEXT:    v_writelane_b32 v4, s7, 13996; SI-NEXT:    s_mov_b32 s34, s63997; SI-NEXT:    s_mov_b32 s7, 0xf0003998; SI-NEXT:    s_mov_b32 s6, -13999; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 04000; SI-NEXT:    s_mov_b64 s[36:37], 04001; SI-NEXT:  .LBB65_1: ; %atomicrmw.start4002; SI-NEXT:    ; =>This Inner Loop Header: Depth=14003; SI-NEXT:    s_waitcnt vmcnt(0)4004; SI-NEXT:    v_or_b32_e32 v0, s34, v14005; SI-NEXT:    s_waitcnt expcnt(0)4006; SI-NEXT:    v_mov_b32_e32 v3, v14007; SI-NEXT:    v_mov_b32_e32 v2, v04008; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc4009; SI-NEXT:    s_waitcnt vmcnt(0)4010; SI-NEXT:    buffer_wbinvl14011; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v14012; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]4013; SI-NEXT:    v_mov_b32_e32 v1, v24014; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]4015; SI-NEXT:    s_cbranch_execnz .LBB65_14016; SI-NEXT:  ; %bb.2: ; %atomicrmw.end4017; SI-NEXT:    s_or_b64 exec, exec, s[36:37]4018; SI-NEXT:    v_readlane_b32 s7, v4, 14019; SI-NEXT:    v_readlane_b32 s6, v4, 04020; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -14021; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload4022; SI-NEXT:    s_mov_b64 exec, s[34:35]4023; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)4024; SI-NEXT:    s_setpc_b64 s[30:31]4025;4026; VI-LABEL: global_atomic_or_i32_noret_scalar:4027; VI:       ; %bb.0:4028; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4029; VI-NEXT:    v_mov_b32_e32 v0, s44030; VI-NEXT:    v_mov_b32_e32 v1, s54031; VI-NEXT:    flat_load_dword v3, v[0:1]4032; VI-NEXT:    s_mov_b64 s[34:35], 04033; VI-NEXT:  .LBB65_1: ; %atomicrmw.start4034; VI-NEXT:    ; =>This Inner Loop Header: Depth=14035; VI-NEXT:    s_waitcnt vmcnt(0)4036; VI-NEXT:    v_or_b32_e32 v2, s6, v34037; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc4038; VI-NEXT:    s_waitcnt vmcnt(0)4039; VI-NEXT:    buffer_wbinvl1_vol4040; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v34041; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]4042; VI-NEXT:    v_mov_b32_e32 v3, v24043; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]4044; VI-NEXT:    s_cbranch_execnz .LBB65_14045; VI-NEXT:  ; %bb.2: ; %atomicrmw.end4046; VI-NEXT:    s_or_b64 exec, exec, s[34:35]4047; VI-NEXT:    s_setpc_b64 s[30:31]4048;4049; GFX9-LABEL: global_atomic_or_i32_noret_scalar:4050; GFX9:       ; %bb.0:4051; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4052; GFX9-NEXT:    v_mov_b32_e32 v2, 04053; GFX9-NEXT:    global_load_dword v1, v2, s[4:5]4054; GFX9-NEXT:    s_mov_b64 s[34:35], 04055; GFX9-NEXT:  .LBB65_1: ; %atomicrmw.start4056; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=14057; GFX9-NEXT:    s_waitcnt vmcnt(0)4058; GFX9-NEXT:    v_or_b32_e32 v0, s6, v14059; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] glc4060; GFX9-NEXT:    s_waitcnt vmcnt(0)4061; GFX9-NEXT:    buffer_wbinvl1_vol4062; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v14063; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]4064; GFX9-NEXT:    v_mov_b32_e32 v1, v04065; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]4066; GFX9-NEXT:    s_cbranch_execnz .LBB65_14067; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end4068; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]4069; GFX9-NEXT:    s_setpc_b64 s[30:31]4070  %tmp0 = atomicrmw or ptr addrspace(1) %ptr, i32 %in seq_cst4071  ret void4072}4073 4074define amdgpu_gfx void @global_atomic_or_i32_noret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {4075; SI-LABEL: global_atomic_or_i32_noret_offset_scalar:4076; SI:       ; %bb.0:4077; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4078; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -14079; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill4080; SI-NEXT:    s_mov_b64 exec, s[34:35]4081; SI-NEXT:    s_waitcnt expcnt(0)4082; SI-NEXT:    v_writelane_b32 v4, s6, 04083; SI-NEXT:    v_writelane_b32 v4, s7, 14084; SI-NEXT:    s_mov_b32 s34, s64085; SI-NEXT:    s_mov_b32 s7, 0xf0004086; SI-NEXT:    s_mov_b32 s6, -14087; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:164088; SI-NEXT:    s_mov_b64 s[36:37], 04089; SI-NEXT:  .LBB66_1: ; %atomicrmw.start4090; SI-NEXT:    ; =>This Inner Loop Header: Depth=14091; SI-NEXT:    s_waitcnt vmcnt(0)4092; SI-NEXT:    v_or_b32_e32 v0, s34, v14093; SI-NEXT:    s_waitcnt expcnt(0)4094; SI-NEXT:    v_mov_b32_e32 v3, v14095; SI-NEXT:    v_mov_b32_e32 v2, v04096; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 offset:16 glc4097; SI-NEXT:    s_waitcnt vmcnt(0)4098; SI-NEXT:    buffer_wbinvl14099; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v14100; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]4101; SI-NEXT:    v_mov_b32_e32 v1, v24102; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]4103; SI-NEXT:    s_cbranch_execnz .LBB66_14104; SI-NEXT:  ; %bb.2: ; %atomicrmw.end4105; SI-NEXT:    s_or_b64 exec, exec, s[36:37]4106; SI-NEXT:    v_readlane_b32 s7, v4, 14107; SI-NEXT:    v_readlane_b32 s6, v4, 04108; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -14109; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload4110; SI-NEXT:    s_mov_b64 exec, s[34:35]4111; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)4112; SI-NEXT:    s_setpc_b64 s[30:31]4113;4114; VI-LABEL: global_atomic_or_i32_noret_offset_scalar:4115; VI:       ; %bb.0:4116; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4117; VI-NEXT:    s_add_u32 s34, s4, 164118; VI-NEXT:    s_addc_u32 s35, s5, 04119; VI-NEXT:    v_mov_b32_e32 v0, s344120; VI-NEXT:    v_mov_b32_e32 v1, s354121; VI-NEXT:    flat_load_dword v3, v[0:1]4122; VI-NEXT:    s_mov_b64 s[34:35], 04123; VI-NEXT:  .LBB66_1: ; %atomicrmw.start4124; VI-NEXT:    ; =>This Inner Loop Header: Depth=14125; VI-NEXT:    s_waitcnt vmcnt(0)4126; VI-NEXT:    v_or_b32_e32 v2, s6, v34127; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc4128; VI-NEXT:    s_waitcnt vmcnt(0)4129; VI-NEXT:    buffer_wbinvl1_vol4130; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v34131; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]4132; VI-NEXT:    v_mov_b32_e32 v3, v24133; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]4134; VI-NEXT:    s_cbranch_execnz .LBB66_14135; VI-NEXT:  ; %bb.2: ; %atomicrmw.end4136; VI-NEXT:    s_or_b64 exec, exec, s[34:35]4137; VI-NEXT:    s_setpc_b64 s[30:31]4138;4139; GFX9-LABEL: global_atomic_or_i32_noret_offset_scalar:4140; GFX9:       ; %bb.0:4141; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4142; GFX9-NEXT:    v_mov_b32_e32 v2, 04143; GFX9-NEXT:    global_load_dword v1, v2, s[4:5] offset:164144; GFX9-NEXT:    s_mov_b64 s[34:35], 04145; GFX9-NEXT:  .LBB66_1: ; %atomicrmw.start4146; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=14147; GFX9-NEXT:    s_waitcnt vmcnt(0)4148; GFX9-NEXT:    v_or_b32_e32 v0, s6, v14149; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 glc4150; GFX9-NEXT:    s_waitcnt vmcnt(0)4151; GFX9-NEXT:    buffer_wbinvl1_vol4152; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v14153; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]4154; GFX9-NEXT:    v_mov_b32_e32 v1, v04155; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]4156; GFX9-NEXT:    s_cbranch_execnz .LBB66_14157; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end4158; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]4159; GFX9-NEXT:    s_setpc_b64 s[30:31]4160  %gep = getelementptr i32, ptr addrspace(1) %out, i32 44161  %tmp0 = atomicrmw or ptr addrspace(1) %gep, i32 %in seq_cst4162  ret void4163}4164 4165define amdgpu_gfx i32 @global_atomic_or_i32_ret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {4166; SI-LABEL: global_atomic_or_i32_ret_scalar:4167; SI:       ; %bb.0:4168; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4169; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -14170; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill4171; SI-NEXT:    s_mov_b64 exec, s[34:35]4172; SI-NEXT:    s_waitcnt expcnt(0)4173; SI-NEXT:    v_writelane_b32 v3, s6, 04174; SI-NEXT:    v_writelane_b32 v3, s7, 14175; SI-NEXT:    s_mov_b32 s34, s64176; SI-NEXT:    s_mov_b32 s7, 0xf0004177; SI-NEXT:    s_mov_b32 s6, -14178; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 04179; SI-NEXT:    s_mov_b64 s[36:37], 04180; SI-NEXT:  .LBB67_1: ; %atomicrmw.start4181; SI-NEXT:    ; =>This Inner Loop Header: Depth=14182; SI-NEXT:    s_waitcnt vmcnt(0)4183; SI-NEXT:    v_mov_b32_e32 v2, v04184; SI-NEXT:    s_waitcnt expcnt(0)4185; SI-NEXT:    v_or_b32_e32 v1, s34, v24186; SI-NEXT:    v_mov_b32_e32 v0, v14187; SI-NEXT:    v_mov_b32_e32 v1, v24188; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc4189; SI-NEXT:    s_waitcnt vmcnt(0)4190; SI-NEXT:    buffer_wbinvl14191; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v24192; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]4193; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]4194; SI-NEXT:    s_cbranch_execnz .LBB67_14195; SI-NEXT:  ; %bb.2: ; %atomicrmw.end4196; SI-NEXT:    s_or_b64 exec, exec, s[36:37]4197; SI-NEXT:    v_readlane_b32 s7, v3, 14198; SI-NEXT:    v_readlane_b32 s6, v3, 04199; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -14200; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload4201; SI-NEXT:    s_mov_b64 exec, s[34:35]4202; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)4203; SI-NEXT:    s_setpc_b64 s[30:31]4204;4205; VI-LABEL: global_atomic_or_i32_ret_scalar:4206; VI:       ; %bb.0:4207; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4208; VI-NEXT:    v_mov_b32_e32 v0, s44209; VI-NEXT:    v_mov_b32_e32 v1, s54210; VI-NEXT:    flat_load_dword v0, v[0:1]4211; VI-NEXT:    v_mov_b32_e32 v1, s44212; VI-NEXT:    s_mov_b64 s[34:35], 04213; VI-NEXT:    v_mov_b32_e32 v2, s54214; VI-NEXT:  .LBB67_1: ; %atomicrmw.start4215; VI-NEXT:    ; =>This Inner Loop Header: Depth=14216; VI-NEXT:    s_waitcnt vmcnt(0)4217; VI-NEXT:    v_mov_b32_e32 v4, v04218; VI-NEXT:    v_or_b32_e32 v3, s6, v44219; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc4220; VI-NEXT:    s_waitcnt vmcnt(0)4221; VI-NEXT:    buffer_wbinvl1_vol4222; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v44223; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]4224; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]4225; VI-NEXT:    s_cbranch_execnz .LBB67_14226; VI-NEXT:  ; %bb.2: ; %atomicrmw.end4227; VI-NEXT:    s_or_b64 exec, exec, s[34:35]4228; VI-NEXT:    s_setpc_b64 s[30:31]4229;4230; GFX9-LABEL: global_atomic_or_i32_ret_scalar:4231; GFX9:       ; %bb.0:4232; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4233; GFX9-NEXT:    v_mov_b32_e32 v1, 04234; GFX9-NEXT:    global_load_dword v0, v1, s[4:5]4235; GFX9-NEXT:    s_mov_b64 s[34:35], 04236; GFX9-NEXT:  .LBB67_1: ; %atomicrmw.start4237; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=14238; GFX9-NEXT:    s_waitcnt vmcnt(0)4239; GFX9-NEXT:    v_mov_b32_e32 v3, v04240; GFX9-NEXT:    v_or_b32_e32 v2, s6, v34241; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] glc4242; GFX9-NEXT:    s_waitcnt vmcnt(0)4243; GFX9-NEXT:    buffer_wbinvl1_vol4244; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v34245; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]4246; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]4247; GFX9-NEXT:    s_cbranch_execnz .LBB67_14248; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end4249; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]4250; GFX9-NEXT:    s_setpc_b64 s[30:31]4251  %result = atomicrmw or ptr addrspace(1) %ptr, i32 %in seq_cst4252  ret i32 %result4253}4254 4255define amdgpu_gfx i32 @global_atomic_or_i32_ret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {4256; SI-LABEL: global_atomic_or_i32_ret_offset_scalar:4257; SI:       ; %bb.0:4258; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4259; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -14260; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill4261; SI-NEXT:    s_mov_b64 exec, s[34:35]4262; SI-NEXT:    s_waitcnt expcnt(0)4263; SI-NEXT:    v_writelane_b32 v3, s6, 04264; SI-NEXT:    v_writelane_b32 v3, s7, 14265; SI-NEXT:    s_mov_b32 s34, s64266; SI-NEXT:    s_mov_b32 s7, 0xf0004267; SI-NEXT:    s_mov_b32 s6, -14268; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:164269; SI-NEXT:    s_mov_b64 s[36:37], 04270; SI-NEXT:  .LBB68_1: ; %atomicrmw.start4271; SI-NEXT:    ; =>This Inner Loop Header: Depth=14272; SI-NEXT:    s_waitcnt vmcnt(0)4273; SI-NEXT:    v_mov_b32_e32 v2, v04274; SI-NEXT:    s_waitcnt expcnt(0)4275; SI-NEXT:    v_or_b32_e32 v1, s34, v24276; SI-NEXT:    v_mov_b32_e32 v0, v14277; SI-NEXT:    v_mov_b32_e32 v1, v24278; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc4279; SI-NEXT:    s_waitcnt vmcnt(0)4280; SI-NEXT:    buffer_wbinvl14281; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v24282; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]4283; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]4284; SI-NEXT:    s_cbranch_execnz .LBB68_14285; SI-NEXT:  ; %bb.2: ; %atomicrmw.end4286; SI-NEXT:    s_or_b64 exec, exec, s[36:37]4287; SI-NEXT:    v_readlane_b32 s7, v3, 14288; SI-NEXT:    v_readlane_b32 s6, v3, 04289; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -14290; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload4291; SI-NEXT:    s_mov_b64 exec, s[34:35]4292; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)4293; SI-NEXT:    s_setpc_b64 s[30:31]4294;4295; VI-LABEL: global_atomic_or_i32_ret_offset_scalar:4296; VI:       ; %bb.0:4297; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4298; VI-NEXT:    s_add_u32 s34, s4, 164299; VI-NEXT:    s_addc_u32 s35, s5, 04300; VI-NEXT:    v_mov_b32_e32 v1, s344301; VI-NEXT:    v_mov_b32_e32 v2, s354302; VI-NEXT:    flat_load_dword v0, v[1:2]4303; VI-NEXT:    s_mov_b64 s[34:35], 04304; VI-NEXT:  .LBB68_1: ; %atomicrmw.start4305; VI-NEXT:    ; =>This Inner Loop Header: Depth=14306; VI-NEXT:    s_waitcnt vmcnt(0)4307; VI-NEXT:    v_mov_b32_e32 v4, v04308; VI-NEXT:    v_or_b32_e32 v3, s6, v44309; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc4310; VI-NEXT:    s_waitcnt vmcnt(0)4311; VI-NEXT:    buffer_wbinvl1_vol4312; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v44313; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]4314; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]4315; VI-NEXT:    s_cbranch_execnz .LBB68_14316; VI-NEXT:  ; %bb.2: ; %atomicrmw.end4317; VI-NEXT:    s_or_b64 exec, exec, s[34:35]4318; VI-NEXT:    s_setpc_b64 s[30:31]4319;4320; GFX9-LABEL: global_atomic_or_i32_ret_offset_scalar:4321; GFX9:       ; %bb.0:4322; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4323; GFX9-NEXT:    v_mov_b32_e32 v1, 04324; GFX9-NEXT:    global_load_dword v0, v1, s[4:5] offset:164325; GFX9-NEXT:    s_mov_b64 s[34:35], 04326; GFX9-NEXT:  .LBB68_1: ; %atomicrmw.start4327; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=14328; GFX9-NEXT:    s_waitcnt vmcnt(0)4329; GFX9-NEXT:    v_mov_b32_e32 v3, v04330; GFX9-NEXT:    v_or_b32_e32 v2, s6, v34331; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] offset:16 glc4332; GFX9-NEXT:    s_waitcnt vmcnt(0)4333; GFX9-NEXT:    buffer_wbinvl1_vol4334; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v34335; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]4336; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]4337; GFX9-NEXT:    s_cbranch_execnz .LBB68_14338; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end4339; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]4340; GFX9-NEXT:    s_setpc_b64 s[30:31]4341  %gep = getelementptr i32, ptr addrspace(1) %out, i32 44342  %result = atomicrmw or ptr addrspace(1) %gep, i32 %in seq_cst4343  ret i32 %result4344}4345 4346define i32 @global_atomic_or_0_i32_ret(ptr addrspace(1) %ptr) {4347; SI-LABEL: global_atomic_or_0_i32_ret:4348; SI:       ; %bb.0:4349; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4350; SI-NEXT:    s_mov_b32 s7, 0xf0004351; SI-NEXT:    s_mov_b32 s6, 04352; SI-NEXT:    v_mov_b32_e32 v2, 04353; SI-NEXT:    s_mov_b32 s4, s64354; SI-NEXT:    s_mov_b32 s5, s64355; SI-NEXT:    buffer_atomic_add v2, v[0:1], s[4:7], 0 addr64 glc4356; SI-NEXT:    s_waitcnt vmcnt(0)4357; SI-NEXT:    buffer_wbinvl14358; SI-NEXT:    v_mov_b32_e32 v0, v24359; SI-NEXT:    s_waitcnt expcnt(0)4360; SI-NEXT:    s_setpc_b64 s[30:31]4361;4362; VI-LABEL: global_atomic_or_0_i32_ret:4363; VI:       ; %bb.0:4364; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4365; VI-NEXT:    v_mov_b32_e32 v2, 04366; VI-NEXT:    flat_atomic_add v0, v[0:1], v2 glc4367; VI-NEXT:    s_waitcnt vmcnt(0)4368; VI-NEXT:    buffer_wbinvl1_vol4369; VI-NEXT:    s_setpc_b64 s[30:31]4370;4371; GFX9-LABEL: global_atomic_or_0_i32_ret:4372; GFX9:       ; %bb.0:4373; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4374; GFX9-NEXT:    v_mov_b32_e32 v2, 04375; GFX9-NEXT:    global_atomic_add v0, v[0:1], v2, off glc4376; GFX9-NEXT:    s_waitcnt vmcnt(0)4377; GFX9-NEXT:    buffer_wbinvl1_vol4378; GFX9-NEXT:    s_setpc_b64 s[30:31]4379  %result = atomicrmw or ptr addrspace(1) %ptr, i32 0 seq_cst4380  ret i32 %result4381}4382 4383define void @global_atomic_or_i32_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {4384; SI-LABEL: global_atomic_or_i32_noret_offset__amdgpu_no_remote_memory:4385; SI:       ; %bb.0:4386; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4387; SI-NEXT:    s_mov_b32 s6, 04388; SI-NEXT:    s_mov_b32 s7, 0xf0004389; SI-NEXT:    s_mov_b32 s4, s64390; SI-NEXT:    s_mov_b32 s5, s64391; SI-NEXT:    buffer_atomic_or v2, v[0:1], s[4:7], 0 addr64 offset:164392; SI-NEXT:    s_waitcnt vmcnt(0)4393; SI-NEXT:    buffer_wbinvl14394; SI-NEXT:    s_waitcnt expcnt(0)4395; SI-NEXT:    s_setpc_b64 s[30:31]4396;4397; VI-LABEL: global_atomic_or_i32_noret_offset__amdgpu_no_remote_memory:4398; VI:       ; %bb.0:4399; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4400; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v04401; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc4402; VI-NEXT:    flat_atomic_or v[0:1], v24403; VI-NEXT:    s_waitcnt vmcnt(0)4404; VI-NEXT:    buffer_wbinvl1_vol4405; VI-NEXT:    s_setpc_b64 s[30:31]4406;4407; GFX9-LABEL: global_atomic_or_i32_noret_offset__amdgpu_no_remote_memory:4408; GFX9:       ; %bb.0:4409; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4410; GFX9-NEXT:    global_atomic_or v[0:1], v2, off offset:164411; GFX9-NEXT:    s_waitcnt vmcnt(0)4412; GFX9-NEXT:    buffer_wbinvl1_vol4413; GFX9-NEXT:    s_setpc_b64 s[30:31]4414  %gep = getelementptr i32, ptr addrspace(1) %out, i64 44415  %tmp0 = atomicrmw or ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !04416  ret void4417}4418 4419define i32 @global_atomic_or_i32_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {4420; SI-LABEL: global_atomic_or_i32_ret_offset__amdgpu_no_remote_memory:4421; SI:       ; %bb.0:4422; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4423; SI-NEXT:    s_mov_b32 s6, 04424; SI-NEXT:    s_mov_b32 s7, 0xf0004425; SI-NEXT:    s_mov_b32 s4, s64426; SI-NEXT:    s_mov_b32 s5, s64427; SI-NEXT:    buffer_atomic_or v2, v[0:1], s[4:7], 0 addr64 offset:16 glc4428; SI-NEXT:    s_waitcnt vmcnt(0)4429; SI-NEXT:    buffer_wbinvl14430; SI-NEXT:    v_mov_b32_e32 v0, v24431; SI-NEXT:    s_waitcnt expcnt(0)4432; SI-NEXT:    s_setpc_b64 s[30:31]4433;4434; VI-LABEL: global_atomic_or_i32_ret_offset__amdgpu_no_remote_memory:4435; VI:       ; %bb.0:4436; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4437; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v04438; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc4439; VI-NEXT:    flat_atomic_or v0, v[0:1], v2 glc4440; VI-NEXT:    s_waitcnt vmcnt(0)4441; VI-NEXT:    buffer_wbinvl1_vol4442; VI-NEXT:    s_setpc_b64 s[30:31]4443;4444; GFX9-LABEL: global_atomic_or_i32_ret_offset__amdgpu_no_remote_memory:4445; GFX9:       ; %bb.0:4446; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4447; GFX9-NEXT:    global_atomic_or v0, v[0:1], v2, off offset:16 glc4448; GFX9-NEXT:    s_waitcnt vmcnt(0)4449; GFX9-NEXT:    buffer_wbinvl1_vol4450; GFX9-NEXT:    s_setpc_b64 s[30:31]4451  %gep = getelementptr i32, ptr addrspace(1) %out, i64 44452  %result = atomicrmw or ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !04453  ret i32 %result4454}4455 4456; ---------------------------------------------------------------------4457; atomicrmw xor4458; ---------------------------------------------------------------------4459 4460define void @global_atomic_xor_i32_noret(ptr addrspace(1) %ptr, i32 %in) {4461; SI-LABEL: global_atomic_xor_i32_noret:4462; SI:       ; %bb.0:4463; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4464; SI-NEXT:    s_mov_b32 s6, 04465; SI-NEXT:    s_mov_b32 s7, 0xf0004466; SI-NEXT:    s_mov_b32 s4, s64467; SI-NEXT:    s_mov_b32 s5, s64468; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr644469; SI-NEXT:    s_mov_b64 s[8:9], 04470; SI-NEXT:  .LBB72_1: ; %atomicrmw.start4471; SI-NEXT:    ; =>This Inner Loop Header: Depth=14472; SI-NEXT:    s_waitcnt vmcnt(0)4473; SI-NEXT:    v_xor_b32_e32 v3, v4, v24474; SI-NEXT:    s_waitcnt expcnt(0)4475; SI-NEXT:    v_mov_b32_e32 v6, v44476; SI-NEXT:    v_mov_b32_e32 v5, v34477; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc4478; SI-NEXT:    s_waitcnt vmcnt(0)4479; SI-NEXT:    buffer_wbinvl14480; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v44481; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]4482; SI-NEXT:    v_mov_b32_e32 v4, v54483; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]4484; SI-NEXT:    s_cbranch_execnz .LBB72_14485; SI-NEXT:  ; %bb.2: ; %atomicrmw.end4486; SI-NEXT:    s_or_b64 exec, exec, s[8:9]4487; SI-NEXT:    s_waitcnt expcnt(0)4488; SI-NEXT:    s_setpc_b64 s[30:31]4489;4490; VI-LABEL: global_atomic_xor_i32_noret:4491; VI:       ; %bb.0:4492; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4493; VI-NEXT:    flat_load_dword v4, v[0:1]4494; VI-NEXT:    s_mov_b64 s[4:5], 04495; VI-NEXT:  .LBB72_1: ; %atomicrmw.start4496; VI-NEXT:    ; =>This Inner Loop Header: Depth=14497; VI-NEXT:    s_waitcnt vmcnt(0)4498; VI-NEXT:    v_xor_b32_e32 v3, v4, v24499; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc4500; VI-NEXT:    s_waitcnt vmcnt(0)4501; VI-NEXT:    buffer_wbinvl1_vol4502; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v44503; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]4504; VI-NEXT:    v_mov_b32_e32 v4, v34505; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]4506; VI-NEXT:    s_cbranch_execnz .LBB72_14507; VI-NEXT:  ; %bb.2: ; %atomicrmw.end4508; VI-NEXT:    s_or_b64 exec, exec, s[4:5]4509; VI-NEXT:    s_setpc_b64 s[30:31]4510;4511; GFX9-LABEL: global_atomic_xor_i32_noret:4512; GFX9:       ; %bb.0:4513; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4514; GFX9-NEXT:    global_load_dword v4, v[0:1], off4515; GFX9-NEXT:    s_mov_b64 s[4:5], 04516; GFX9-NEXT:  .LBB72_1: ; %atomicrmw.start4517; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=14518; GFX9-NEXT:    s_waitcnt vmcnt(0)4519; GFX9-NEXT:    v_xor_b32_e32 v3, v4, v24520; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc4521; GFX9-NEXT:    s_waitcnt vmcnt(0)4522; GFX9-NEXT:    buffer_wbinvl1_vol4523; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v44524; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]4525; GFX9-NEXT:    v_mov_b32_e32 v4, v34526; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]4527; GFX9-NEXT:    s_cbranch_execnz .LBB72_14528; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end4529; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]4530; GFX9-NEXT:    s_setpc_b64 s[30:31]4531  %tmp0 = atomicrmw xor ptr addrspace(1) %ptr, i32 %in seq_cst4532  ret void4533}4534 4535define void @global_atomic_xor_i32_noret_offset(ptr addrspace(1) %out, i32 %in) {4536; SI-LABEL: global_atomic_xor_i32_noret_offset:4537; SI:       ; %bb.0:4538; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4539; SI-NEXT:    s_mov_b32 s6, 04540; SI-NEXT:    s_mov_b32 s7, 0xf0004541; SI-NEXT:    s_mov_b32 s4, s64542; SI-NEXT:    s_mov_b32 s5, s64543; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:164544; SI-NEXT:    s_mov_b64 s[8:9], 04545; SI-NEXT:  .LBB73_1: ; %atomicrmw.start4546; SI-NEXT:    ; =>This Inner Loop Header: Depth=14547; SI-NEXT:    s_waitcnt vmcnt(0)4548; SI-NEXT:    v_xor_b32_e32 v3, v4, v24549; SI-NEXT:    s_waitcnt expcnt(0)4550; SI-NEXT:    v_mov_b32_e32 v6, v44551; SI-NEXT:    v_mov_b32_e32 v5, v34552; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:16 glc4553; SI-NEXT:    s_waitcnt vmcnt(0)4554; SI-NEXT:    buffer_wbinvl14555; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v44556; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]4557; SI-NEXT:    v_mov_b32_e32 v4, v54558; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]4559; SI-NEXT:    s_cbranch_execnz .LBB73_14560; SI-NEXT:  ; %bb.2: ; %atomicrmw.end4561; SI-NEXT:    s_or_b64 exec, exec, s[8:9]4562; SI-NEXT:    s_waitcnt expcnt(0)4563; SI-NEXT:    s_setpc_b64 s[30:31]4564;4565; VI-LABEL: global_atomic_xor_i32_noret_offset:4566; VI:       ; %bb.0:4567; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4568; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v04569; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc4570; VI-NEXT:    flat_load_dword v4, v[0:1]4571; VI-NEXT:    s_mov_b64 s[4:5], 04572; VI-NEXT:  .LBB73_1: ; %atomicrmw.start4573; VI-NEXT:    ; =>This Inner Loop Header: Depth=14574; VI-NEXT:    s_waitcnt vmcnt(0)4575; VI-NEXT:    v_xor_b32_e32 v3, v4, v24576; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc4577; VI-NEXT:    s_waitcnt vmcnt(0)4578; VI-NEXT:    buffer_wbinvl1_vol4579; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v44580; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]4581; VI-NEXT:    v_mov_b32_e32 v4, v34582; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]4583; VI-NEXT:    s_cbranch_execnz .LBB73_14584; VI-NEXT:  ; %bb.2: ; %atomicrmw.end4585; VI-NEXT:    s_or_b64 exec, exec, s[4:5]4586; VI-NEXT:    s_setpc_b64 s[30:31]4587;4588; GFX9-LABEL: global_atomic_xor_i32_noret_offset:4589; GFX9:       ; %bb.0:4590; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4591; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:164592; GFX9-NEXT:    s_mov_b64 s[4:5], 04593; GFX9-NEXT:  .LBB73_1: ; %atomicrmw.start4594; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=14595; GFX9-NEXT:    s_waitcnt vmcnt(0)4596; GFX9-NEXT:    v_xor_b32_e32 v3, v4, v24597; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc4598; GFX9-NEXT:    s_waitcnt vmcnt(0)4599; GFX9-NEXT:    buffer_wbinvl1_vol4600; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v44601; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]4602; GFX9-NEXT:    v_mov_b32_e32 v4, v34603; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]4604; GFX9-NEXT:    s_cbranch_execnz .LBB73_14605; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end4606; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]4607; GFX9-NEXT:    s_setpc_b64 s[30:31]4608  %gep = getelementptr i32, ptr addrspace(1) %out, i32 44609  %tmp0 = atomicrmw xor ptr addrspace(1) %gep, i32 %in seq_cst4610  ret void4611}4612 4613define i32 @global_atomic_xor_i32_ret(ptr addrspace(1) %ptr, i32 %in) {4614; SI-LABEL: global_atomic_xor_i32_ret:4615; SI:       ; %bb.0:4616; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4617; SI-NEXT:    s_mov_b32 s6, 04618; SI-NEXT:    s_mov_b32 s7, 0xf0004619; SI-NEXT:    s_mov_b32 s4, s64620; SI-NEXT:    s_mov_b32 s5, s64621; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr644622; SI-NEXT:    s_mov_b64 s[8:9], 04623; SI-NEXT:  .LBB74_1: ; %atomicrmw.start4624; SI-NEXT:    ; =>This Inner Loop Header: Depth=14625; SI-NEXT:    s_waitcnt vmcnt(0)4626; SI-NEXT:    v_mov_b32_e32 v5, v34627; SI-NEXT:    s_waitcnt expcnt(0)4628; SI-NEXT:    v_xor_b32_e32 v4, v5, v24629; SI-NEXT:    v_mov_b32_e32 v3, v44630; SI-NEXT:    v_mov_b32_e32 v4, v54631; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc4632; SI-NEXT:    s_waitcnt vmcnt(0)4633; SI-NEXT:    buffer_wbinvl14634; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v54635; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]4636; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]4637; SI-NEXT:    s_cbranch_execnz .LBB74_14638; SI-NEXT:  ; %bb.2: ; %atomicrmw.end4639; SI-NEXT:    s_or_b64 exec, exec, s[8:9]4640; SI-NEXT:    v_mov_b32_e32 v0, v34641; SI-NEXT:    s_waitcnt expcnt(0)4642; SI-NEXT:    s_setpc_b64 s[30:31]4643;4644; VI-LABEL: global_atomic_xor_i32_ret:4645; VI:       ; %bb.0:4646; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4647; VI-NEXT:    flat_load_dword v3, v[0:1]4648; VI-NEXT:    s_mov_b64 s[4:5], 04649; VI-NEXT:  .LBB74_1: ; %atomicrmw.start4650; VI-NEXT:    ; =>This Inner Loop Header: Depth=14651; VI-NEXT:    s_waitcnt vmcnt(0)4652; VI-NEXT:    v_mov_b32_e32 v4, v34653; VI-NEXT:    v_xor_b32_e32 v3, v4, v24654; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc4655; VI-NEXT:    s_waitcnt vmcnt(0)4656; VI-NEXT:    buffer_wbinvl1_vol4657; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v44658; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]4659; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]4660; VI-NEXT:    s_cbranch_execnz .LBB74_14661; VI-NEXT:  ; %bb.2: ; %atomicrmw.end4662; VI-NEXT:    s_or_b64 exec, exec, s[4:5]4663; VI-NEXT:    v_mov_b32_e32 v0, v34664; VI-NEXT:    s_setpc_b64 s[30:31]4665;4666; GFX9-LABEL: global_atomic_xor_i32_ret:4667; GFX9:       ; %bb.0:4668; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4669; GFX9-NEXT:    global_load_dword v3, v[0:1], off4670; GFX9-NEXT:    s_mov_b64 s[4:5], 04671; GFX9-NEXT:  .LBB74_1: ; %atomicrmw.start4672; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=14673; GFX9-NEXT:    s_waitcnt vmcnt(0)4674; GFX9-NEXT:    v_mov_b32_e32 v4, v34675; GFX9-NEXT:    v_xor_b32_e32 v3, v4, v24676; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc4677; GFX9-NEXT:    s_waitcnt vmcnt(0)4678; GFX9-NEXT:    buffer_wbinvl1_vol4679; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v44680; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]4681; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]4682; GFX9-NEXT:    s_cbranch_execnz .LBB74_14683; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end4684; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]4685; GFX9-NEXT:    v_mov_b32_e32 v0, v34686; GFX9-NEXT:    s_setpc_b64 s[30:31]4687  %result = atomicrmw xor ptr addrspace(1) %ptr, i32 %in seq_cst4688  ret i32 %result4689}4690 4691define i32 @global_atomic_xor_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {4692; SI-LABEL: global_atomic_xor_i32_ret_offset:4693; SI:       ; %bb.0:4694; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4695; SI-NEXT:    s_mov_b32 s6, 04696; SI-NEXT:    s_mov_b32 s7, 0xf0004697; SI-NEXT:    s_mov_b32 s4, s64698; SI-NEXT:    s_mov_b32 s5, s64699; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:164700; SI-NEXT:    s_mov_b64 s[8:9], 04701; SI-NEXT:  .LBB75_1: ; %atomicrmw.start4702; SI-NEXT:    ; =>This Inner Loop Header: Depth=14703; SI-NEXT:    s_waitcnt vmcnt(0)4704; SI-NEXT:    v_mov_b32_e32 v5, v34705; SI-NEXT:    s_waitcnt expcnt(0)4706; SI-NEXT:    v_xor_b32_e32 v4, v5, v24707; SI-NEXT:    v_mov_b32_e32 v3, v44708; SI-NEXT:    v_mov_b32_e32 v4, v54709; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:16 glc4710; SI-NEXT:    s_waitcnt vmcnt(0)4711; SI-NEXT:    buffer_wbinvl14712; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v54713; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]4714; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]4715; SI-NEXT:    s_cbranch_execnz .LBB75_14716; SI-NEXT:  ; %bb.2: ; %atomicrmw.end4717; SI-NEXT:    s_or_b64 exec, exec, s[8:9]4718; SI-NEXT:    v_mov_b32_e32 v0, v34719; SI-NEXT:    s_waitcnt expcnt(0)4720; SI-NEXT:    s_setpc_b64 s[30:31]4721;4722; VI-LABEL: global_atomic_xor_i32_ret_offset:4723; VI:       ; %bb.0:4724; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4725; VI-NEXT:    v_add_u32_e32 v3, vcc, 16, v04726; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc4727; VI-NEXT:    flat_load_dword v0, v[3:4]4728; VI-NEXT:    s_mov_b64 s[4:5], 04729; VI-NEXT:  .LBB75_1: ; %atomicrmw.start4730; VI-NEXT:    ; =>This Inner Loop Header: Depth=14731; VI-NEXT:    s_waitcnt vmcnt(0)4732; VI-NEXT:    v_mov_b32_e32 v1, v04733; VI-NEXT:    v_xor_b32_e32 v0, v1, v24734; VI-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc4735; VI-NEXT:    s_waitcnt vmcnt(0)4736; VI-NEXT:    buffer_wbinvl1_vol4737; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v14738; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]4739; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]4740; VI-NEXT:    s_cbranch_execnz .LBB75_14741; VI-NEXT:  ; %bb.2: ; %atomicrmw.end4742; VI-NEXT:    s_or_b64 exec, exec, s[4:5]4743; VI-NEXT:    s_setpc_b64 s[30:31]4744;4745; GFX9-LABEL: global_atomic_xor_i32_ret_offset:4746; GFX9:       ; %bb.0:4747; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4748; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:164749; GFX9-NEXT:    s_mov_b64 s[4:5], 04750; GFX9-NEXT:  .LBB75_1: ; %atomicrmw.start4751; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=14752; GFX9-NEXT:    s_waitcnt vmcnt(0)4753; GFX9-NEXT:    v_mov_b32_e32 v4, v34754; GFX9-NEXT:    v_xor_b32_e32 v3, v4, v24755; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc4756; GFX9-NEXT:    s_waitcnt vmcnt(0)4757; GFX9-NEXT:    buffer_wbinvl1_vol4758; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v44759; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]4760; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]4761; GFX9-NEXT:    s_cbranch_execnz .LBB75_14762; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end4763; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]4764; GFX9-NEXT:    v_mov_b32_e32 v0, v34765; GFX9-NEXT:    s_setpc_b64 s[30:31]4766  %gep = getelementptr i32, ptr addrspace(1) %out, i32 44767  %result = atomicrmw xor ptr addrspace(1) %gep, i32 %in seq_cst4768  ret i32 %result4769}4770 4771define amdgpu_gfx void @global_atomic_xor_i32_noret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {4772; SI-LABEL: global_atomic_xor_i32_noret_scalar:4773; SI:       ; %bb.0:4774; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4775; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -14776; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill4777; SI-NEXT:    s_mov_b64 exec, s[34:35]4778; SI-NEXT:    s_waitcnt expcnt(0)4779; SI-NEXT:    v_writelane_b32 v4, s6, 04780; SI-NEXT:    v_writelane_b32 v4, s7, 14781; SI-NEXT:    s_mov_b32 s34, s64782; SI-NEXT:    s_mov_b32 s7, 0xf0004783; SI-NEXT:    s_mov_b32 s6, -14784; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 04785; SI-NEXT:    s_mov_b64 s[36:37], 04786; SI-NEXT:  .LBB76_1: ; %atomicrmw.start4787; SI-NEXT:    ; =>This Inner Loop Header: Depth=14788; SI-NEXT:    s_waitcnt vmcnt(0)4789; SI-NEXT:    v_xor_b32_e32 v0, s34, v14790; SI-NEXT:    s_waitcnt expcnt(0)4791; SI-NEXT:    v_mov_b32_e32 v3, v14792; SI-NEXT:    v_mov_b32_e32 v2, v04793; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc4794; SI-NEXT:    s_waitcnt vmcnt(0)4795; SI-NEXT:    buffer_wbinvl14796; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v14797; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]4798; SI-NEXT:    v_mov_b32_e32 v1, v24799; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]4800; SI-NEXT:    s_cbranch_execnz .LBB76_14801; SI-NEXT:  ; %bb.2: ; %atomicrmw.end4802; SI-NEXT:    s_or_b64 exec, exec, s[36:37]4803; SI-NEXT:    v_readlane_b32 s7, v4, 14804; SI-NEXT:    v_readlane_b32 s6, v4, 04805; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -14806; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload4807; SI-NEXT:    s_mov_b64 exec, s[34:35]4808; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)4809; SI-NEXT:    s_setpc_b64 s[30:31]4810;4811; VI-LABEL: global_atomic_xor_i32_noret_scalar:4812; VI:       ; %bb.0:4813; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4814; VI-NEXT:    v_mov_b32_e32 v0, s44815; VI-NEXT:    v_mov_b32_e32 v1, s54816; VI-NEXT:    flat_load_dword v3, v[0:1]4817; VI-NEXT:    s_mov_b64 s[34:35], 04818; VI-NEXT:  .LBB76_1: ; %atomicrmw.start4819; VI-NEXT:    ; =>This Inner Loop Header: Depth=14820; VI-NEXT:    s_waitcnt vmcnt(0)4821; VI-NEXT:    v_xor_b32_e32 v2, s6, v34822; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc4823; VI-NEXT:    s_waitcnt vmcnt(0)4824; VI-NEXT:    buffer_wbinvl1_vol4825; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v34826; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]4827; VI-NEXT:    v_mov_b32_e32 v3, v24828; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]4829; VI-NEXT:    s_cbranch_execnz .LBB76_14830; VI-NEXT:  ; %bb.2: ; %atomicrmw.end4831; VI-NEXT:    s_or_b64 exec, exec, s[34:35]4832; VI-NEXT:    s_setpc_b64 s[30:31]4833;4834; GFX9-LABEL: global_atomic_xor_i32_noret_scalar:4835; GFX9:       ; %bb.0:4836; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4837; GFX9-NEXT:    v_mov_b32_e32 v2, 04838; GFX9-NEXT:    global_load_dword v1, v2, s[4:5]4839; GFX9-NEXT:    s_mov_b64 s[34:35], 04840; GFX9-NEXT:  .LBB76_1: ; %atomicrmw.start4841; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=14842; GFX9-NEXT:    s_waitcnt vmcnt(0)4843; GFX9-NEXT:    v_xor_b32_e32 v0, s6, v14844; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] glc4845; GFX9-NEXT:    s_waitcnt vmcnt(0)4846; GFX9-NEXT:    buffer_wbinvl1_vol4847; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v14848; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]4849; GFX9-NEXT:    v_mov_b32_e32 v1, v04850; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]4851; GFX9-NEXT:    s_cbranch_execnz .LBB76_14852; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end4853; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]4854; GFX9-NEXT:    s_setpc_b64 s[30:31]4855  %tmp0 = atomicrmw xor ptr addrspace(1) %ptr, i32 %in seq_cst4856  ret void4857}4858 4859define amdgpu_gfx void @global_atomic_xor_i32_noret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {4860; SI-LABEL: global_atomic_xor_i32_noret_offset_scalar:4861; SI:       ; %bb.0:4862; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4863; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -14864; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill4865; SI-NEXT:    s_mov_b64 exec, s[34:35]4866; SI-NEXT:    s_waitcnt expcnt(0)4867; SI-NEXT:    v_writelane_b32 v4, s6, 04868; SI-NEXT:    v_writelane_b32 v4, s7, 14869; SI-NEXT:    s_mov_b32 s34, s64870; SI-NEXT:    s_mov_b32 s7, 0xf0004871; SI-NEXT:    s_mov_b32 s6, -14872; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:164873; SI-NEXT:    s_mov_b64 s[36:37], 04874; SI-NEXT:  .LBB77_1: ; %atomicrmw.start4875; SI-NEXT:    ; =>This Inner Loop Header: Depth=14876; SI-NEXT:    s_waitcnt vmcnt(0)4877; SI-NEXT:    v_xor_b32_e32 v0, s34, v14878; SI-NEXT:    s_waitcnt expcnt(0)4879; SI-NEXT:    v_mov_b32_e32 v3, v14880; SI-NEXT:    v_mov_b32_e32 v2, v04881; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 offset:16 glc4882; SI-NEXT:    s_waitcnt vmcnt(0)4883; SI-NEXT:    buffer_wbinvl14884; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v14885; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]4886; SI-NEXT:    v_mov_b32_e32 v1, v24887; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]4888; SI-NEXT:    s_cbranch_execnz .LBB77_14889; SI-NEXT:  ; %bb.2: ; %atomicrmw.end4890; SI-NEXT:    s_or_b64 exec, exec, s[36:37]4891; SI-NEXT:    v_readlane_b32 s7, v4, 14892; SI-NEXT:    v_readlane_b32 s6, v4, 04893; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -14894; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload4895; SI-NEXT:    s_mov_b64 exec, s[34:35]4896; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)4897; SI-NEXT:    s_setpc_b64 s[30:31]4898;4899; VI-LABEL: global_atomic_xor_i32_noret_offset_scalar:4900; VI:       ; %bb.0:4901; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4902; VI-NEXT:    s_add_u32 s34, s4, 164903; VI-NEXT:    s_addc_u32 s35, s5, 04904; VI-NEXT:    v_mov_b32_e32 v0, s344905; VI-NEXT:    v_mov_b32_e32 v1, s354906; VI-NEXT:    flat_load_dword v3, v[0:1]4907; VI-NEXT:    s_mov_b64 s[34:35], 04908; VI-NEXT:  .LBB77_1: ; %atomicrmw.start4909; VI-NEXT:    ; =>This Inner Loop Header: Depth=14910; VI-NEXT:    s_waitcnt vmcnt(0)4911; VI-NEXT:    v_xor_b32_e32 v2, s6, v34912; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc4913; VI-NEXT:    s_waitcnt vmcnt(0)4914; VI-NEXT:    buffer_wbinvl1_vol4915; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v34916; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]4917; VI-NEXT:    v_mov_b32_e32 v3, v24918; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]4919; VI-NEXT:    s_cbranch_execnz .LBB77_14920; VI-NEXT:  ; %bb.2: ; %atomicrmw.end4921; VI-NEXT:    s_or_b64 exec, exec, s[34:35]4922; VI-NEXT:    s_setpc_b64 s[30:31]4923;4924; GFX9-LABEL: global_atomic_xor_i32_noret_offset_scalar:4925; GFX9:       ; %bb.0:4926; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4927; GFX9-NEXT:    v_mov_b32_e32 v2, 04928; GFX9-NEXT:    global_load_dword v1, v2, s[4:5] offset:164929; GFX9-NEXT:    s_mov_b64 s[34:35], 04930; GFX9-NEXT:  .LBB77_1: ; %atomicrmw.start4931; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=14932; GFX9-NEXT:    s_waitcnt vmcnt(0)4933; GFX9-NEXT:    v_xor_b32_e32 v0, s6, v14934; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 glc4935; GFX9-NEXT:    s_waitcnt vmcnt(0)4936; GFX9-NEXT:    buffer_wbinvl1_vol4937; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v14938; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]4939; GFX9-NEXT:    v_mov_b32_e32 v1, v04940; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]4941; GFX9-NEXT:    s_cbranch_execnz .LBB77_14942; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end4943; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]4944; GFX9-NEXT:    s_setpc_b64 s[30:31]4945  %gep = getelementptr i32, ptr addrspace(1) %out, i32 44946  %tmp0 = atomicrmw xor ptr addrspace(1) %gep, i32 %in seq_cst4947  ret void4948}4949 4950define amdgpu_gfx i32 @global_atomic_xor_i32_ret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {4951; SI-LABEL: global_atomic_xor_i32_ret_scalar:4952; SI:       ; %bb.0:4953; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4954; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -14955; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill4956; SI-NEXT:    s_mov_b64 exec, s[34:35]4957; SI-NEXT:    s_waitcnt expcnt(0)4958; SI-NEXT:    v_writelane_b32 v3, s6, 04959; SI-NEXT:    v_writelane_b32 v3, s7, 14960; SI-NEXT:    s_mov_b32 s34, s64961; SI-NEXT:    s_mov_b32 s7, 0xf0004962; SI-NEXT:    s_mov_b32 s6, -14963; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 04964; SI-NEXT:    s_mov_b64 s[36:37], 04965; SI-NEXT:  .LBB78_1: ; %atomicrmw.start4966; SI-NEXT:    ; =>This Inner Loop Header: Depth=14967; SI-NEXT:    s_waitcnt vmcnt(0)4968; SI-NEXT:    v_mov_b32_e32 v2, v04969; SI-NEXT:    s_waitcnt expcnt(0)4970; SI-NEXT:    v_xor_b32_e32 v1, s34, v24971; SI-NEXT:    v_mov_b32_e32 v0, v14972; SI-NEXT:    v_mov_b32_e32 v1, v24973; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc4974; SI-NEXT:    s_waitcnt vmcnt(0)4975; SI-NEXT:    buffer_wbinvl14976; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v24977; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]4978; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]4979; SI-NEXT:    s_cbranch_execnz .LBB78_14980; SI-NEXT:  ; %bb.2: ; %atomicrmw.end4981; SI-NEXT:    s_or_b64 exec, exec, s[36:37]4982; SI-NEXT:    v_readlane_b32 s7, v3, 14983; SI-NEXT:    v_readlane_b32 s6, v3, 04984; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -14985; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload4986; SI-NEXT:    s_mov_b64 exec, s[34:35]4987; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)4988; SI-NEXT:    s_setpc_b64 s[30:31]4989;4990; VI-LABEL: global_atomic_xor_i32_ret_scalar:4991; VI:       ; %bb.0:4992; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4993; VI-NEXT:    v_mov_b32_e32 v0, s44994; VI-NEXT:    v_mov_b32_e32 v1, s54995; VI-NEXT:    flat_load_dword v0, v[0:1]4996; VI-NEXT:    v_mov_b32_e32 v1, s44997; VI-NEXT:    s_mov_b64 s[34:35], 04998; VI-NEXT:    v_mov_b32_e32 v2, s54999; VI-NEXT:  .LBB78_1: ; %atomicrmw.start5000; VI-NEXT:    ; =>This Inner Loop Header: Depth=15001; VI-NEXT:    s_waitcnt vmcnt(0)5002; VI-NEXT:    v_mov_b32_e32 v4, v05003; VI-NEXT:    v_xor_b32_e32 v3, s6, v45004; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc5005; VI-NEXT:    s_waitcnt vmcnt(0)5006; VI-NEXT:    buffer_wbinvl1_vol5007; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v45008; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]5009; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]5010; VI-NEXT:    s_cbranch_execnz .LBB78_15011; VI-NEXT:  ; %bb.2: ; %atomicrmw.end5012; VI-NEXT:    s_or_b64 exec, exec, s[34:35]5013; VI-NEXT:    s_setpc_b64 s[30:31]5014;5015; GFX9-LABEL: global_atomic_xor_i32_ret_scalar:5016; GFX9:       ; %bb.0:5017; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5018; GFX9-NEXT:    v_mov_b32_e32 v1, 05019; GFX9-NEXT:    global_load_dword v0, v1, s[4:5]5020; GFX9-NEXT:    s_mov_b64 s[34:35], 05021; GFX9-NEXT:  .LBB78_1: ; %atomicrmw.start5022; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=15023; GFX9-NEXT:    s_waitcnt vmcnt(0)5024; GFX9-NEXT:    v_mov_b32_e32 v3, v05025; GFX9-NEXT:    v_xor_b32_e32 v2, s6, v35026; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] glc5027; GFX9-NEXT:    s_waitcnt vmcnt(0)5028; GFX9-NEXT:    buffer_wbinvl1_vol5029; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v35030; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]5031; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]5032; GFX9-NEXT:    s_cbranch_execnz .LBB78_15033; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end5034; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]5035; GFX9-NEXT:    s_setpc_b64 s[30:31]5036  %result = atomicrmw xor ptr addrspace(1) %ptr, i32 %in seq_cst5037  ret i32 %result5038}5039 5040define amdgpu_gfx i32 @global_atomic_xor_i32_ret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {5041; SI-LABEL: global_atomic_xor_i32_ret_offset_scalar:5042; SI:       ; %bb.0:5043; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5044; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -15045; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill5046; SI-NEXT:    s_mov_b64 exec, s[34:35]5047; SI-NEXT:    s_waitcnt expcnt(0)5048; SI-NEXT:    v_writelane_b32 v3, s6, 05049; SI-NEXT:    v_writelane_b32 v3, s7, 15050; SI-NEXT:    s_mov_b32 s34, s65051; SI-NEXT:    s_mov_b32 s7, 0xf0005052; SI-NEXT:    s_mov_b32 s6, -15053; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:165054; SI-NEXT:    s_mov_b64 s[36:37], 05055; SI-NEXT:  .LBB79_1: ; %atomicrmw.start5056; SI-NEXT:    ; =>This Inner Loop Header: Depth=15057; SI-NEXT:    s_waitcnt vmcnt(0)5058; SI-NEXT:    v_mov_b32_e32 v2, v05059; SI-NEXT:    s_waitcnt expcnt(0)5060; SI-NEXT:    v_xor_b32_e32 v1, s34, v25061; SI-NEXT:    v_mov_b32_e32 v0, v15062; SI-NEXT:    v_mov_b32_e32 v1, v25063; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc5064; SI-NEXT:    s_waitcnt vmcnt(0)5065; SI-NEXT:    buffer_wbinvl15066; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v25067; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]5068; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]5069; SI-NEXT:    s_cbranch_execnz .LBB79_15070; SI-NEXT:  ; %bb.2: ; %atomicrmw.end5071; SI-NEXT:    s_or_b64 exec, exec, s[36:37]5072; SI-NEXT:    v_readlane_b32 s7, v3, 15073; SI-NEXT:    v_readlane_b32 s6, v3, 05074; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -15075; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload5076; SI-NEXT:    s_mov_b64 exec, s[34:35]5077; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)5078; SI-NEXT:    s_setpc_b64 s[30:31]5079;5080; VI-LABEL: global_atomic_xor_i32_ret_offset_scalar:5081; VI:       ; %bb.0:5082; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5083; VI-NEXT:    s_add_u32 s34, s4, 165084; VI-NEXT:    s_addc_u32 s35, s5, 05085; VI-NEXT:    v_mov_b32_e32 v1, s345086; VI-NEXT:    v_mov_b32_e32 v2, s355087; VI-NEXT:    flat_load_dword v0, v[1:2]5088; VI-NEXT:    s_mov_b64 s[34:35], 05089; VI-NEXT:  .LBB79_1: ; %atomicrmw.start5090; VI-NEXT:    ; =>This Inner Loop Header: Depth=15091; VI-NEXT:    s_waitcnt vmcnt(0)5092; VI-NEXT:    v_mov_b32_e32 v4, v05093; VI-NEXT:    v_xor_b32_e32 v3, s6, v45094; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc5095; VI-NEXT:    s_waitcnt vmcnt(0)5096; VI-NEXT:    buffer_wbinvl1_vol5097; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v45098; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]5099; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]5100; VI-NEXT:    s_cbranch_execnz .LBB79_15101; VI-NEXT:  ; %bb.2: ; %atomicrmw.end5102; VI-NEXT:    s_or_b64 exec, exec, s[34:35]5103; VI-NEXT:    s_setpc_b64 s[30:31]5104;5105; GFX9-LABEL: global_atomic_xor_i32_ret_offset_scalar:5106; GFX9:       ; %bb.0:5107; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5108; GFX9-NEXT:    v_mov_b32_e32 v1, 05109; GFX9-NEXT:    global_load_dword v0, v1, s[4:5] offset:165110; GFX9-NEXT:    s_mov_b64 s[34:35], 05111; GFX9-NEXT:  .LBB79_1: ; %atomicrmw.start5112; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=15113; GFX9-NEXT:    s_waitcnt vmcnt(0)5114; GFX9-NEXT:    v_mov_b32_e32 v3, v05115; GFX9-NEXT:    v_xor_b32_e32 v2, s6, v35116; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] offset:16 glc5117; GFX9-NEXT:    s_waitcnt vmcnt(0)5118; GFX9-NEXT:    buffer_wbinvl1_vol5119; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v35120; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]5121; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]5122; GFX9-NEXT:    s_cbranch_execnz .LBB79_15123; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end5124; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]5125; GFX9-NEXT:    s_setpc_b64 s[30:31]5126  %gep = getelementptr i32, ptr addrspace(1) %out, i32 45127  %result = atomicrmw xor ptr addrspace(1) %gep, i32 %in seq_cst5128  ret i32 %result5129}5130 5131define i32 @global_atomic_xor_0_i32_ret(ptr addrspace(1) %ptr) {5132; SI-LABEL: global_atomic_xor_0_i32_ret:5133; SI:       ; %bb.0:5134; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5135; SI-NEXT:    s_mov_b32 s7, 0xf0005136; SI-NEXT:    s_mov_b32 s6, 05137; SI-NEXT:    v_mov_b32_e32 v2, 05138; SI-NEXT:    s_mov_b32 s4, s65139; SI-NEXT:    s_mov_b32 s5, s65140; SI-NEXT:    buffer_atomic_add v2, v[0:1], s[4:7], 0 addr64 glc5141; SI-NEXT:    s_waitcnt vmcnt(0)5142; SI-NEXT:    buffer_wbinvl15143; SI-NEXT:    v_mov_b32_e32 v0, v25144; SI-NEXT:    s_waitcnt expcnt(0)5145; SI-NEXT:    s_setpc_b64 s[30:31]5146;5147; VI-LABEL: global_atomic_xor_0_i32_ret:5148; VI:       ; %bb.0:5149; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5150; VI-NEXT:    v_mov_b32_e32 v2, 05151; VI-NEXT:    flat_atomic_add v0, v[0:1], v2 glc5152; VI-NEXT:    s_waitcnt vmcnt(0)5153; VI-NEXT:    buffer_wbinvl1_vol5154; VI-NEXT:    s_setpc_b64 s[30:31]5155;5156; GFX9-LABEL: global_atomic_xor_0_i32_ret:5157; GFX9:       ; %bb.0:5158; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5159; GFX9-NEXT:    v_mov_b32_e32 v2, 05160; GFX9-NEXT:    global_atomic_add v0, v[0:1], v2, off glc5161; GFX9-NEXT:    s_waitcnt vmcnt(0)5162; GFX9-NEXT:    buffer_wbinvl1_vol5163; GFX9-NEXT:    s_setpc_b64 s[30:31]5164  %result = atomicrmw xor ptr addrspace(1) %ptr, i32 0 seq_cst5165  ret i32 %result5166}5167 5168define void @global_atomic_xor_i32_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {5169; SI-LABEL: global_atomic_xor_i32_noret_offset__amdgpu_no_remote_memory:5170; SI:       ; %bb.0:5171; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5172; SI-NEXT:    s_mov_b32 s6, 05173; SI-NEXT:    s_mov_b32 s7, 0xf0005174; SI-NEXT:    s_mov_b32 s4, s65175; SI-NEXT:    s_mov_b32 s5, s65176; SI-NEXT:    buffer_atomic_xor v2, v[0:1], s[4:7], 0 addr64 offset:165177; SI-NEXT:    s_waitcnt vmcnt(0)5178; SI-NEXT:    buffer_wbinvl15179; SI-NEXT:    s_waitcnt expcnt(0)5180; SI-NEXT:    s_setpc_b64 s[30:31]5181;5182; VI-LABEL: global_atomic_xor_i32_noret_offset__amdgpu_no_remote_memory:5183; VI:       ; %bb.0:5184; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5185; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v05186; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc5187; VI-NEXT:    flat_atomic_xor v[0:1], v25188; VI-NEXT:    s_waitcnt vmcnt(0)5189; VI-NEXT:    buffer_wbinvl1_vol5190; VI-NEXT:    s_setpc_b64 s[30:31]5191;5192; GFX9-LABEL: global_atomic_xor_i32_noret_offset__amdgpu_no_remote_memory:5193; GFX9:       ; %bb.0:5194; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5195; GFX9-NEXT:    global_atomic_xor v[0:1], v2, off offset:165196; GFX9-NEXT:    s_waitcnt vmcnt(0)5197; GFX9-NEXT:    buffer_wbinvl1_vol5198; GFX9-NEXT:    s_setpc_b64 s[30:31]5199  %gep = getelementptr i32, ptr addrspace(1) %out, i64 45200  %tmp0 = atomicrmw xor ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !05201  ret void5202}5203 5204define i32 @global_atomic_xor_i32_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {5205; SI-LABEL: global_atomic_xor_i32_ret_offset__amdgpu_no_remote_memory:5206; SI:       ; %bb.0:5207; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5208; SI-NEXT:    s_mov_b32 s6, 05209; SI-NEXT:    s_mov_b32 s7, 0xf0005210; SI-NEXT:    s_mov_b32 s4, s65211; SI-NEXT:    s_mov_b32 s5, s65212; SI-NEXT:    buffer_atomic_xor v2, v[0:1], s[4:7], 0 addr64 offset:16 glc5213; SI-NEXT:    s_waitcnt vmcnt(0)5214; SI-NEXT:    buffer_wbinvl15215; SI-NEXT:    v_mov_b32_e32 v0, v25216; SI-NEXT:    s_waitcnt expcnt(0)5217; SI-NEXT:    s_setpc_b64 s[30:31]5218;5219; VI-LABEL: global_atomic_xor_i32_ret_offset__amdgpu_no_remote_memory:5220; VI:       ; %bb.0:5221; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5222; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v05223; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc5224; VI-NEXT:    flat_atomic_xor v0, v[0:1], v2 glc5225; VI-NEXT:    s_waitcnt vmcnt(0)5226; VI-NEXT:    buffer_wbinvl1_vol5227; VI-NEXT:    s_setpc_b64 s[30:31]5228;5229; GFX9-LABEL: global_atomic_xor_i32_ret_offset__amdgpu_no_remote_memory:5230; GFX9:       ; %bb.0:5231; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5232; GFX9-NEXT:    global_atomic_xor v0, v[0:1], v2, off offset:16 glc5233; GFX9-NEXT:    s_waitcnt vmcnt(0)5234; GFX9-NEXT:    buffer_wbinvl1_vol5235; GFX9-NEXT:    s_setpc_b64 s[30:31]5236  %gep = getelementptr i32, ptr addrspace(1) %out, i64 45237  %result = atomicrmw xor ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !05238  ret i32 %result5239}5240 5241; ---------------------------------------------------------------------5242; atomicrmw max5243; ---------------------------------------------------------------------5244 5245define void @global_atomic_max_i32_noret(ptr addrspace(1) %ptr, i32 %in) {5246; SI-LABEL: global_atomic_max_i32_noret:5247; SI:       ; %bb.0:5248; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5249; SI-NEXT:    s_mov_b32 s6, 05250; SI-NEXT:    s_mov_b32 s7, 0xf0005251; SI-NEXT:    s_mov_b32 s4, s65252; SI-NEXT:    s_mov_b32 s5, s65253; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr645254; SI-NEXT:    s_mov_b64 s[8:9], 05255; SI-NEXT:  .LBB83_1: ; %atomicrmw.start5256; SI-NEXT:    ; =>This Inner Loop Header: Depth=15257; SI-NEXT:    s_waitcnt vmcnt(0)5258; SI-NEXT:    v_max_i32_e32 v3, v4, v25259; SI-NEXT:    s_waitcnt expcnt(0)5260; SI-NEXT:    v_mov_b32_e32 v6, v45261; SI-NEXT:    v_mov_b32_e32 v5, v35262; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc5263; SI-NEXT:    s_waitcnt vmcnt(0)5264; SI-NEXT:    buffer_wbinvl15265; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v45266; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]5267; SI-NEXT:    v_mov_b32_e32 v4, v55268; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]5269; SI-NEXT:    s_cbranch_execnz .LBB83_15270; SI-NEXT:  ; %bb.2: ; %atomicrmw.end5271; SI-NEXT:    s_or_b64 exec, exec, s[8:9]5272; SI-NEXT:    s_waitcnt expcnt(0)5273; SI-NEXT:    s_setpc_b64 s[30:31]5274;5275; VI-LABEL: global_atomic_max_i32_noret:5276; VI:       ; %bb.0:5277; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5278; VI-NEXT:    flat_load_dword v4, v[0:1]5279; VI-NEXT:    s_mov_b64 s[4:5], 05280; VI-NEXT:  .LBB83_1: ; %atomicrmw.start5281; VI-NEXT:    ; =>This Inner Loop Header: Depth=15282; VI-NEXT:    s_waitcnt vmcnt(0)5283; VI-NEXT:    v_max_i32_e32 v3, v4, v25284; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc5285; VI-NEXT:    s_waitcnt vmcnt(0)5286; VI-NEXT:    buffer_wbinvl1_vol5287; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v45288; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]5289; VI-NEXT:    v_mov_b32_e32 v4, v35290; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]5291; VI-NEXT:    s_cbranch_execnz .LBB83_15292; VI-NEXT:  ; %bb.2: ; %atomicrmw.end5293; VI-NEXT:    s_or_b64 exec, exec, s[4:5]5294; VI-NEXT:    s_setpc_b64 s[30:31]5295;5296; GFX9-LABEL: global_atomic_max_i32_noret:5297; GFX9:       ; %bb.0:5298; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5299; GFX9-NEXT:    global_load_dword v4, v[0:1], off5300; GFX9-NEXT:    s_mov_b64 s[4:5], 05301; GFX9-NEXT:  .LBB83_1: ; %atomicrmw.start5302; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=15303; GFX9-NEXT:    s_waitcnt vmcnt(0)5304; GFX9-NEXT:    v_max_i32_e32 v3, v4, v25305; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc5306; GFX9-NEXT:    s_waitcnt vmcnt(0)5307; GFX9-NEXT:    buffer_wbinvl1_vol5308; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v45309; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]5310; GFX9-NEXT:    v_mov_b32_e32 v4, v35311; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]5312; GFX9-NEXT:    s_cbranch_execnz .LBB83_15313; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end5314; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]5315; GFX9-NEXT:    s_setpc_b64 s[30:31]5316  %tmp0 = atomicrmw max ptr addrspace(1) %ptr, i32 %in seq_cst5317  ret void5318}5319 5320define void @global_atomic_max_i32_noret_offset(ptr addrspace(1) %out, i32 %in) {5321; SI-LABEL: global_atomic_max_i32_noret_offset:5322; SI:       ; %bb.0:5323; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5324; SI-NEXT:    s_mov_b32 s6, 05325; SI-NEXT:    s_mov_b32 s7, 0xf0005326; SI-NEXT:    s_mov_b32 s4, s65327; SI-NEXT:    s_mov_b32 s5, s65328; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:165329; SI-NEXT:    s_mov_b64 s[8:9], 05330; SI-NEXT:  .LBB84_1: ; %atomicrmw.start5331; SI-NEXT:    ; =>This Inner Loop Header: Depth=15332; SI-NEXT:    s_waitcnt vmcnt(0)5333; SI-NEXT:    v_max_i32_e32 v3, v4, v25334; SI-NEXT:    s_waitcnt expcnt(0)5335; SI-NEXT:    v_mov_b32_e32 v6, v45336; SI-NEXT:    v_mov_b32_e32 v5, v35337; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:16 glc5338; SI-NEXT:    s_waitcnt vmcnt(0)5339; SI-NEXT:    buffer_wbinvl15340; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v45341; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]5342; SI-NEXT:    v_mov_b32_e32 v4, v55343; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]5344; SI-NEXT:    s_cbranch_execnz .LBB84_15345; SI-NEXT:  ; %bb.2: ; %atomicrmw.end5346; SI-NEXT:    s_or_b64 exec, exec, s[8:9]5347; SI-NEXT:    s_waitcnt expcnt(0)5348; SI-NEXT:    s_setpc_b64 s[30:31]5349;5350; VI-LABEL: global_atomic_max_i32_noret_offset:5351; VI:       ; %bb.0:5352; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5353; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v05354; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc5355; VI-NEXT:    flat_load_dword v4, v[0:1]5356; VI-NEXT:    s_mov_b64 s[4:5], 05357; VI-NEXT:  .LBB84_1: ; %atomicrmw.start5358; VI-NEXT:    ; =>This Inner Loop Header: Depth=15359; VI-NEXT:    s_waitcnt vmcnt(0)5360; VI-NEXT:    v_max_i32_e32 v3, v4, v25361; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc5362; VI-NEXT:    s_waitcnt vmcnt(0)5363; VI-NEXT:    buffer_wbinvl1_vol5364; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v45365; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]5366; VI-NEXT:    v_mov_b32_e32 v4, v35367; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]5368; VI-NEXT:    s_cbranch_execnz .LBB84_15369; VI-NEXT:  ; %bb.2: ; %atomicrmw.end5370; VI-NEXT:    s_or_b64 exec, exec, s[4:5]5371; VI-NEXT:    s_setpc_b64 s[30:31]5372;5373; GFX9-LABEL: global_atomic_max_i32_noret_offset:5374; GFX9:       ; %bb.0:5375; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5376; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:165377; GFX9-NEXT:    s_mov_b64 s[4:5], 05378; GFX9-NEXT:  .LBB84_1: ; %atomicrmw.start5379; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=15380; GFX9-NEXT:    s_waitcnt vmcnt(0)5381; GFX9-NEXT:    v_max_i32_e32 v3, v4, v25382; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc5383; GFX9-NEXT:    s_waitcnt vmcnt(0)5384; GFX9-NEXT:    buffer_wbinvl1_vol5385; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v45386; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]5387; GFX9-NEXT:    v_mov_b32_e32 v4, v35388; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]5389; GFX9-NEXT:    s_cbranch_execnz .LBB84_15390; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end5391; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]5392; GFX9-NEXT:    s_setpc_b64 s[30:31]5393  %gep = getelementptr i32, ptr addrspace(1) %out, i32 45394  %tmp0 = atomicrmw max ptr addrspace(1) %gep, i32 %in seq_cst5395  ret void5396}5397 5398define i32 @global_atomic_max_i32_ret(ptr addrspace(1) %ptr, i32 %in) {5399; SI-LABEL: global_atomic_max_i32_ret:5400; SI:       ; %bb.0:5401; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5402; SI-NEXT:    s_mov_b32 s6, 05403; SI-NEXT:    s_mov_b32 s7, 0xf0005404; SI-NEXT:    s_mov_b32 s4, s65405; SI-NEXT:    s_mov_b32 s5, s65406; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr645407; SI-NEXT:    s_mov_b64 s[8:9], 05408; SI-NEXT:  .LBB85_1: ; %atomicrmw.start5409; SI-NEXT:    ; =>This Inner Loop Header: Depth=15410; SI-NEXT:    s_waitcnt vmcnt(0)5411; SI-NEXT:    v_mov_b32_e32 v5, v35412; SI-NEXT:    s_waitcnt expcnt(0)5413; SI-NEXT:    v_max_i32_e32 v4, v5, v25414; SI-NEXT:    v_mov_b32_e32 v3, v45415; SI-NEXT:    v_mov_b32_e32 v4, v55416; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc5417; SI-NEXT:    s_waitcnt vmcnt(0)5418; SI-NEXT:    buffer_wbinvl15419; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v55420; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]5421; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]5422; SI-NEXT:    s_cbranch_execnz .LBB85_15423; SI-NEXT:  ; %bb.2: ; %atomicrmw.end5424; SI-NEXT:    s_or_b64 exec, exec, s[8:9]5425; SI-NEXT:    v_mov_b32_e32 v0, v35426; SI-NEXT:    s_waitcnt expcnt(0)5427; SI-NEXT:    s_setpc_b64 s[30:31]5428;5429; VI-LABEL: global_atomic_max_i32_ret:5430; VI:       ; %bb.0:5431; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5432; VI-NEXT:    flat_load_dword v3, v[0:1]5433; VI-NEXT:    s_mov_b64 s[4:5], 05434; VI-NEXT:  .LBB85_1: ; %atomicrmw.start5435; VI-NEXT:    ; =>This Inner Loop Header: Depth=15436; VI-NEXT:    s_waitcnt vmcnt(0)5437; VI-NEXT:    v_mov_b32_e32 v4, v35438; VI-NEXT:    v_max_i32_e32 v3, v4, v25439; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc5440; VI-NEXT:    s_waitcnt vmcnt(0)5441; VI-NEXT:    buffer_wbinvl1_vol5442; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v45443; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]5444; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]5445; VI-NEXT:    s_cbranch_execnz .LBB85_15446; VI-NEXT:  ; %bb.2: ; %atomicrmw.end5447; VI-NEXT:    s_or_b64 exec, exec, s[4:5]5448; VI-NEXT:    v_mov_b32_e32 v0, v35449; VI-NEXT:    s_setpc_b64 s[30:31]5450;5451; GFX9-LABEL: global_atomic_max_i32_ret:5452; GFX9:       ; %bb.0:5453; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5454; GFX9-NEXT:    global_load_dword v3, v[0:1], off5455; GFX9-NEXT:    s_mov_b64 s[4:5], 05456; GFX9-NEXT:  .LBB85_1: ; %atomicrmw.start5457; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=15458; GFX9-NEXT:    s_waitcnt vmcnt(0)5459; GFX9-NEXT:    v_mov_b32_e32 v4, v35460; GFX9-NEXT:    v_max_i32_e32 v3, v4, v25461; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc5462; GFX9-NEXT:    s_waitcnt vmcnt(0)5463; GFX9-NEXT:    buffer_wbinvl1_vol5464; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v45465; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]5466; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]5467; GFX9-NEXT:    s_cbranch_execnz .LBB85_15468; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end5469; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]5470; GFX9-NEXT:    v_mov_b32_e32 v0, v35471; GFX9-NEXT:    s_setpc_b64 s[30:31]5472  %result = atomicrmw max ptr addrspace(1) %ptr, i32 %in seq_cst5473  ret i32 %result5474}5475 5476define i32 @global_atomic_max_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {5477; SI-LABEL: global_atomic_max_i32_ret_offset:5478; SI:       ; %bb.0:5479; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5480; SI-NEXT:    s_mov_b32 s6, 05481; SI-NEXT:    s_mov_b32 s7, 0xf0005482; SI-NEXT:    s_mov_b32 s4, s65483; SI-NEXT:    s_mov_b32 s5, s65484; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:165485; SI-NEXT:    s_mov_b64 s[8:9], 05486; SI-NEXT:  .LBB86_1: ; %atomicrmw.start5487; SI-NEXT:    ; =>This Inner Loop Header: Depth=15488; SI-NEXT:    s_waitcnt vmcnt(0)5489; SI-NEXT:    v_mov_b32_e32 v5, v35490; SI-NEXT:    s_waitcnt expcnt(0)5491; SI-NEXT:    v_max_i32_e32 v4, v5, v25492; SI-NEXT:    v_mov_b32_e32 v3, v45493; SI-NEXT:    v_mov_b32_e32 v4, v55494; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:16 glc5495; SI-NEXT:    s_waitcnt vmcnt(0)5496; SI-NEXT:    buffer_wbinvl15497; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v55498; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]5499; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]5500; SI-NEXT:    s_cbranch_execnz .LBB86_15501; SI-NEXT:  ; %bb.2: ; %atomicrmw.end5502; SI-NEXT:    s_or_b64 exec, exec, s[8:9]5503; SI-NEXT:    v_mov_b32_e32 v0, v35504; SI-NEXT:    s_waitcnt expcnt(0)5505; SI-NEXT:    s_setpc_b64 s[30:31]5506;5507; VI-LABEL: global_atomic_max_i32_ret_offset:5508; VI:       ; %bb.0:5509; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5510; VI-NEXT:    v_add_u32_e32 v3, vcc, 16, v05511; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc5512; VI-NEXT:    flat_load_dword v0, v[3:4]5513; VI-NEXT:    s_mov_b64 s[4:5], 05514; VI-NEXT:  .LBB86_1: ; %atomicrmw.start5515; VI-NEXT:    ; =>This Inner Loop Header: Depth=15516; VI-NEXT:    s_waitcnt vmcnt(0)5517; VI-NEXT:    v_mov_b32_e32 v1, v05518; VI-NEXT:    v_max_i32_e32 v0, v1, v25519; VI-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc5520; VI-NEXT:    s_waitcnt vmcnt(0)5521; VI-NEXT:    buffer_wbinvl1_vol5522; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v15523; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]5524; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]5525; VI-NEXT:    s_cbranch_execnz .LBB86_15526; VI-NEXT:  ; %bb.2: ; %atomicrmw.end5527; VI-NEXT:    s_or_b64 exec, exec, s[4:5]5528; VI-NEXT:    s_setpc_b64 s[30:31]5529;5530; GFX9-LABEL: global_atomic_max_i32_ret_offset:5531; GFX9:       ; %bb.0:5532; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5533; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:165534; GFX9-NEXT:    s_mov_b64 s[4:5], 05535; GFX9-NEXT:  .LBB86_1: ; %atomicrmw.start5536; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=15537; GFX9-NEXT:    s_waitcnt vmcnt(0)5538; GFX9-NEXT:    v_mov_b32_e32 v4, v35539; GFX9-NEXT:    v_max_i32_e32 v3, v4, v25540; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc5541; GFX9-NEXT:    s_waitcnt vmcnt(0)5542; GFX9-NEXT:    buffer_wbinvl1_vol5543; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v45544; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]5545; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]5546; GFX9-NEXT:    s_cbranch_execnz .LBB86_15547; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end5548; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]5549; GFX9-NEXT:    v_mov_b32_e32 v0, v35550; GFX9-NEXT:    s_setpc_b64 s[30:31]5551  %gep = getelementptr i32, ptr addrspace(1) %out, i32 45552  %result = atomicrmw max ptr addrspace(1) %gep, i32 %in seq_cst5553  ret i32 %result5554}5555 5556define amdgpu_gfx void @global_atomic_max_i32_noret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {5557; SI-LABEL: global_atomic_max_i32_noret_scalar:5558; SI:       ; %bb.0:5559; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5560; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -15561; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill5562; SI-NEXT:    s_mov_b64 exec, s[34:35]5563; SI-NEXT:    s_waitcnt expcnt(0)5564; SI-NEXT:    v_writelane_b32 v4, s6, 05565; SI-NEXT:    v_writelane_b32 v4, s7, 15566; SI-NEXT:    s_mov_b32 s34, s65567; SI-NEXT:    s_mov_b32 s7, 0xf0005568; SI-NEXT:    s_mov_b32 s6, -15569; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 05570; SI-NEXT:    s_mov_b64 s[36:37], 05571; SI-NEXT:  .LBB87_1: ; %atomicrmw.start5572; SI-NEXT:    ; =>This Inner Loop Header: Depth=15573; SI-NEXT:    s_waitcnt vmcnt(0)5574; SI-NEXT:    v_max_i32_e32 v0, s34, v15575; SI-NEXT:    s_waitcnt expcnt(0)5576; SI-NEXT:    v_mov_b32_e32 v3, v15577; SI-NEXT:    v_mov_b32_e32 v2, v05578; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc5579; SI-NEXT:    s_waitcnt vmcnt(0)5580; SI-NEXT:    buffer_wbinvl15581; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v15582; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]5583; SI-NEXT:    v_mov_b32_e32 v1, v25584; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]5585; SI-NEXT:    s_cbranch_execnz .LBB87_15586; SI-NEXT:  ; %bb.2: ; %atomicrmw.end5587; SI-NEXT:    s_or_b64 exec, exec, s[36:37]5588; SI-NEXT:    v_readlane_b32 s7, v4, 15589; SI-NEXT:    v_readlane_b32 s6, v4, 05590; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -15591; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload5592; SI-NEXT:    s_mov_b64 exec, s[34:35]5593; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)5594; SI-NEXT:    s_setpc_b64 s[30:31]5595;5596; VI-LABEL: global_atomic_max_i32_noret_scalar:5597; VI:       ; %bb.0:5598; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5599; VI-NEXT:    v_mov_b32_e32 v0, s45600; VI-NEXT:    v_mov_b32_e32 v1, s55601; VI-NEXT:    flat_load_dword v3, v[0:1]5602; VI-NEXT:    s_mov_b64 s[34:35], 05603; VI-NEXT:  .LBB87_1: ; %atomicrmw.start5604; VI-NEXT:    ; =>This Inner Loop Header: Depth=15605; VI-NEXT:    s_waitcnt vmcnt(0)5606; VI-NEXT:    v_max_i32_e32 v2, s6, v35607; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc5608; VI-NEXT:    s_waitcnt vmcnt(0)5609; VI-NEXT:    buffer_wbinvl1_vol5610; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v35611; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]5612; VI-NEXT:    v_mov_b32_e32 v3, v25613; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]5614; VI-NEXT:    s_cbranch_execnz .LBB87_15615; VI-NEXT:  ; %bb.2: ; %atomicrmw.end5616; VI-NEXT:    s_or_b64 exec, exec, s[34:35]5617; VI-NEXT:    s_setpc_b64 s[30:31]5618;5619; GFX9-LABEL: global_atomic_max_i32_noret_scalar:5620; GFX9:       ; %bb.0:5621; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5622; GFX9-NEXT:    v_mov_b32_e32 v2, 05623; GFX9-NEXT:    global_load_dword v1, v2, s[4:5]5624; GFX9-NEXT:    s_mov_b64 s[34:35], 05625; GFX9-NEXT:  .LBB87_1: ; %atomicrmw.start5626; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=15627; GFX9-NEXT:    s_waitcnt vmcnt(0)5628; GFX9-NEXT:    v_max_i32_e32 v0, s6, v15629; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] glc5630; GFX9-NEXT:    s_waitcnt vmcnt(0)5631; GFX9-NEXT:    buffer_wbinvl1_vol5632; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v15633; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]5634; GFX9-NEXT:    v_mov_b32_e32 v1, v05635; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]5636; GFX9-NEXT:    s_cbranch_execnz .LBB87_15637; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end5638; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]5639; GFX9-NEXT:    s_setpc_b64 s[30:31]5640  %tmp0 = atomicrmw max ptr addrspace(1) %ptr, i32 %in seq_cst5641  ret void5642}5643 5644define amdgpu_gfx void @global_atomic_max_i32_noret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {5645; SI-LABEL: global_atomic_max_i32_noret_offset_scalar:5646; SI:       ; %bb.0:5647; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5648; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -15649; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill5650; SI-NEXT:    s_mov_b64 exec, s[34:35]5651; SI-NEXT:    s_waitcnt expcnt(0)5652; SI-NEXT:    v_writelane_b32 v4, s6, 05653; SI-NEXT:    v_writelane_b32 v4, s7, 15654; SI-NEXT:    s_mov_b32 s34, s65655; SI-NEXT:    s_mov_b32 s7, 0xf0005656; SI-NEXT:    s_mov_b32 s6, -15657; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:165658; SI-NEXT:    s_mov_b64 s[36:37], 05659; SI-NEXT:  .LBB88_1: ; %atomicrmw.start5660; SI-NEXT:    ; =>This Inner Loop Header: Depth=15661; SI-NEXT:    s_waitcnt vmcnt(0)5662; SI-NEXT:    v_max_i32_e32 v0, s34, v15663; SI-NEXT:    s_waitcnt expcnt(0)5664; SI-NEXT:    v_mov_b32_e32 v3, v15665; SI-NEXT:    v_mov_b32_e32 v2, v05666; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 offset:16 glc5667; SI-NEXT:    s_waitcnt vmcnt(0)5668; SI-NEXT:    buffer_wbinvl15669; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v15670; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]5671; SI-NEXT:    v_mov_b32_e32 v1, v25672; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]5673; SI-NEXT:    s_cbranch_execnz .LBB88_15674; SI-NEXT:  ; %bb.2: ; %atomicrmw.end5675; SI-NEXT:    s_or_b64 exec, exec, s[36:37]5676; SI-NEXT:    v_readlane_b32 s7, v4, 15677; SI-NEXT:    v_readlane_b32 s6, v4, 05678; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -15679; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload5680; SI-NEXT:    s_mov_b64 exec, s[34:35]5681; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)5682; SI-NEXT:    s_setpc_b64 s[30:31]5683;5684; VI-LABEL: global_atomic_max_i32_noret_offset_scalar:5685; VI:       ; %bb.0:5686; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5687; VI-NEXT:    s_add_u32 s34, s4, 165688; VI-NEXT:    s_addc_u32 s35, s5, 05689; VI-NEXT:    v_mov_b32_e32 v0, s345690; VI-NEXT:    v_mov_b32_e32 v1, s355691; VI-NEXT:    flat_load_dword v3, v[0:1]5692; VI-NEXT:    s_mov_b64 s[34:35], 05693; VI-NEXT:  .LBB88_1: ; %atomicrmw.start5694; VI-NEXT:    ; =>This Inner Loop Header: Depth=15695; VI-NEXT:    s_waitcnt vmcnt(0)5696; VI-NEXT:    v_max_i32_e32 v2, s6, v35697; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc5698; VI-NEXT:    s_waitcnt vmcnt(0)5699; VI-NEXT:    buffer_wbinvl1_vol5700; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v35701; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]5702; VI-NEXT:    v_mov_b32_e32 v3, v25703; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]5704; VI-NEXT:    s_cbranch_execnz .LBB88_15705; VI-NEXT:  ; %bb.2: ; %atomicrmw.end5706; VI-NEXT:    s_or_b64 exec, exec, s[34:35]5707; VI-NEXT:    s_setpc_b64 s[30:31]5708;5709; GFX9-LABEL: global_atomic_max_i32_noret_offset_scalar:5710; GFX9:       ; %bb.0:5711; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5712; GFX9-NEXT:    v_mov_b32_e32 v2, 05713; GFX9-NEXT:    global_load_dword v1, v2, s[4:5] offset:165714; GFX9-NEXT:    s_mov_b64 s[34:35], 05715; GFX9-NEXT:  .LBB88_1: ; %atomicrmw.start5716; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=15717; GFX9-NEXT:    s_waitcnt vmcnt(0)5718; GFX9-NEXT:    v_max_i32_e32 v0, s6, v15719; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 glc5720; GFX9-NEXT:    s_waitcnt vmcnt(0)5721; GFX9-NEXT:    buffer_wbinvl1_vol5722; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v15723; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]5724; GFX9-NEXT:    v_mov_b32_e32 v1, v05725; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]5726; GFX9-NEXT:    s_cbranch_execnz .LBB88_15727; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end5728; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]5729; GFX9-NEXT:    s_setpc_b64 s[30:31]5730  %gep = getelementptr i32, ptr addrspace(1) %out, i32 45731  %tmp0 = atomicrmw max ptr addrspace(1) %gep, i32 %in seq_cst5732  ret void5733}5734 5735define amdgpu_gfx i32 @global_atomic_max_i32_ret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {5736; SI-LABEL: global_atomic_max_i32_ret_scalar:5737; SI:       ; %bb.0:5738; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5739; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -15740; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill5741; SI-NEXT:    s_mov_b64 exec, s[34:35]5742; SI-NEXT:    s_waitcnt expcnt(0)5743; SI-NEXT:    v_writelane_b32 v3, s6, 05744; SI-NEXT:    v_writelane_b32 v3, s7, 15745; SI-NEXT:    s_mov_b32 s34, s65746; SI-NEXT:    s_mov_b32 s7, 0xf0005747; SI-NEXT:    s_mov_b32 s6, -15748; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 05749; SI-NEXT:    s_mov_b64 s[36:37], 05750; SI-NEXT:  .LBB89_1: ; %atomicrmw.start5751; SI-NEXT:    ; =>This Inner Loop Header: Depth=15752; SI-NEXT:    s_waitcnt vmcnt(0)5753; SI-NEXT:    v_mov_b32_e32 v2, v05754; SI-NEXT:    s_waitcnt expcnt(0)5755; SI-NEXT:    v_max_i32_e32 v1, s34, v25756; SI-NEXT:    v_mov_b32_e32 v0, v15757; SI-NEXT:    v_mov_b32_e32 v1, v25758; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc5759; SI-NEXT:    s_waitcnt vmcnt(0)5760; SI-NEXT:    buffer_wbinvl15761; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v25762; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]5763; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]5764; SI-NEXT:    s_cbranch_execnz .LBB89_15765; SI-NEXT:  ; %bb.2: ; %atomicrmw.end5766; SI-NEXT:    s_or_b64 exec, exec, s[36:37]5767; SI-NEXT:    v_readlane_b32 s7, v3, 15768; SI-NEXT:    v_readlane_b32 s6, v3, 05769; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -15770; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload5771; SI-NEXT:    s_mov_b64 exec, s[34:35]5772; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)5773; SI-NEXT:    s_setpc_b64 s[30:31]5774;5775; VI-LABEL: global_atomic_max_i32_ret_scalar:5776; VI:       ; %bb.0:5777; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5778; VI-NEXT:    v_mov_b32_e32 v0, s45779; VI-NEXT:    v_mov_b32_e32 v1, s55780; VI-NEXT:    flat_load_dword v0, v[0:1]5781; VI-NEXT:    v_mov_b32_e32 v1, s45782; VI-NEXT:    s_mov_b64 s[34:35], 05783; VI-NEXT:    v_mov_b32_e32 v2, s55784; VI-NEXT:  .LBB89_1: ; %atomicrmw.start5785; VI-NEXT:    ; =>This Inner Loop Header: Depth=15786; VI-NEXT:    s_waitcnt vmcnt(0)5787; VI-NEXT:    v_mov_b32_e32 v4, v05788; VI-NEXT:    v_max_i32_e32 v3, s6, v45789; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc5790; VI-NEXT:    s_waitcnt vmcnt(0)5791; VI-NEXT:    buffer_wbinvl1_vol5792; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v45793; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]5794; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]5795; VI-NEXT:    s_cbranch_execnz .LBB89_15796; VI-NEXT:  ; %bb.2: ; %atomicrmw.end5797; VI-NEXT:    s_or_b64 exec, exec, s[34:35]5798; VI-NEXT:    s_setpc_b64 s[30:31]5799;5800; GFX9-LABEL: global_atomic_max_i32_ret_scalar:5801; GFX9:       ; %bb.0:5802; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5803; GFX9-NEXT:    v_mov_b32_e32 v1, 05804; GFX9-NEXT:    global_load_dword v0, v1, s[4:5]5805; GFX9-NEXT:    s_mov_b64 s[34:35], 05806; GFX9-NEXT:  .LBB89_1: ; %atomicrmw.start5807; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=15808; GFX9-NEXT:    s_waitcnt vmcnt(0)5809; GFX9-NEXT:    v_mov_b32_e32 v3, v05810; GFX9-NEXT:    v_max_i32_e32 v2, s6, v35811; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] glc5812; GFX9-NEXT:    s_waitcnt vmcnt(0)5813; GFX9-NEXT:    buffer_wbinvl1_vol5814; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v35815; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]5816; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]5817; GFX9-NEXT:    s_cbranch_execnz .LBB89_15818; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end5819; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]5820; GFX9-NEXT:    s_setpc_b64 s[30:31]5821  %result = atomicrmw max ptr addrspace(1) %ptr, i32 %in seq_cst5822  ret i32 %result5823}5824 5825define amdgpu_gfx i32 @global_atomic_max_i32_ret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {5826; SI-LABEL: global_atomic_max_i32_ret_offset_scalar:5827; SI:       ; %bb.0:5828; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5829; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -15830; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill5831; SI-NEXT:    s_mov_b64 exec, s[34:35]5832; SI-NEXT:    s_waitcnt expcnt(0)5833; SI-NEXT:    v_writelane_b32 v3, s6, 05834; SI-NEXT:    v_writelane_b32 v3, s7, 15835; SI-NEXT:    s_mov_b32 s34, s65836; SI-NEXT:    s_mov_b32 s7, 0xf0005837; SI-NEXT:    s_mov_b32 s6, -15838; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:165839; SI-NEXT:    s_mov_b64 s[36:37], 05840; SI-NEXT:  .LBB90_1: ; %atomicrmw.start5841; SI-NEXT:    ; =>This Inner Loop Header: Depth=15842; SI-NEXT:    s_waitcnt vmcnt(0)5843; SI-NEXT:    v_mov_b32_e32 v2, v05844; SI-NEXT:    s_waitcnt expcnt(0)5845; SI-NEXT:    v_max_i32_e32 v1, s34, v25846; SI-NEXT:    v_mov_b32_e32 v0, v15847; SI-NEXT:    v_mov_b32_e32 v1, v25848; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc5849; SI-NEXT:    s_waitcnt vmcnt(0)5850; SI-NEXT:    buffer_wbinvl15851; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v25852; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]5853; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]5854; SI-NEXT:    s_cbranch_execnz .LBB90_15855; SI-NEXT:  ; %bb.2: ; %atomicrmw.end5856; SI-NEXT:    s_or_b64 exec, exec, s[36:37]5857; SI-NEXT:    v_readlane_b32 s7, v3, 15858; SI-NEXT:    v_readlane_b32 s6, v3, 05859; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -15860; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload5861; SI-NEXT:    s_mov_b64 exec, s[34:35]5862; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)5863; SI-NEXT:    s_setpc_b64 s[30:31]5864;5865; VI-LABEL: global_atomic_max_i32_ret_offset_scalar:5866; VI:       ; %bb.0:5867; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5868; VI-NEXT:    s_add_u32 s34, s4, 165869; VI-NEXT:    s_addc_u32 s35, s5, 05870; VI-NEXT:    v_mov_b32_e32 v1, s345871; VI-NEXT:    v_mov_b32_e32 v2, s355872; VI-NEXT:    flat_load_dword v0, v[1:2]5873; VI-NEXT:    s_mov_b64 s[34:35], 05874; VI-NEXT:  .LBB90_1: ; %atomicrmw.start5875; VI-NEXT:    ; =>This Inner Loop Header: Depth=15876; VI-NEXT:    s_waitcnt vmcnt(0)5877; VI-NEXT:    v_mov_b32_e32 v4, v05878; VI-NEXT:    v_max_i32_e32 v3, s6, v45879; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc5880; VI-NEXT:    s_waitcnt vmcnt(0)5881; VI-NEXT:    buffer_wbinvl1_vol5882; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v45883; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]5884; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]5885; VI-NEXT:    s_cbranch_execnz .LBB90_15886; VI-NEXT:  ; %bb.2: ; %atomicrmw.end5887; VI-NEXT:    s_or_b64 exec, exec, s[34:35]5888; VI-NEXT:    s_setpc_b64 s[30:31]5889;5890; GFX9-LABEL: global_atomic_max_i32_ret_offset_scalar:5891; GFX9:       ; %bb.0:5892; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5893; GFX9-NEXT:    v_mov_b32_e32 v1, 05894; GFX9-NEXT:    global_load_dword v0, v1, s[4:5] offset:165895; GFX9-NEXT:    s_mov_b64 s[34:35], 05896; GFX9-NEXT:  .LBB90_1: ; %atomicrmw.start5897; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=15898; GFX9-NEXT:    s_waitcnt vmcnt(0)5899; GFX9-NEXT:    v_mov_b32_e32 v3, v05900; GFX9-NEXT:    v_max_i32_e32 v2, s6, v35901; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] offset:16 glc5902; GFX9-NEXT:    s_waitcnt vmcnt(0)5903; GFX9-NEXT:    buffer_wbinvl1_vol5904; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v35905; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]5906; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]5907; GFX9-NEXT:    s_cbranch_execnz .LBB90_15908; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end5909; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]5910; GFX9-NEXT:    s_setpc_b64 s[30:31]5911  %gep = getelementptr i32, ptr addrspace(1) %out, i32 45912  %result = atomicrmw max ptr addrspace(1) %gep, i32 %in seq_cst5913  ret i32 %result5914}5915 5916define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i32 %index) {5917; SI-LABEL: atomic_max_i32_addr64_offset:5918; SI:       ; %bb.0: ; %entry5919; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x95920; SI-NEXT:    s_waitcnt lgkmcnt(0)5921; SI-NEXT:    s_ashr_i32 s5, s3, 315922; SI-NEXT:    s_mov_b32 s4, s35923; SI-NEXT:    s_lshl_b64 s[4:5], s[4:5], 25924; SI-NEXT:    s_add_u32 s4, s0, s45925; SI-NEXT:    s_addc_u32 s5, s1, s55926; SI-NEXT:    s_load_dword s3, s[4:5], 0x45927; SI-NEXT:    s_mov_b64 s[0:1], 05928; SI-NEXT:    s_mov_b32 s7, 0xf0005929; SI-NEXT:    s_waitcnt lgkmcnt(0)5930; SI-NEXT:    v_mov_b32_e32 v1, s35931; SI-NEXT:    s_mov_b32 s6, -15932; SI-NEXT:  .LBB91_1: ; %atomicrmw.start5933; SI-NEXT:    ; =>This Inner Loop Header: Depth=15934; SI-NEXT:    v_max_i32_e32 v0, s2, v15935; SI-NEXT:    s_waitcnt expcnt(0)5936; SI-NEXT:    v_mov_b32_e32 v3, v15937; SI-NEXT:    v_mov_b32_e32 v2, v05938; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 offset:16 glc5939; SI-NEXT:    s_waitcnt vmcnt(0)5940; SI-NEXT:    buffer_wbinvl15941; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v15942; SI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]5943; SI-NEXT:    v_mov_b32_e32 v1, v25944; SI-NEXT:    s_andn2_b64 exec, exec, s[0:1]5945; SI-NEXT:    s_cbranch_execnz .LBB91_15946; SI-NEXT:  ; %bb.2: ; %atomicrmw.end5947; SI-NEXT:    s_endpgm5948;5949; VI-LABEL: atomic_max_i32_addr64_offset:5950; VI:       ; %bb.0: ; %entry5951; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245952; VI-NEXT:    s_waitcnt lgkmcnt(0)5953; VI-NEXT:    s_ashr_i32 s5, s3, 315954; VI-NEXT:    s_mov_b32 s4, s35955; VI-NEXT:    s_lshl_b64 s[4:5], s[4:5], 25956; VI-NEXT:    s_add_u32 s4, s0, s45957; VI-NEXT:    s_addc_u32 s5, s1, s55958; VI-NEXT:    s_load_dword s3, s[4:5], 0x105959; VI-NEXT:    s_add_u32 s4, s4, 165960; VI-NEXT:    s_addc_u32 s5, s5, 05961; VI-NEXT:    v_mov_b32_e32 v0, s45962; VI-NEXT:    s_mov_b64 s[0:1], 05963; VI-NEXT:    s_waitcnt lgkmcnt(0)5964; VI-NEXT:    v_mov_b32_e32 v3, s35965; VI-NEXT:    v_mov_b32_e32 v1, s55966; VI-NEXT:  .LBB91_1: ; %atomicrmw.start5967; VI-NEXT:    ; =>This Inner Loop Header: Depth=15968; VI-NEXT:    v_max_i32_e32 v2, s2, v35969; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc5970; VI-NEXT:    s_waitcnt vmcnt(0)5971; VI-NEXT:    buffer_wbinvl1_vol5972; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v35973; VI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]5974; VI-NEXT:    v_mov_b32_e32 v3, v25975; VI-NEXT:    s_andn2_b64 exec, exec, s[0:1]5976; VI-NEXT:    s_cbranch_execnz .LBB91_15977; VI-NEXT:  ; %bb.2: ; %atomicrmw.end5978; VI-NEXT:    s_endpgm5979;5980; GFX9-LABEL: atomic_max_i32_addr64_offset:5981; GFX9:       ; %bb.0: ; %entry5982; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x245983; GFX9-NEXT:    v_mov_b32_e32 v2, 05984; GFX9-NEXT:    s_waitcnt lgkmcnt(0)5985; GFX9-NEXT:    s_ashr_i32 s5, s3, 315986; GFX9-NEXT:    s_mov_b32 s4, s35987; GFX9-NEXT:    s_lshl_b64 s[4:5], s[4:5], 25988; GFX9-NEXT:    s_add_u32 s0, s0, s45989; GFX9-NEXT:    s_addc_u32 s1, s1, s55990; GFX9-NEXT:    s_load_dword s3, s[0:1], 0x105991; GFX9-NEXT:    s_mov_b64 s[4:5], 05992; GFX9-NEXT:    s_waitcnt lgkmcnt(0)5993; GFX9-NEXT:    v_mov_b32_e32 v1, s35994; GFX9-NEXT:  .LBB91_1: ; %atomicrmw.start5995; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=15996; GFX9-NEXT:    v_max_i32_e32 v0, s2, v15997; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc5998; GFX9-NEXT:    s_waitcnt vmcnt(0)5999; GFX9-NEXT:    buffer_wbinvl1_vol6000; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v16001; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]6002; GFX9-NEXT:    v_mov_b32_e32 v1, v06003; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]6004; GFX9-NEXT:    s_cbranch_execnz .LBB91_16005; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end6006; GFX9-NEXT:    s_endpgm6007entry:6008  %ptr = getelementptr i32, ptr addrspace(1) %out, i32 %index6009  %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 46010  %tmp0 = atomicrmw max ptr addrspace(1) %gep, i32 %in seq_cst6011  ret void6012}6013 6014define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i32 %index) {6015; SI-LABEL: atomic_max_i32_ret_addr64_offset:6016; SI:       ; %bb.0: ; %entry6017; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd6018; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96019; SI-NEXT:    s_waitcnt lgkmcnt(0)6020; SI-NEXT:    s_ashr_i32 s5, s9, 316021; SI-NEXT:    s_mov_b32 s4, s96022; SI-NEXT:    s_lshl_b64 s[4:5], s[4:5], 26023; SI-NEXT:    s_add_u32 s4, s0, s46024; SI-NEXT:    s_addc_u32 s5, s1, s56025; SI-NEXT:    s_load_dword s6, s[4:5], 0x46026; SI-NEXT:    s_mov_b64 s[0:1], 06027; SI-NEXT:    s_mov_b32 s7, 0xf0006028; SI-NEXT:    s_waitcnt lgkmcnt(0)6029; SI-NEXT:    v_mov_b32_e32 v1, s66030; SI-NEXT:    s_mov_b32 s6, -16031; SI-NEXT:  .LBB92_1: ; %atomicrmw.start6032; SI-NEXT:    ; =>This Inner Loop Header: Depth=16033; SI-NEXT:    v_max_i32_e32 v0, s8, v16034; SI-NEXT:    s_waitcnt expcnt(0)6035; SI-NEXT:    v_mov_b32_e32 v3, v16036; SI-NEXT:    v_mov_b32_e32 v2, v06037; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 offset:16 glc6038; SI-NEXT:    s_waitcnt vmcnt(0)6039; SI-NEXT:    buffer_wbinvl16040; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v16041; SI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]6042; SI-NEXT:    v_mov_b32_e32 v1, v26043; SI-NEXT:    s_andn2_b64 exec, exec, s[0:1]6044; SI-NEXT:    s_cbranch_execnz .LBB92_16045; SI-NEXT:  ; %bb.2: ; %atomicrmw.end6046; SI-NEXT:    s_or_b64 exec, exec, s[0:1]6047; SI-NEXT:    s_mov_b32 s7, 0xf0006048; SI-NEXT:    s_mov_b32 s6, -16049; SI-NEXT:    s_mov_b32 s4, s26050; SI-NEXT:    s_mov_b32 s5, s36051; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 06052; SI-NEXT:    s_endpgm6053;6054; VI-LABEL: atomic_max_i32_ret_addr64_offset:6055; VI:       ; %bb.0: ; %entry6056; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x346057; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246058; VI-NEXT:    s_waitcnt lgkmcnt(0)6059; VI-NEXT:    s_ashr_i32 s5, s7, 316060; VI-NEXT:    s_mov_b32 s4, s76061; VI-NEXT:    s_lshl_b64 s[4:5], s[4:5], 26062; VI-NEXT:    s_add_u32 s4, s0, s46063; VI-NEXT:    s_addc_u32 s5, s1, s56064; VI-NEXT:    s_load_dword s7, s[4:5], 0x106065; VI-NEXT:    s_add_u32 s4, s4, 166066; VI-NEXT:    s_addc_u32 s5, s5, 06067; VI-NEXT:    v_mov_b32_e32 v0, s46068; VI-NEXT:    s_mov_b64 s[0:1], 06069; VI-NEXT:    s_waitcnt lgkmcnt(0)6070; VI-NEXT:    v_mov_b32_e32 v2, s76071; VI-NEXT:    v_mov_b32_e32 v1, s56072; VI-NEXT:  .LBB92_1: ; %atomicrmw.start6073; VI-NEXT:    ; =>This Inner Loop Header: Depth=16074; VI-NEXT:    v_mov_b32_e32 v3, v26075; VI-NEXT:    v_max_i32_e32 v2, s6, v36076; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc6077; VI-NEXT:    s_waitcnt vmcnt(0)6078; VI-NEXT:    buffer_wbinvl1_vol6079; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v36080; VI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]6081; VI-NEXT:    s_andn2_b64 exec, exec, s[0:1]6082; VI-NEXT:    s_cbranch_execnz .LBB92_16083; VI-NEXT:  ; %bb.2: ; %atomicrmw.end6084; VI-NEXT:    s_or_b64 exec, exec, s[0:1]6085; VI-NEXT:    v_mov_b32_e32 v0, s26086; VI-NEXT:    v_mov_b32_e32 v1, s36087; VI-NEXT:    flat_store_dword v[0:1], v26088; VI-NEXT:    s_endpgm6089;6090; GFX9-LABEL: atomic_max_i32_ret_addr64_offset:6091; GFX9:       ; %bb.0: ; %entry6092; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x346093; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246094; GFX9-NEXT:    v_mov_b32_e32 v1, 06095; GFX9-NEXT:    s_waitcnt lgkmcnt(0)6096; GFX9-NEXT:    s_ashr_i32 s5, s7, 316097; GFX9-NEXT:    s_mov_b32 s4, s76098; GFX9-NEXT:    s_lshl_b64 s[4:5], s[4:5], 26099; GFX9-NEXT:    s_add_u32 s0, s0, s46100; GFX9-NEXT:    s_addc_u32 s1, s1, s56101; GFX9-NEXT:    s_load_dword s7, s[0:1], 0x106102; GFX9-NEXT:    s_mov_b64 s[4:5], 06103; GFX9-NEXT:    s_waitcnt lgkmcnt(0)6104; GFX9-NEXT:    v_mov_b32_e32 v0, s76105; GFX9-NEXT:  .LBB92_1: ; %atomicrmw.start6106; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=16107; GFX9-NEXT:    v_mov_b32_e32 v3, v06108; GFX9-NEXT:    v_max_i32_e32 v2, s6, v36109; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[0:1] offset:16 glc6110; GFX9-NEXT:    s_waitcnt vmcnt(0)6111; GFX9-NEXT:    buffer_wbinvl1_vol6112; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v36113; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]6114; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]6115; GFX9-NEXT:    s_cbranch_execnz .LBB92_16116; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end6117; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]6118; GFX9-NEXT:    v_mov_b32_e32 v1, 06119; GFX9-NEXT:    global_store_dword v1, v0, s[2:3]6120; GFX9-NEXT:    s_endpgm6121entry:6122  %ptr = getelementptr i32, ptr addrspace(1) %out, i32 %index6123  %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 46124  %tmp0 = atomicrmw max ptr addrspace(1) %gep, i32 %in seq_cst6125  store i32 %tmp0, ptr addrspace(1) %out26126  ret void6127}6128 6129define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in, i32 %index) {6130; SI-LABEL: atomic_max_i32_addr64:6131; SI:       ; %bb.0: ; %entry6132; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96133; SI-NEXT:    s_waitcnt lgkmcnt(0)6134; SI-NEXT:    s_ashr_i32 s5, s3, 316135; SI-NEXT:    s_mov_b32 s4, s36136; SI-NEXT:    s_lshl_b64 s[4:5], s[4:5], 26137; SI-NEXT:    s_add_u32 s4, s0, s46138; SI-NEXT:    s_addc_u32 s5, s1, s56139; SI-NEXT:    s_load_dword s3, s[4:5], 0x06140; SI-NEXT:    s_mov_b64 s[0:1], 06141; SI-NEXT:    s_mov_b32 s7, 0xf0006142; SI-NEXT:    s_waitcnt lgkmcnt(0)6143; SI-NEXT:    v_mov_b32_e32 v1, s36144; SI-NEXT:    s_mov_b32 s6, -16145; SI-NEXT:  .LBB93_1: ; %atomicrmw.start6146; SI-NEXT:    ; =>This Inner Loop Header: Depth=16147; SI-NEXT:    v_max_i32_e32 v0, s2, v16148; SI-NEXT:    s_waitcnt expcnt(0)6149; SI-NEXT:    v_mov_b32_e32 v3, v16150; SI-NEXT:    v_mov_b32_e32 v2, v06151; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc6152; SI-NEXT:    s_waitcnt vmcnt(0)6153; SI-NEXT:    buffer_wbinvl16154; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v16155; SI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]6156; SI-NEXT:    v_mov_b32_e32 v1, v26157; SI-NEXT:    s_andn2_b64 exec, exec, s[0:1]6158; SI-NEXT:    s_cbranch_execnz .LBB93_16159; SI-NEXT:  ; %bb.2: ; %atomicrmw.end6160; SI-NEXT:    s_endpgm6161;6162; VI-LABEL: atomic_max_i32_addr64:6163; VI:       ; %bb.0: ; %entry6164; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246165; VI-NEXT:    s_waitcnt lgkmcnt(0)6166; VI-NEXT:    s_ashr_i32 s5, s3, 316167; VI-NEXT:    s_mov_b32 s4, s36168; VI-NEXT:    s_lshl_b64 s[4:5], s[4:5], 26169; VI-NEXT:    s_add_u32 s4, s0, s46170; VI-NEXT:    s_addc_u32 s5, s1, s56171; VI-NEXT:    s_load_dword s3, s[4:5], 0x06172; VI-NEXT:    v_mov_b32_e32 v0, s46173; VI-NEXT:    s_mov_b64 s[0:1], 06174; VI-NEXT:    v_mov_b32_e32 v1, s56175; VI-NEXT:    s_waitcnt lgkmcnt(0)6176; VI-NEXT:    v_mov_b32_e32 v3, s36177; VI-NEXT:  .LBB93_1: ; %atomicrmw.start6178; VI-NEXT:    ; =>This Inner Loop Header: Depth=16179; VI-NEXT:    v_max_i32_e32 v2, s2, v36180; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc6181; VI-NEXT:    s_waitcnt vmcnt(0)6182; VI-NEXT:    buffer_wbinvl1_vol6183; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v36184; VI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]6185; VI-NEXT:    v_mov_b32_e32 v3, v26186; VI-NEXT:    s_andn2_b64 exec, exec, s[0:1]6187; VI-NEXT:    s_cbranch_execnz .LBB93_16188; VI-NEXT:  ; %bb.2: ; %atomicrmw.end6189; VI-NEXT:    s_endpgm6190;6191; GFX9-LABEL: atomic_max_i32_addr64:6192; GFX9:       ; %bb.0: ; %entry6193; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246194; GFX9-NEXT:    v_mov_b32_e32 v2, 06195; GFX9-NEXT:    s_waitcnt lgkmcnt(0)6196; GFX9-NEXT:    s_ashr_i32 s5, s3, 316197; GFX9-NEXT:    s_mov_b32 s4, s36198; GFX9-NEXT:    s_lshl_b64 s[4:5], s[4:5], 26199; GFX9-NEXT:    s_add_u32 s0, s0, s46200; GFX9-NEXT:    s_addc_u32 s1, s1, s56201; GFX9-NEXT:    s_load_dword s3, s[0:1], 0x06202; GFX9-NEXT:    s_mov_b64 s[4:5], 06203; GFX9-NEXT:    s_waitcnt lgkmcnt(0)6204; GFX9-NEXT:    v_mov_b32_e32 v1, s36205; GFX9-NEXT:  .LBB93_1: ; %atomicrmw.start6206; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=16207; GFX9-NEXT:    v_max_i32_e32 v0, s2, v16208; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc6209; GFX9-NEXT:    s_waitcnt vmcnt(0)6210; GFX9-NEXT:    buffer_wbinvl1_vol6211; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v16212; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]6213; GFX9-NEXT:    v_mov_b32_e32 v1, v06214; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]6215; GFX9-NEXT:    s_cbranch_execnz .LBB93_16216; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end6217; GFX9-NEXT:    s_endpgm6218entry:6219  %ptr = getelementptr i32, ptr addrspace(1) %out, i32 %index6220  %tmp0 = atomicrmw max ptr addrspace(1) %ptr, i32 %in seq_cst6221  ret void6222}6223 6224define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i32 %index) {6225; SI-LABEL: atomic_max_i32_ret_addr64:6226; SI:       ; %bb.0: ; %entry6227; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd6228; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x96229; SI-NEXT:    s_waitcnt lgkmcnt(0)6230; SI-NEXT:    s_ashr_i32 s5, s9, 316231; SI-NEXT:    s_mov_b32 s4, s96232; SI-NEXT:    s_lshl_b64 s[4:5], s[4:5], 26233; SI-NEXT:    s_add_u32 s4, s0, s46234; SI-NEXT:    s_addc_u32 s5, s1, s56235; SI-NEXT:    s_load_dword s6, s[4:5], 0x06236; SI-NEXT:    s_mov_b64 s[0:1], 06237; SI-NEXT:    s_mov_b32 s7, 0xf0006238; SI-NEXT:    s_waitcnt lgkmcnt(0)6239; SI-NEXT:    v_mov_b32_e32 v1, s66240; SI-NEXT:    s_mov_b32 s6, -16241; SI-NEXT:  .LBB94_1: ; %atomicrmw.start6242; SI-NEXT:    ; =>This Inner Loop Header: Depth=16243; SI-NEXT:    v_max_i32_e32 v0, s8, v16244; SI-NEXT:    s_waitcnt expcnt(0)6245; SI-NEXT:    v_mov_b32_e32 v3, v16246; SI-NEXT:    v_mov_b32_e32 v2, v06247; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc6248; SI-NEXT:    s_waitcnt vmcnt(0)6249; SI-NEXT:    buffer_wbinvl16250; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v16251; SI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]6252; SI-NEXT:    v_mov_b32_e32 v1, v26253; SI-NEXT:    s_andn2_b64 exec, exec, s[0:1]6254; SI-NEXT:    s_cbranch_execnz .LBB94_16255; SI-NEXT:  ; %bb.2: ; %atomicrmw.end6256; SI-NEXT:    s_or_b64 exec, exec, s[0:1]6257; SI-NEXT:    s_mov_b32 s7, 0xf0006258; SI-NEXT:    s_mov_b32 s6, -16259; SI-NEXT:    s_mov_b32 s4, s26260; SI-NEXT:    s_mov_b32 s5, s36261; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 06262; SI-NEXT:    s_endpgm6263;6264; VI-LABEL: atomic_max_i32_ret_addr64:6265; VI:       ; %bb.0: ; %entry6266; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x346267; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246268; VI-NEXT:    s_waitcnt lgkmcnt(0)6269; VI-NEXT:    s_ashr_i32 s5, s7, 316270; VI-NEXT:    s_mov_b32 s4, s76271; VI-NEXT:    s_lshl_b64 s[4:5], s[4:5], 26272; VI-NEXT:    s_add_u32 s4, s0, s46273; VI-NEXT:    s_addc_u32 s5, s1, s56274; VI-NEXT:    s_load_dword s7, s[4:5], 0x06275; VI-NEXT:    v_mov_b32_e32 v0, s46276; VI-NEXT:    s_mov_b64 s[0:1], 06277; VI-NEXT:    v_mov_b32_e32 v1, s56278; VI-NEXT:    s_waitcnt lgkmcnt(0)6279; VI-NEXT:    v_mov_b32_e32 v2, s76280; VI-NEXT:  .LBB94_1: ; %atomicrmw.start6281; VI-NEXT:    ; =>This Inner Loop Header: Depth=16282; VI-NEXT:    v_mov_b32_e32 v3, v26283; VI-NEXT:    v_max_i32_e32 v2, s6, v36284; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc6285; VI-NEXT:    s_waitcnt vmcnt(0)6286; VI-NEXT:    buffer_wbinvl1_vol6287; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v36288; VI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]6289; VI-NEXT:    s_andn2_b64 exec, exec, s[0:1]6290; VI-NEXT:    s_cbranch_execnz .LBB94_16291; VI-NEXT:  ; %bb.2: ; %atomicrmw.end6292; VI-NEXT:    s_or_b64 exec, exec, s[0:1]6293; VI-NEXT:    v_mov_b32_e32 v0, s26294; VI-NEXT:    v_mov_b32_e32 v1, s36295; VI-NEXT:    flat_store_dword v[0:1], v26296; VI-NEXT:    s_endpgm6297;6298; GFX9-LABEL: atomic_max_i32_ret_addr64:6299; GFX9:       ; %bb.0: ; %entry6300; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x346301; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x246302; GFX9-NEXT:    v_mov_b32_e32 v1, 06303; GFX9-NEXT:    s_waitcnt lgkmcnt(0)6304; GFX9-NEXT:    s_ashr_i32 s5, s7, 316305; GFX9-NEXT:    s_mov_b32 s4, s76306; GFX9-NEXT:    s_lshl_b64 s[4:5], s[4:5], 26307; GFX9-NEXT:    s_add_u32 s0, s0, s46308; GFX9-NEXT:    s_addc_u32 s1, s1, s56309; GFX9-NEXT:    s_load_dword s7, s[0:1], 0x06310; GFX9-NEXT:    s_mov_b64 s[4:5], 06311; GFX9-NEXT:    s_waitcnt lgkmcnt(0)6312; GFX9-NEXT:    v_mov_b32_e32 v0, s76313; GFX9-NEXT:  .LBB94_1: ; %atomicrmw.start6314; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=16315; GFX9-NEXT:    v_mov_b32_e32 v3, v06316; GFX9-NEXT:    v_max_i32_e32 v2, s6, v36317; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[0:1] glc6318; GFX9-NEXT:    s_waitcnt vmcnt(0)6319; GFX9-NEXT:    buffer_wbinvl1_vol6320; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v36321; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]6322; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]6323; GFX9-NEXT:    s_cbranch_execnz .LBB94_16324; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end6325; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]6326; GFX9-NEXT:    v_mov_b32_e32 v1, 06327; GFX9-NEXT:    global_store_dword v1, v0, s[2:3]6328; GFX9-NEXT:    s_endpgm6329entry:6330  %ptr = getelementptr i32, ptr addrspace(1) %out, i32 %index6331  %tmp0 = atomicrmw max ptr addrspace(1) %ptr, i32 %in seq_cst6332  store i32 %tmp0, ptr addrspace(1) %out26333  ret void6334}6335 6336define void @global_atomic_max_i32_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {6337; SI-LABEL: global_atomic_max_i32_noret_offset__amdgpu_no_remote_memory:6338; SI:       ; %bb.0:6339; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6340; SI-NEXT:    s_mov_b32 s6, 06341; SI-NEXT:    s_mov_b32 s7, 0xf0006342; SI-NEXT:    s_mov_b32 s4, s66343; SI-NEXT:    s_mov_b32 s5, s66344; SI-NEXT:    buffer_atomic_smax v2, v[0:1], s[4:7], 0 addr64 offset:166345; SI-NEXT:    s_waitcnt vmcnt(0)6346; SI-NEXT:    buffer_wbinvl16347; SI-NEXT:    s_waitcnt expcnt(0)6348; SI-NEXT:    s_setpc_b64 s[30:31]6349;6350; VI-LABEL: global_atomic_max_i32_noret_offset__amdgpu_no_remote_memory:6351; VI:       ; %bb.0:6352; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6353; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v06354; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc6355; VI-NEXT:    flat_atomic_smax v[0:1], v26356; VI-NEXT:    s_waitcnt vmcnt(0)6357; VI-NEXT:    buffer_wbinvl1_vol6358; VI-NEXT:    s_setpc_b64 s[30:31]6359;6360; GFX9-LABEL: global_atomic_max_i32_noret_offset__amdgpu_no_remote_memory:6361; GFX9:       ; %bb.0:6362; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6363; GFX9-NEXT:    global_atomic_smax v[0:1], v2, off offset:166364; GFX9-NEXT:    s_waitcnt vmcnt(0)6365; GFX9-NEXT:    buffer_wbinvl1_vol6366; GFX9-NEXT:    s_setpc_b64 s[30:31]6367  %gep = getelementptr i32, ptr addrspace(1) %out, i64 46368  %tmp0 = atomicrmw max ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !06369  ret void6370}6371 6372define i32 @global_atomic_max_i32_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {6373; SI-LABEL: global_atomic_max_i32_ret_offset__amdgpu_no_remote_memory:6374; SI:       ; %bb.0:6375; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6376; SI-NEXT:    s_mov_b32 s6, 06377; SI-NEXT:    s_mov_b32 s7, 0xf0006378; SI-NEXT:    s_mov_b32 s4, s66379; SI-NEXT:    s_mov_b32 s5, s66380; SI-NEXT:    buffer_atomic_smax v2, v[0:1], s[4:7], 0 addr64 offset:16 glc6381; SI-NEXT:    s_waitcnt vmcnt(0)6382; SI-NEXT:    buffer_wbinvl16383; SI-NEXT:    v_mov_b32_e32 v0, v26384; SI-NEXT:    s_waitcnt expcnt(0)6385; SI-NEXT:    s_setpc_b64 s[30:31]6386;6387; VI-LABEL: global_atomic_max_i32_ret_offset__amdgpu_no_remote_memory:6388; VI:       ; %bb.0:6389; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6390; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v06391; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc6392; VI-NEXT:    flat_atomic_smax v0, v[0:1], v2 glc6393; VI-NEXT:    s_waitcnt vmcnt(0)6394; VI-NEXT:    buffer_wbinvl1_vol6395; VI-NEXT:    s_setpc_b64 s[30:31]6396;6397; GFX9-LABEL: global_atomic_max_i32_ret_offset__amdgpu_no_remote_memory:6398; GFX9:       ; %bb.0:6399; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6400; GFX9-NEXT:    global_atomic_smax v0, v[0:1], v2, off offset:16 glc6401; GFX9-NEXT:    s_waitcnt vmcnt(0)6402; GFX9-NEXT:    buffer_wbinvl1_vol6403; GFX9-NEXT:    s_setpc_b64 s[30:31]6404  %gep = getelementptr i32, ptr addrspace(1) %out, i64 46405  %result = atomicrmw max ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !06406  ret i32 %result6407}6408 6409; ---------------------------------------------------------------------6410; atomicrmw umax6411; ---------------------------------------------------------------------6412 6413define void @global_atomic_umax_i32_noret(ptr addrspace(1) %ptr, i32 %in) {6414; SI-LABEL: global_atomic_umax_i32_noret:6415; SI:       ; %bb.0:6416; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6417; SI-NEXT:    s_mov_b32 s6, 06418; SI-NEXT:    s_mov_b32 s7, 0xf0006419; SI-NEXT:    s_mov_b32 s4, s66420; SI-NEXT:    s_mov_b32 s5, s66421; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr646422; SI-NEXT:    s_mov_b64 s[8:9], 06423; SI-NEXT:  .LBB97_1: ; %atomicrmw.start6424; SI-NEXT:    ; =>This Inner Loop Header: Depth=16425; SI-NEXT:    s_waitcnt vmcnt(0)6426; SI-NEXT:    v_max_u32_e32 v3, v4, v26427; SI-NEXT:    s_waitcnt expcnt(0)6428; SI-NEXT:    v_mov_b32_e32 v6, v46429; SI-NEXT:    v_mov_b32_e32 v5, v36430; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc6431; SI-NEXT:    s_waitcnt vmcnt(0)6432; SI-NEXT:    buffer_wbinvl16433; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v46434; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]6435; SI-NEXT:    v_mov_b32_e32 v4, v56436; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]6437; SI-NEXT:    s_cbranch_execnz .LBB97_16438; SI-NEXT:  ; %bb.2: ; %atomicrmw.end6439; SI-NEXT:    s_or_b64 exec, exec, s[8:9]6440; SI-NEXT:    s_waitcnt expcnt(0)6441; SI-NEXT:    s_setpc_b64 s[30:31]6442;6443; VI-LABEL: global_atomic_umax_i32_noret:6444; VI:       ; %bb.0:6445; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6446; VI-NEXT:    flat_load_dword v4, v[0:1]6447; VI-NEXT:    s_mov_b64 s[4:5], 06448; VI-NEXT:  .LBB97_1: ; %atomicrmw.start6449; VI-NEXT:    ; =>This Inner Loop Header: Depth=16450; VI-NEXT:    s_waitcnt vmcnt(0)6451; VI-NEXT:    v_max_u32_e32 v3, v4, v26452; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc6453; VI-NEXT:    s_waitcnt vmcnt(0)6454; VI-NEXT:    buffer_wbinvl1_vol6455; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v46456; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]6457; VI-NEXT:    v_mov_b32_e32 v4, v36458; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]6459; VI-NEXT:    s_cbranch_execnz .LBB97_16460; VI-NEXT:  ; %bb.2: ; %atomicrmw.end6461; VI-NEXT:    s_or_b64 exec, exec, s[4:5]6462; VI-NEXT:    s_setpc_b64 s[30:31]6463;6464; GFX9-LABEL: global_atomic_umax_i32_noret:6465; GFX9:       ; %bb.0:6466; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6467; GFX9-NEXT:    global_load_dword v4, v[0:1], off6468; GFX9-NEXT:    s_mov_b64 s[4:5], 06469; GFX9-NEXT:  .LBB97_1: ; %atomicrmw.start6470; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=16471; GFX9-NEXT:    s_waitcnt vmcnt(0)6472; GFX9-NEXT:    v_max_u32_e32 v3, v4, v26473; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc6474; GFX9-NEXT:    s_waitcnt vmcnt(0)6475; GFX9-NEXT:    buffer_wbinvl1_vol6476; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v46477; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]6478; GFX9-NEXT:    v_mov_b32_e32 v4, v36479; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]6480; GFX9-NEXT:    s_cbranch_execnz .LBB97_16481; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end6482; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]6483; GFX9-NEXT:    s_setpc_b64 s[30:31]6484  %tmp0 = atomicrmw umax ptr addrspace(1) %ptr, i32 %in seq_cst6485  ret void6486}6487 6488define void @global_atomic_umax_i32_noret_offset(ptr addrspace(1) %out, i32 %in) {6489; SI-LABEL: global_atomic_umax_i32_noret_offset:6490; SI:       ; %bb.0:6491; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6492; SI-NEXT:    s_mov_b32 s6, 06493; SI-NEXT:    s_mov_b32 s7, 0xf0006494; SI-NEXT:    s_mov_b32 s4, s66495; SI-NEXT:    s_mov_b32 s5, s66496; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:166497; SI-NEXT:    s_mov_b64 s[8:9], 06498; SI-NEXT:  .LBB98_1: ; %atomicrmw.start6499; SI-NEXT:    ; =>This Inner Loop Header: Depth=16500; SI-NEXT:    s_waitcnt vmcnt(0)6501; SI-NEXT:    v_max_u32_e32 v3, v4, v26502; SI-NEXT:    s_waitcnt expcnt(0)6503; SI-NEXT:    v_mov_b32_e32 v6, v46504; SI-NEXT:    v_mov_b32_e32 v5, v36505; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:16 glc6506; SI-NEXT:    s_waitcnt vmcnt(0)6507; SI-NEXT:    buffer_wbinvl16508; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v46509; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]6510; SI-NEXT:    v_mov_b32_e32 v4, v56511; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]6512; SI-NEXT:    s_cbranch_execnz .LBB98_16513; SI-NEXT:  ; %bb.2: ; %atomicrmw.end6514; SI-NEXT:    s_or_b64 exec, exec, s[8:9]6515; SI-NEXT:    s_waitcnt expcnt(0)6516; SI-NEXT:    s_setpc_b64 s[30:31]6517;6518; VI-LABEL: global_atomic_umax_i32_noret_offset:6519; VI:       ; %bb.0:6520; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6521; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v06522; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc6523; VI-NEXT:    flat_load_dword v4, v[0:1]6524; VI-NEXT:    s_mov_b64 s[4:5], 06525; VI-NEXT:  .LBB98_1: ; %atomicrmw.start6526; VI-NEXT:    ; =>This Inner Loop Header: Depth=16527; VI-NEXT:    s_waitcnt vmcnt(0)6528; VI-NEXT:    v_max_u32_e32 v3, v4, v26529; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc6530; VI-NEXT:    s_waitcnt vmcnt(0)6531; VI-NEXT:    buffer_wbinvl1_vol6532; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v46533; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]6534; VI-NEXT:    v_mov_b32_e32 v4, v36535; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]6536; VI-NEXT:    s_cbranch_execnz .LBB98_16537; VI-NEXT:  ; %bb.2: ; %atomicrmw.end6538; VI-NEXT:    s_or_b64 exec, exec, s[4:5]6539; VI-NEXT:    s_setpc_b64 s[30:31]6540;6541; GFX9-LABEL: global_atomic_umax_i32_noret_offset:6542; GFX9:       ; %bb.0:6543; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6544; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:166545; GFX9-NEXT:    s_mov_b64 s[4:5], 06546; GFX9-NEXT:  .LBB98_1: ; %atomicrmw.start6547; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=16548; GFX9-NEXT:    s_waitcnt vmcnt(0)6549; GFX9-NEXT:    v_max_u32_e32 v3, v4, v26550; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc6551; GFX9-NEXT:    s_waitcnt vmcnt(0)6552; GFX9-NEXT:    buffer_wbinvl1_vol6553; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v46554; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]6555; GFX9-NEXT:    v_mov_b32_e32 v4, v36556; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]6557; GFX9-NEXT:    s_cbranch_execnz .LBB98_16558; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end6559; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]6560; GFX9-NEXT:    s_setpc_b64 s[30:31]6561  %gep = getelementptr i32, ptr addrspace(1) %out, i32 46562  %tmp0 = atomicrmw umax ptr addrspace(1) %gep, i32 %in seq_cst6563  ret void6564}6565 6566define i32 @global_atomic_umax_i32_ret(ptr addrspace(1) %ptr, i32 %in) {6567; SI-LABEL: global_atomic_umax_i32_ret:6568; SI:       ; %bb.0:6569; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6570; SI-NEXT:    s_mov_b32 s6, 06571; SI-NEXT:    s_mov_b32 s7, 0xf0006572; SI-NEXT:    s_mov_b32 s4, s66573; SI-NEXT:    s_mov_b32 s5, s66574; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr646575; SI-NEXT:    s_mov_b64 s[8:9], 06576; SI-NEXT:  .LBB99_1: ; %atomicrmw.start6577; SI-NEXT:    ; =>This Inner Loop Header: Depth=16578; SI-NEXT:    s_waitcnt vmcnt(0)6579; SI-NEXT:    v_mov_b32_e32 v5, v36580; SI-NEXT:    s_waitcnt expcnt(0)6581; SI-NEXT:    v_max_u32_e32 v4, v5, v26582; SI-NEXT:    v_mov_b32_e32 v3, v46583; SI-NEXT:    v_mov_b32_e32 v4, v56584; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc6585; SI-NEXT:    s_waitcnt vmcnt(0)6586; SI-NEXT:    buffer_wbinvl16587; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v56588; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]6589; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]6590; SI-NEXT:    s_cbranch_execnz .LBB99_16591; SI-NEXT:  ; %bb.2: ; %atomicrmw.end6592; SI-NEXT:    s_or_b64 exec, exec, s[8:9]6593; SI-NEXT:    v_mov_b32_e32 v0, v36594; SI-NEXT:    s_waitcnt expcnt(0)6595; SI-NEXT:    s_setpc_b64 s[30:31]6596;6597; VI-LABEL: global_atomic_umax_i32_ret:6598; VI:       ; %bb.0:6599; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6600; VI-NEXT:    flat_load_dword v3, v[0:1]6601; VI-NEXT:    s_mov_b64 s[4:5], 06602; VI-NEXT:  .LBB99_1: ; %atomicrmw.start6603; VI-NEXT:    ; =>This Inner Loop Header: Depth=16604; VI-NEXT:    s_waitcnt vmcnt(0)6605; VI-NEXT:    v_mov_b32_e32 v4, v36606; VI-NEXT:    v_max_u32_e32 v3, v4, v26607; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc6608; VI-NEXT:    s_waitcnt vmcnt(0)6609; VI-NEXT:    buffer_wbinvl1_vol6610; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v46611; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]6612; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]6613; VI-NEXT:    s_cbranch_execnz .LBB99_16614; VI-NEXT:  ; %bb.2: ; %atomicrmw.end6615; VI-NEXT:    s_or_b64 exec, exec, s[4:5]6616; VI-NEXT:    v_mov_b32_e32 v0, v36617; VI-NEXT:    s_setpc_b64 s[30:31]6618;6619; GFX9-LABEL: global_atomic_umax_i32_ret:6620; GFX9:       ; %bb.0:6621; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6622; GFX9-NEXT:    global_load_dword v3, v[0:1], off6623; GFX9-NEXT:    s_mov_b64 s[4:5], 06624; GFX9-NEXT:  .LBB99_1: ; %atomicrmw.start6625; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=16626; GFX9-NEXT:    s_waitcnt vmcnt(0)6627; GFX9-NEXT:    v_mov_b32_e32 v4, v36628; GFX9-NEXT:    v_max_u32_e32 v3, v4, v26629; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc6630; GFX9-NEXT:    s_waitcnt vmcnt(0)6631; GFX9-NEXT:    buffer_wbinvl1_vol6632; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v46633; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]6634; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]6635; GFX9-NEXT:    s_cbranch_execnz .LBB99_16636; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end6637; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]6638; GFX9-NEXT:    v_mov_b32_e32 v0, v36639; GFX9-NEXT:    s_setpc_b64 s[30:31]6640  %result = atomicrmw umax ptr addrspace(1) %ptr, i32 %in seq_cst6641  ret i32 %result6642}6643 6644define i32 @global_atomic_umax_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {6645; SI-LABEL: global_atomic_umax_i32_ret_offset:6646; SI:       ; %bb.0:6647; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6648; SI-NEXT:    s_mov_b32 s6, 06649; SI-NEXT:    s_mov_b32 s7, 0xf0006650; SI-NEXT:    s_mov_b32 s4, s66651; SI-NEXT:    s_mov_b32 s5, s66652; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:166653; SI-NEXT:    s_mov_b64 s[8:9], 06654; SI-NEXT:  .LBB100_1: ; %atomicrmw.start6655; SI-NEXT:    ; =>This Inner Loop Header: Depth=16656; SI-NEXT:    s_waitcnt vmcnt(0)6657; SI-NEXT:    v_mov_b32_e32 v5, v36658; SI-NEXT:    s_waitcnt expcnt(0)6659; SI-NEXT:    v_max_u32_e32 v4, v5, v26660; SI-NEXT:    v_mov_b32_e32 v3, v46661; SI-NEXT:    v_mov_b32_e32 v4, v56662; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:16 glc6663; SI-NEXT:    s_waitcnt vmcnt(0)6664; SI-NEXT:    buffer_wbinvl16665; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v56666; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]6667; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]6668; SI-NEXT:    s_cbranch_execnz .LBB100_16669; SI-NEXT:  ; %bb.2: ; %atomicrmw.end6670; SI-NEXT:    s_or_b64 exec, exec, s[8:9]6671; SI-NEXT:    v_mov_b32_e32 v0, v36672; SI-NEXT:    s_waitcnt expcnt(0)6673; SI-NEXT:    s_setpc_b64 s[30:31]6674;6675; VI-LABEL: global_atomic_umax_i32_ret_offset:6676; VI:       ; %bb.0:6677; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6678; VI-NEXT:    v_add_u32_e32 v3, vcc, 16, v06679; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc6680; VI-NEXT:    flat_load_dword v0, v[3:4]6681; VI-NEXT:    s_mov_b64 s[4:5], 06682; VI-NEXT:  .LBB100_1: ; %atomicrmw.start6683; VI-NEXT:    ; =>This Inner Loop Header: Depth=16684; VI-NEXT:    s_waitcnt vmcnt(0)6685; VI-NEXT:    v_mov_b32_e32 v1, v06686; VI-NEXT:    v_max_u32_e32 v0, v1, v26687; VI-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc6688; VI-NEXT:    s_waitcnt vmcnt(0)6689; VI-NEXT:    buffer_wbinvl1_vol6690; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v16691; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]6692; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]6693; VI-NEXT:    s_cbranch_execnz .LBB100_16694; VI-NEXT:  ; %bb.2: ; %atomicrmw.end6695; VI-NEXT:    s_or_b64 exec, exec, s[4:5]6696; VI-NEXT:    s_setpc_b64 s[30:31]6697;6698; GFX9-LABEL: global_atomic_umax_i32_ret_offset:6699; GFX9:       ; %bb.0:6700; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6701; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:166702; GFX9-NEXT:    s_mov_b64 s[4:5], 06703; GFX9-NEXT:  .LBB100_1: ; %atomicrmw.start6704; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=16705; GFX9-NEXT:    s_waitcnt vmcnt(0)6706; GFX9-NEXT:    v_mov_b32_e32 v4, v36707; GFX9-NEXT:    v_max_u32_e32 v3, v4, v26708; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc6709; GFX9-NEXT:    s_waitcnt vmcnt(0)6710; GFX9-NEXT:    buffer_wbinvl1_vol6711; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v46712; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]6713; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]6714; GFX9-NEXT:    s_cbranch_execnz .LBB100_16715; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end6716; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]6717; GFX9-NEXT:    v_mov_b32_e32 v0, v36718; GFX9-NEXT:    s_setpc_b64 s[30:31]6719  %gep = getelementptr i32, ptr addrspace(1) %out, i32 46720  %result = atomicrmw umax ptr addrspace(1) %gep, i32 %in seq_cst6721  ret i32 %result6722}6723 6724define amdgpu_gfx void @global_atomic_umax_i32_noret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {6725; SI-LABEL: global_atomic_umax_i32_noret_scalar:6726; SI:       ; %bb.0:6727; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6728; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -16729; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill6730; SI-NEXT:    s_mov_b64 exec, s[34:35]6731; SI-NEXT:    s_waitcnt expcnt(0)6732; SI-NEXT:    v_writelane_b32 v4, s6, 06733; SI-NEXT:    v_writelane_b32 v4, s7, 16734; SI-NEXT:    s_mov_b32 s34, s66735; SI-NEXT:    s_mov_b32 s7, 0xf0006736; SI-NEXT:    s_mov_b32 s6, -16737; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 06738; SI-NEXT:    s_mov_b64 s[36:37], 06739; SI-NEXT:  .LBB101_1: ; %atomicrmw.start6740; SI-NEXT:    ; =>This Inner Loop Header: Depth=16741; SI-NEXT:    s_waitcnt vmcnt(0)6742; SI-NEXT:    v_max_u32_e32 v0, s34, v16743; SI-NEXT:    s_waitcnt expcnt(0)6744; SI-NEXT:    v_mov_b32_e32 v3, v16745; SI-NEXT:    v_mov_b32_e32 v2, v06746; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc6747; SI-NEXT:    s_waitcnt vmcnt(0)6748; SI-NEXT:    buffer_wbinvl16749; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v16750; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]6751; SI-NEXT:    v_mov_b32_e32 v1, v26752; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]6753; SI-NEXT:    s_cbranch_execnz .LBB101_16754; SI-NEXT:  ; %bb.2: ; %atomicrmw.end6755; SI-NEXT:    s_or_b64 exec, exec, s[36:37]6756; SI-NEXT:    v_readlane_b32 s7, v4, 16757; SI-NEXT:    v_readlane_b32 s6, v4, 06758; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -16759; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload6760; SI-NEXT:    s_mov_b64 exec, s[34:35]6761; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)6762; SI-NEXT:    s_setpc_b64 s[30:31]6763;6764; VI-LABEL: global_atomic_umax_i32_noret_scalar:6765; VI:       ; %bb.0:6766; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6767; VI-NEXT:    v_mov_b32_e32 v0, s46768; VI-NEXT:    v_mov_b32_e32 v1, s56769; VI-NEXT:    flat_load_dword v3, v[0:1]6770; VI-NEXT:    s_mov_b64 s[34:35], 06771; VI-NEXT:  .LBB101_1: ; %atomicrmw.start6772; VI-NEXT:    ; =>This Inner Loop Header: Depth=16773; VI-NEXT:    s_waitcnt vmcnt(0)6774; VI-NEXT:    v_max_u32_e32 v2, s6, v36775; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc6776; VI-NEXT:    s_waitcnt vmcnt(0)6777; VI-NEXT:    buffer_wbinvl1_vol6778; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v36779; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]6780; VI-NEXT:    v_mov_b32_e32 v3, v26781; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]6782; VI-NEXT:    s_cbranch_execnz .LBB101_16783; VI-NEXT:  ; %bb.2: ; %atomicrmw.end6784; VI-NEXT:    s_or_b64 exec, exec, s[34:35]6785; VI-NEXT:    s_setpc_b64 s[30:31]6786;6787; GFX9-LABEL: global_atomic_umax_i32_noret_scalar:6788; GFX9:       ; %bb.0:6789; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6790; GFX9-NEXT:    v_mov_b32_e32 v2, 06791; GFX9-NEXT:    global_load_dword v1, v2, s[4:5]6792; GFX9-NEXT:    s_mov_b64 s[34:35], 06793; GFX9-NEXT:  .LBB101_1: ; %atomicrmw.start6794; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=16795; GFX9-NEXT:    s_waitcnt vmcnt(0)6796; GFX9-NEXT:    v_max_u32_e32 v0, s6, v16797; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] glc6798; GFX9-NEXT:    s_waitcnt vmcnt(0)6799; GFX9-NEXT:    buffer_wbinvl1_vol6800; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v16801; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]6802; GFX9-NEXT:    v_mov_b32_e32 v1, v06803; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]6804; GFX9-NEXT:    s_cbranch_execnz .LBB101_16805; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end6806; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]6807; GFX9-NEXT:    s_setpc_b64 s[30:31]6808  %tmp0 = atomicrmw umax ptr addrspace(1) %ptr, i32 %in seq_cst6809  ret void6810}6811 6812define amdgpu_gfx void @global_atomic_umax_i32_noret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {6813; SI-LABEL: global_atomic_umax_i32_noret_offset_scalar:6814; SI:       ; %bb.0:6815; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6816; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -16817; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill6818; SI-NEXT:    s_mov_b64 exec, s[34:35]6819; SI-NEXT:    s_waitcnt expcnt(0)6820; SI-NEXT:    v_writelane_b32 v4, s6, 06821; SI-NEXT:    v_writelane_b32 v4, s7, 16822; SI-NEXT:    s_mov_b32 s34, s66823; SI-NEXT:    s_mov_b32 s7, 0xf0006824; SI-NEXT:    s_mov_b32 s6, -16825; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:166826; SI-NEXT:    s_mov_b64 s[36:37], 06827; SI-NEXT:  .LBB102_1: ; %atomicrmw.start6828; SI-NEXT:    ; =>This Inner Loop Header: Depth=16829; SI-NEXT:    s_waitcnt vmcnt(0)6830; SI-NEXT:    v_max_u32_e32 v0, s34, v16831; SI-NEXT:    s_waitcnt expcnt(0)6832; SI-NEXT:    v_mov_b32_e32 v3, v16833; SI-NEXT:    v_mov_b32_e32 v2, v06834; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 offset:16 glc6835; SI-NEXT:    s_waitcnt vmcnt(0)6836; SI-NEXT:    buffer_wbinvl16837; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v16838; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]6839; SI-NEXT:    v_mov_b32_e32 v1, v26840; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]6841; SI-NEXT:    s_cbranch_execnz .LBB102_16842; SI-NEXT:  ; %bb.2: ; %atomicrmw.end6843; SI-NEXT:    s_or_b64 exec, exec, s[36:37]6844; SI-NEXT:    v_readlane_b32 s7, v4, 16845; SI-NEXT:    v_readlane_b32 s6, v4, 06846; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -16847; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload6848; SI-NEXT:    s_mov_b64 exec, s[34:35]6849; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)6850; SI-NEXT:    s_setpc_b64 s[30:31]6851;6852; VI-LABEL: global_atomic_umax_i32_noret_offset_scalar:6853; VI:       ; %bb.0:6854; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6855; VI-NEXT:    s_add_u32 s34, s4, 166856; VI-NEXT:    s_addc_u32 s35, s5, 06857; VI-NEXT:    v_mov_b32_e32 v0, s346858; VI-NEXT:    v_mov_b32_e32 v1, s356859; VI-NEXT:    flat_load_dword v3, v[0:1]6860; VI-NEXT:    s_mov_b64 s[34:35], 06861; VI-NEXT:  .LBB102_1: ; %atomicrmw.start6862; VI-NEXT:    ; =>This Inner Loop Header: Depth=16863; VI-NEXT:    s_waitcnt vmcnt(0)6864; VI-NEXT:    v_max_u32_e32 v2, s6, v36865; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc6866; VI-NEXT:    s_waitcnt vmcnt(0)6867; VI-NEXT:    buffer_wbinvl1_vol6868; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v36869; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]6870; VI-NEXT:    v_mov_b32_e32 v3, v26871; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]6872; VI-NEXT:    s_cbranch_execnz .LBB102_16873; VI-NEXT:  ; %bb.2: ; %atomicrmw.end6874; VI-NEXT:    s_or_b64 exec, exec, s[34:35]6875; VI-NEXT:    s_setpc_b64 s[30:31]6876;6877; GFX9-LABEL: global_atomic_umax_i32_noret_offset_scalar:6878; GFX9:       ; %bb.0:6879; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6880; GFX9-NEXT:    v_mov_b32_e32 v2, 06881; GFX9-NEXT:    global_load_dword v1, v2, s[4:5] offset:166882; GFX9-NEXT:    s_mov_b64 s[34:35], 06883; GFX9-NEXT:  .LBB102_1: ; %atomicrmw.start6884; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=16885; GFX9-NEXT:    s_waitcnt vmcnt(0)6886; GFX9-NEXT:    v_max_u32_e32 v0, s6, v16887; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 glc6888; GFX9-NEXT:    s_waitcnt vmcnt(0)6889; GFX9-NEXT:    buffer_wbinvl1_vol6890; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v16891; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]6892; GFX9-NEXT:    v_mov_b32_e32 v1, v06893; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]6894; GFX9-NEXT:    s_cbranch_execnz .LBB102_16895; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end6896; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]6897; GFX9-NEXT:    s_setpc_b64 s[30:31]6898  %gep = getelementptr i32, ptr addrspace(1) %out, i32 46899  %tmp0 = atomicrmw umax ptr addrspace(1) %gep, i32 %in seq_cst6900  ret void6901}6902 6903define amdgpu_gfx i32 @global_atomic_umax_i32_ret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {6904; SI-LABEL: global_atomic_umax_i32_ret_scalar:6905; SI:       ; %bb.0:6906; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6907; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -16908; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill6909; SI-NEXT:    s_mov_b64 exec, s[34:35]6910; SI-NEXT:    s_waitcnt expcnt(0)6911; SI-NEXT:    v_writelane_b32 v3, s6, 06912; SI-NEXT:    v_writelane_b32 v3, s7, 16913; SI-NEXT:    s_mov_b32 s34, s66914; SI-NEXT:    s_mov_b32 s7, 0xf0006915; SI-NEXT:    s_mov_b32 s6, -16916; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 06917; SI-NEXT:    s_mov_b64 s[36:37], 06918; SI-NEXT:  .LBB103_1: ; %atomicrmw.start6919; SI-NEXT:    ; =>This Inner Loop Header: Depth=16920; SI-NEXT:    s_waitcnt vmcnt(0)6921; SI-NEXT:    v_mov_b32_e32 v2, v06922; SI-NEXT:    s_waitcnt expcnt(0)6923; SI-NEXT:    v_max_u32_e32 v1, s34, v26924; SI-NEXT:    v_mov_b32_e32 v0, v16925; SI-NEXT:    v_mov_b32_e32 v1, v26926; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc6927; SI-NEXT:    s_waitcnt vmcnt(0)6928; SI-NEXT:    buffer_wbinvl16929; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v26930; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]6931; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]6932; SI-NEXT:    s_cbranch_execnz .LBB103_16933; SI-NEXT:  ; %bb.2: ; %atomicrmw.end6934; SI-NEXT:    s_or_b64 exec, exec, s[36:37]6935; SI-NEXT:    v_readlane_b32 s7, v3, 16936; SI-NEXT:    v_readlane_b32 s6, v3, 06937; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -16938; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload6939; SI-NEXT:    s_mov_b64 exec, s[34:35]6940; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)6941; SI-NEXT:    s_setpc_b64 s[30:31]6942;6943; VI-LABEL: global_atomic_umax_i32_ret_scalar:6944; VI:       ; %bb.0:6945; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6946; VI-NEXT:    v_mov_b32_e32 v0, s46947; VI-NEXT:    v_mov_b32_e32 v1, s56948; VI-NEXT:    flat_load_dword v0, v[0:1]6949; VI-NEXT:    v_mov_b32_e32 v1, s46950; VI-NEXT:    s_mov_b64 s[34:35], 06951; VI-NEXT:    v_mov_b32_e32 v2, s56952; VI-NEXT:  .LBB103_1: ; %atomicrmw.start6953; VI-NEXT:    ; =>This Inner Loop Header: Depth=16954; VI-NEXT:    s_waitcnt vmcnt(0)6955; VI-NEXT:    v_mov_b32_e32 v4, v06956; VI-NEXT:    v_max_u32_e32 v3, s6, v46957; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc6958; VI-NEXT:    s_waitcnt vmcnt(0)6959; VI-NEXT:    buffer_wbinvl1_vol6960; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v46961; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]6962; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]6963; VI-NEXT:    s_cbranch_execnz .LBB103_16964; VI-NEXT:  ; %bb.2: ; %atomicrmw.end6965; VI-NEXT:    s_or_b64 exec, exec, s[34:35]6966; VI-NEXT:    s_setpc_b64 s[30:31]6967;6968; GFX9-LABEL: global_atomic_umax_i32_ret_scalar:6969; GFX9:       ; %bb.0:6970; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6971; GFX9-NEXT:    v_mov_b32_e32 v1, 06972; GFX9-NEXT:    global_load_dword v0, v1, s[4:5]6973; GFX9-NEXT:    s_mov_b64 s[34:35], 06974; GFX9-NEXT:  .LBB103_1: ; %atomicrmw.start6975; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=16976; GFX9-NEXT:    s_waitcnt vmcnt(0)6977; GFX9-NEXT:    v_mov_b32_e32 v3, v06978; GFX9-NEXT:    v_max_u32_e32 v2, s6, v36979; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] glc6980; GFX9-NEXT:    s_waitcnt vmcnt(0)6981; GFX9-NEXT:    buffer_wbinvl1_vol6982; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v36983; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]6984; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]6985; GFX9-NEXT:    s_cbranch_execnz .LBB103_16986; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end6987; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]6988; GFX9-NEXT:    s_setpc_b64 s[30:31]6989  %result = atomicrmw umax ptr addrspace(1) %ptr, i32 %in seq_cst6990  ret i32 %result6991}6992 6993define amdgpu_gfx i32 @global_atomic_umax_i32_ret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {6994; SI-LABEL: global_atomic_umax_i32_ret_offset_scalar:6995; SI:       ; %bb.0:6996; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6997; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -16998; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill6999; SI-NEXT:    s_mov_b64 exec, s[34:35]7000; SI-NEXT:    s_waitcnt expcnt(0)7001; SI-NEXT:    v_writelane_b32 v3, s6, 07002; SI-NEXT:    v_writelane_b32 v3, s7, 17003; SI-NEXT:    s_mov_b32 s34, s67004; SI-NEXT:    s_mov_b32 s7, 0xf0007005; SI-NEXT:    s_mov_b32 s6, -17006; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:167007; SI-NEXT:    s_mov_b64 s[36:37], 07008; SI-NEXT:  .LBB104_1: ; %atomicrmw.start7009; SI-NEXT:    ; =>This Inner Loop Header: Depth=17010; SI-NEXT:    s_waitcnt vmcnt(0)7011; SI-NEXT:    v_mov_b32_e32 v2, v07012; SI-NEXT:    s_waitcnt expcnt(0)7013; SI-NEXT:    v_max_u32_e32 v1, s34, v27014; SI-NEXT:    v_mov_b32_e32 v0, v17015; SI-NEXT:    v_mov_b32_e32 v1, v27016; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc7017; SI-NEXT:    s_waitcnt vmcnt(0)7018; SI-NEXT:    buffer_wbinvl17019; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v27020; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]7021; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]7022; SI-NEXT:    s_cbranch_execnz .LBB104_17023; SI-NEXT:  ; %bb.2: ; %atomicrmw.end7024; SI-NEXT:    s_or_b64 exec, exec, s[36:37]7025; SI-NEXT:    v_readlane_b32 s7, v3, 17026; SI-NEXT:    v_readlane_b32 s6, v3, 07027; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -17028; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload7029; SI-NEXT:    s_mov_b64 exec, s[34:35]7030; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)7031; SI-NEXT:    s_setpc_b64 s[30:31]7032;7033; VI-LABEL: global_atomic_umax_i32_ret_offset_scalar:7034; VI:       ; %bb.0:7035; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7036; VI-NEXT:    s_add_u32 s34, s4, 167037; VI-NEXT:    s_addc_u32 s35, s5, 07038; VI-NEXT:    v_mov_b32_e32 v1, s347039; VI-NEXT:    v_mov_b32_e32 v2, s357040; VI-NEXT:    flat_load_dword v0, v[1:2]7041; VI-NEXT:    s_mov_b64 s[34:35], 07042; VI-NEXT:  .LBB104_1: ; %atomicrmw.start7043; VI-NEXT:    ; =>This Inner Loop Header: Depth=17044; VI-NEXT:    s_waitcnt vmcnt(0)7045; VI-NEXT:    v_mov_b32_e32 v4, v07046; VI-NEXT:    v_max_u32_e32 v3, s6, v47047; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc7048; VI-NEXT:    s_waitcnt vmcnt(0)7049; VI-NEXT:    buffer_wbinvl1_vol7050; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v47051; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]7052; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]7053; VI-NEXT:    s_cbranch_execnz .LBB104_17054; VI-NEXT:  ; %bb.2: ; %atomicrmw.end7055; VI-NEXT:    s_or_b64 exec, exec, s[34:35]7056; VI-NEXT:    s_setpc_b64 s[30:31]7057;7058; GFX9-LABEL: global_atomic_umax_i32_ret_offset_scalar:7059; GFX9:       ; %bb.0:7060; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7061; GFX9-NEXT:    v_mov_b32_e32 v1, 07062; GFX9-NEXT:    global_load_dword v0, v1, s[4:5] offset:167063; GFX9-NEXT:    s_mov_b64 s[34:35], 07064; GFX9-NEXT:  .LBB104_1: ; %atomicrmw.start7065; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=17066; GFX9-NEXT:    s_waitcnt vmcnt(0)7067; GFX9-NEXT:    v_mov_b32_e32 v3, v07068; GFX9-NEXT:    v_max_u32_e32 v2, s6, v37069; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] offset:16 glc7070; GFX9-NEXT:    s_waitcnt vmcnt(0)7071; GFX9-NEXT:    buffer_wbinvl1_vol7072; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v37073; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]7074; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]7075; GFX9-NEXT:    s_cbranch_execnz .LBB104_17076; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end7077; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]7078; GFX9-NEXT:    s_setpc_b64 s[30:31]7079  %gep = getelementptr i32, ptr addrspace(1) %out, i32 47080  %result = atomicrmw umax ptr addrspace(1) %gep, i32 %in seq_cst7081  ret i32 %result7082}7083 7084define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i32 %index) {7085; SI-LABEL: atomic_umax_i32_addr64_offset:7086; SI:       ; %bb.0: ; %entry7087; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x97088; SI-NEXT:    s_waitcnt lgkmcnt(0)7089; SI-NEXT:    s_ashr_i32 s5, s3, 317090; SI-NEXT:    s_mov_b32 s4, s37091; SI-NEXT:    s_lshl_b64 s[4:5], s[4:5], 27092; SI-NEXT:    s_add_u32 s4, s0, s47093; SI-NEXT:    s_addc_u32 s5, s1, s57094; SI-NEXT:    s_load_dword s3, s[4:5], 0x47095; SI-NEXT:    s_mov_b64 s[0:1], 07096; SI-NEXT:    s_mov_b32 s7, 0xf0007097; SI-NEXT:    s_waitcnt lgkmcnt(0)7098; SI-NEXT:    v_mov_b32_e32 v1, s37099; SI-NEXT:    s_mov_b32 s6, -17100; SI-NEXT:  .LBB105_1: ; %atomicrmw.start7101; SI-NEXT:    ; =>This Inner Loop Header: Depth=17102; SI-NEXT:    v_max_u32_e32 v0, s2, v17103; SI-NEXT:    s_waitcnt expcnt(0)7104; SI-NEXT:    v_mov_b32_e32 v3, v17105; SI-NEXT:    v_mov_b32_e32 v2, v07106; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 offset:16 glc7107; SI-NEXT:    s_waitcnt vmcnt(0)7108; SI-NEXT:    buffer_wbinvl17109; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v17110; SI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]7111; SI-NEXT:    v_mov_b32_e32 v1, v27112; SI-NEXT:    s_andn2_b64 exec, exec, s[0:1]7113; SI-NEXT:    s_cbranch_execnz .LBB105_17114; SI-NEXT:  ; %bb.2: ; %atomicrmw.end7115; SI-NEXT:    s_endpgm7116;7117; VI-LABEL: atomic_umax_i32_addr64_offset:7118; VI:       ; %bb.0: ; %entry7119; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247120; VI-NEXT:    s_waitcnt lgkmcnt(0)7121; VI-NEXT:    s_ashr_i32 s5, s3, 317122; VI-NEXT:    s_mov_b32 s4, s37123; VI-NEXT:    s_lshl_b64 s[4:5], s[4:5], 27124; VI-NEXT:    s_add_u32 s4, s0, s47125; VI-NEXT:    s_addc_u32 s5, s1, s57126; VI-NEXT:    s_load_dword s3, s[4:5], 0x107127; VI-NEXT:    s_add_u32 s4, s4, 167128; VI-NEXT:    s_addc_u32 s5, s5, 07129; VI-NEXT:    v_mov_b32_e32 v0, s47130; VI-NEXT:    s_mov_b64 s[0:1], 07131; VI-NEXT:    s_waitcnt lgkmcnt(0)7132; VI-NEXT:    v_mov_b32_e32 v3, s37133; VI-NEXT:    v_mov_b32_e32 v1, s57134; VI-NEXT:  .LBB105_1: ; %atomicrmw.start7135; VI-NEXT:    ; =>This Inner Loop Header: Depth=17136; VI-NEXT:    v_max_u32_e32 v2, s2, v37137; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc7138; VI-NEXT:    s_waitcnt vmcnt(0)7139; VI-NEXT:    buffer_wbinvl1_vol7140; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v37141; VI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]7142; VI-NEXT:    v_mov_b32_e32 v3, v27143; VI-NEXT:    s_andn2_b64 exec, exec, s[0:1]7144; VI-NEXT:    s_cbranch_execnz .LBB105_17145; VI-NEXT:  ; %bb.2: ; %atomicrmw.end7146; VI-NEXT:    s_endpgm7147;7148; GFX9-LABEL: atomic_umax_i32_addr64_offset:7149; GFX9:       ; %bb.0: ; %entry7150; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247151; GFX9-NEXT:    v_mov_b32_e32 v2, 07152; GFX9-NEXT:    s_waitcnt lgkmcnt(0)7153; GFX9-NEXT:    s_ashr_i32 s5, s3, 317154; GFX9-NEXT:    s_mov_b32 s4, s37155; GFX9-NEXT:    s_lshl_b64 s[4:5], s[4:5], 27156; GFX9-NEXT:    s_add_u32 s0, s0, s47157; GFX9-NEXT:    s_addc_u32 s1, s1, s57158; GFX9-NEXT:    s_load_dword s3, s[0:1], 0x107159; GFX9-NEXT:    s_mov_b64 s[4:5], 07160; GFX9-NEXT:    s_waitcnt lgkmcnt(0)7161; GFX9-NEXT:    v_mov_b32_e32 v1, s37162; GFX9-NEXT:  .LBB105_1: ; %atomicrmw.start7163; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=17164; GFX9-NEXT:    v_max_u32_e32 v0, s2, v17165; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc7166; GFX9-NEXT:    s_waitcnt vmcnt(0)7167; GFX9-NEXT:    buffer_wbinvl1_vol7168; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v17169; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]7170; GFX9-NEXT:    v_mov_b32_e32 v1, v07171; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]7172; GFX9-NEXT:    s_cbranch_execnz .LBB105_17173; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end7174; GFX9-NEXT:    s_endpgm7175entry:7176  %ptr = getelementptr i32, ptr addrspace(1) %out, i32 %index7177  %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 47178  %tmp0 = atomicrmw umax ptr addrspace(1) %gep, i32 %in seq_cst7179  ret void7180}7181 7182define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i32 %index) {7183; SI-LABEL: atomic_umax_i32_ret_addr64_offset:7184; SI:       ; %bb.0: ; %entry7185; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd7186; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x97187; SI-NEXT:    s_waitcnt lgkmcnt(0)7188; SI-NEXT:    s_ashr_i32 s5, s9, 317189; SI-NEXT:    s_mov_b32 s4, s97190; SI-NEXT:    s_lshl_b64 s[4:5], s[4:5], 27191; SI-NEXT:    s_add_u32 s4, s0, s47192; SI-NEXT:    s_addc_u32 s5, s1, s57193; SI-NEXT:    s_load_dword s6, s[4:5], 0x47194; SI-NEXT:    s_mov_b64 s[0:1], 07195; SI-NEXT:    s_mov_b32 s7, 0xf0007196; SI-NEXT:    s_waitcnt lgkmcnt(0)7197; SI-NEXT:    v_mov_b32_e32 v1, s67198; SI-NEXT:    s_mov_b32 s6, -17199; SI-NEXT:  .LBB106_1: ; %atomicrmw.start7200; SI-NEXT:    ; =>This Inner Loop Header: Depth=17201; SI-NEXT:    v_max_u32_e32 v0, s8, v17202; SI-NEXT:    s_waitcnt expcnt(0)7203; SI-NEXT:    v_mov_b32_e32 v3, v17204; SI-NEXT:    v_mov_b32_e32 v2, v07205; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 offset:16 glc7206; SI-NEXT:    s_waitcnt vmcnt(0)7207; SI-NEXT:    buffer_wbinvl17208; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v17209; SI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]7210; SI-NEXT:    v_mov_b32_e32 v1, v27211; SI-NEXT:    s_andn2_b64 exec, exec, s[0:1]7212; SI-NEXT:    s_cbranch_execnz .LBB106_17213; SI-NEXT:  ; %bb.2: ; %atomicrmw.end7214; SI-NEXT:    s_or_b64 exec, exec, s[0:1]7215; SI-NEXT:    s_mov_b32 s7, 0xf0007216; SI-NEXT:    s_mov_b32 s6, -17217; SI-NEXT:    s_mov_b32 s4, s27218; SI-NEXT:    s_mov_b32 s5, s37219; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 07220; SI-NEXT:    s_endpgm7221;7222; VI-LABEL: atomic_umax_i32_ret_addr64_offset:7223; VI:       ; %bb.0: ; %entry7224; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x347225; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247226; VI-NEXT:    s_waitcnt lgkmcnt(0)7227; VI-NEXT:    s_ashr_i32 s5, s7, 317228; VI-NEXT:    s_mov_b32 s4, s77229; VI-NEXT:    s_lshl_b64 s[4:5], s[4:5], 27230; VI-NEXT:    s_add_u32 s4, s0, s47231; VI-NEXT:    s_addc_u32 s5, s1, s57232; VI-NEXT:    s_load_dword s7, s[4:5], 0x107233; VI-NEXT:    s_add_u32 s4, s4, 167234; VI-NEXT:    s_addc_u32 s5, s5, 07235; VI-NEXT:    v_mov_b32_e32 v0, s47236; VI-NEXT:    s_mov_b64 s[0:1], 07237; VI-NEXT:    s_waitcnt lgkmcnt(0)7238; VI-NEXT:    v_mov_b32_e32 v2, s77239; VI-NEXT:    v_mov_b32_e32 v1, s57240; VI-NEXT:  .LBB106_1: ; %atomicrmw.start7241; VI-NEXT:    ; =>This Inner Loop Header: Depth=17242; VI-NEXT:    v_mov_b32_e32 v3, v27243; VI-NEXT:    v_max_u32_e32 v2, s6, v37244; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc7245; VI-NEXT:    s_waitcnt vmcnt(0)7246; VI-NEXT:    buffer_wbinvl1_vol7247; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v37248; VI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]7249; VI-NEXT:    s_andn2_b64 exec, exec, s[0:1]7250; VI-NEXT:    s_cbranch_execnz .LBB106_17251; VI-NEXT:  ; %bb.2: ; %atomicrmw.end7252; VI-NEXT:    s_or_b64 exec, exec, s[0:1]7253; VI-NEXT:    v_mov_b32_e32 v0, s27254; VI-NEXT:    v_mov_b32_e32 v1, s37255; VI-NEXT:    flat_store_dword v[0:1], v27256; VI-NEXT:    s_endpgm7257;7258; GFX9-LABEL: atomic_umax_i32_ret_addr64_offset:7259; GFX9:       ; %bb.0: ; %entry7260; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x347261; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247262; GFX9-NEXT:    v_mov_b32_e32 v1, 07263; GFX9-NEXT:    s_waitcnt lgkmcnt(0)7264; GFX9-NEXT:    s_ashr_i32 s5, s7, 317265; GFX9-NEXT:    s_mov_b32 s4, s77266; GFX9-NEXT:    s_lshl_b64 s[4:5], s[4:5], 27267; GFX9-NEXT:    s_add_u32 s0, s0, s47268; GFX9-NEXT:    s_addc_u32 s1, s1, s57269; GFX9-NEXT:    s_load_dword s7, s[0:1], 0x107270; GFX9-NEXT:    s_mov_b64 s[4:5], 07271; GFX9-NEXT:    s_waitcnt lgkmcnt(0)7272; GFX9-NEXT:    v_mov_b32_e32 v0, s77273; GFX9-NEXT:  .LBB106_1: ; %atomicrmw.start7274; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=17275; GFX9-NEXT:    v_mov_b32_e32 v3, v07276; GFX9-NEXT:    v_max_u32_e32 v2, s6, v37277; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[0:1] offset:16 glc7278; GFX9-NEXT:    s_waitcnt vmcnt(0)7279; GFX9-NEXT:    buffer_wbinvl1_vol7280; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v37281; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]7282; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]7283; GFX9-NEXT:    s_cbranch_execnz .LBB106_17284; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end7285; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]7286; GFX9-NEXT:    v_mov_b32_e32 v1, 07287; GFX9-NEXT:    global_store_dword v1, v0, s[2:3]7288; GFX9-NEXT:    s_endpgm7289entry:7290  %ptr = getelementptr i32, ptr addrspace(1) %out, i32 %index7291  %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 47292  %tmp0 = atomicrmw umax ptr addrspace(1) %gep, i32 %in seq_cst7293  store i32 %tmp0, ptr addrspace(1) %out27294  ret void7295}7296 7297define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i32 %index) {7298; SI-LABEL: atomic_umax_i32_ret_addr64:7299; SI:       ; %bb.0: ; %entry7300; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd7301; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x97302; SI-NEXT:    s_waitcnt lgkmcnt(0)7303; SI-NEXT:    s_ashr_i32 s5, s9, 317304; SI-NEXT:    s_mov_b32 s4, s97305; SI-NEXT:    s_lshl_b64 s[4:5], s[4:5], 27306; SI-NEXT:    s_add_u32 s4, s0, s47307; SI-NEXT:    s_addc_u32 s5, s1, s57308; SI-NEXT:    s_load_dword s6, s[4:5], 0x07309; SI-NEXT:    s_mov_b64 s[0:1], 07310; SI-NEXT:    s_mov_b32 s7, 0xf0007311; SI-NEXT:    s_waitcnt lgkmcnt(0)7312; SI-NEXT:    v_mov_b32_e32 v1, s67313; SI-NEXT:    s_mov_b32 s6, -17314; SI-NEXT:  .LBB107_1: ; %atomicrmw.start7315; SI-NEXT:    ; =>This Inner Loop Header: Depth=17316; SI-NEXT:    v_max_u32_e32 v0, s8, v17317; SI-NEXT:    s_waitcnt expcnt(0)7318; SI-NEXT:    v_mov_b32_e32 v3, v17319; SI-NEXT:    v_mov_b32_e32 v2, v07320; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc7321; SI-NEXT:    s_waitcnt vmcnt(0)7322; SI-NEXT:    buffer_wbinvl17323; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v17324; SI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]7325; SI-NEXT:    v_mov_b32_e32 v1, v27326; SI-NEXT:    s_andn2_b64 exec, exec, s[0:1]7327; SI-NEXT:    s_cbranch_execnz .LBB107_17328; SI-NEXT:  ; %bb.2: ; %atomicrmw.end7329; SI-NEXT:    s_or_b64 exec, exec, s[0:1]7330; SI-NEXT:    s_mov_b32 s7, 0xf0007331; SI-NEXT:    s_mov_b32 s6, -17332; SI-NEXT:    s_mov_b32 s4, s27333; SI-NEXT:    s_mov_b32 s5, s37334; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 07335; SI-NEXT:    s_endpgm7336;7337; VI-LABEL: atomic_umax_i32_ret_addr64:7338; VI:       ; %bb.0: ; %entry7339; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x347340; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247341; VI-NEXT:    s_waitcnt lgkmcnt(0)7342; VI-NEXT:    s_ashr_i32 s5, s7, 317343; VI-NEXT:    s_mov_b32 s4, s77344; VI-NEXT:    s_lshl_b64 s[4:5], s[4:5], 27345; VI-NEXT:    s_add_u32 s4, s0, s47346; VI-NEXT:    s_addc_u32 s5, s1, s57347; VI-NEXT:    s_load_dword s7, s[4:5], 0x07348; VI-NEXT:    v_mov_b32_e32 v0, s47349; VI-NEXT:    s_mov_b64 s[0:1], 07350; VI-NEXT:    v_mov_b32_e32 v1, s57351; VI-NEXT:    s_waitcnt lgkmcnt(0)7352; VI-NEXT:    v_mov_b32_e32 v2, s77353; VI-NEXT:  .LBB107_1: ; %atomicrmw.start7354; VI-NEXT:    ; =>This Inner Loop Header: Depth=17355; VI-NEXT:    v_mov_b32_e32 v3, v27356; VI-NEXT:    v_max_u32_e32 v2, s6, v37357; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc7358; VI-NEXT:    s_waitcnt vmcnt(0)7359; VI-NEXT:    buffer_wbinvl1_vol7360; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v37361; VI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]7362; VI-NEXT:    s_andn2_b64 exec, exec, s[0:1]7363; VI-NEXT:    s_cbranch_execnz .LBB107_17364; VI-NEXT:  ; %bb.2: ; %atomicrmw.end7365; VI-NEXT:    s_or_b64 exec, exec, s[0:1]7366; VI-NEXT:    v_mov_b32_e32 v0, s27367; VI-NEXT:    v_mov_b32_e32 v1, s37368; VI-NEXT:    flat_store_dword v[0:1], v27369; VI-NEXT:    s_endpgm7370;7371; GFX9-LABEL: atomic_umax_i32_ret_addr64:7372; GFX9:       ; %bb.0: ; %entry7373; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x347374; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x247375; GFX9-NEXT:    v_mov_b32_e32 v1, 07376; GFX9-NEXT:    s_waitcnt lgkmcnt(0)7377; GFX9-NEXT:    s_ashr_i32 s5, s7, 317378; GFX9-NEXT:    s_mov_b32 s4, s77379; GFX9-NEXT:    s_lshl_b64 s[4:5], s[4:5], 27380; GFX9-NEXT:    s_add_u32 s0, s0, s47381; GFX9-NEXT:    s_addc_u32 s1, s1, s57382; GFX9-NEXT:    s_load_dword s7, s[0:1], 0x07383; GFX9-NEXT:    s_mov_b64 s[4:5], 07384; GFX9-NEXT:    s_waitcnt lgkmcnt(0)7385; GFX9-NEXT:    v_mov_b32_e32 v0, s77386; GFX9-NEXT:  .LBB107_1: ; %atomicrmw.start7387; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=17388; GFX9-NEXT:    v_mov_b32_e32 v3, v07389; GFX9-NEXT:    v_max_u32_e32 v2, s6, v37390; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[0:1] glc7391; GFX9-NEXT:    s_waitcnt vmcnt(0)7392; GFX9-NEXT:    buffer_wbinvl1_vol7393; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v37394; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]7395; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]7396; GFX9-NEXT:    s_cbranch_execnz .LBB107_17397; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end7398; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]7399; GFX9-NEXT:    v_mov_b32_e32 v1, 07400; GFX9-NEXT:    global_store_dword v1, v0, s[2:3]7401; GFX9-NEXT:    s_endpgm7402entry:7403  %ptr = getelementptr i32, ptr addrspace(1) %out, i32 %index7404  %tmp0 = atomicrmw umax ptr addrspace(1) %ptr, i32 %in seq_cst7405  store i32 %tmp0, ptr addrspace(1) %out27406  ret void7407}7408 7409define void @global_atomic_umax_i32_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {7410; SI-LABEL: global_atomic_umax_i32_noret_offset__amdgpu_no_remote_memory:7411; SI:       ; %bb.0:7412; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7413; SI-NEXT:    s_mov_b32 s6, 07414; SI-NEXT:    s_mov_b32 s7, 0xf0007415; SI-NEXT:    s_mov_b32 s4, s67416; SI-NEXT:    s_mov_b32 s5, s67417; SI-NEXT:    buffer_atomic_umax v2, v[0:1], s[4:7], 0 addr64 offset:167418; SI-NEXT:    s_waitcnt vmcnt(0)7419; SI-NEXT:    buffer_wbinvl17420; SI-NEXT:    s_waitcnt expcnt(0)7421; SI-NEXT:    s_setpc_b64 s[30:31]7422;7423; VI-LABEL: global_atomic_umax_i32_noret_offset__amdgpu_no_remote_memory:7424; VI:       ; %bb.0:7425; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7426; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v07427; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc7428; VI-NEXT:    flat_atomic_umax v[0:1], v27429; VI-NEXT:    s_waitcnt vmcnt(0)7430; VI-NEXT:    buffer_wbinvl1_vol7431; VI-NEXT:    s_setpc_b64 s[30:31]7432;7433; GFX9-LABEL: global_atomic_umax_i32_noret_offset__amdgpu_no_remote_memory:7434; GFX9:       ; %bb.0:7435; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7436; GFX9-NEXT:    global_atomic_umax v[0:1], v2, off offset:167437; GFX9-NEXT:    s_waitcnt vmcnt(0)7438; GFX9-NEXT:    buffer_wbinvl1_vol7439; GFX9-NEXT:    s_setpc_b64 s[30:31]7440  %gep = getelementptr i32, ptr addrspace(1) %out, i64 47441  %tmp0 = atomicrmw umax ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !07442  ret void7443}7444 7445define i32 @global_atomic_umax_i32_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {7446; SI-LABEL: global_atomic_umax_i32_ret_offset__amdgpu_no_remote_memory:7447; SI:       ; %bb.0:7448; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7449; SI-NEXT:    s_mov_b32 s6, 07450; SI-NEXT:    s_mov_b32 s7, 0xf0007451; SI-NEXT:    s_mov_b32 s4, s67452; SI-NEXT:    s_mov_b32 s5, s67453; SI-NEXT:    buffer_atomic_umax v2, v[0:1], s[4:7], 0 addr64 offset:16 glc7454; SI-NEXT:    s_waitcnt vmcnt(0)7455; SI-NEXT:    buffer_wbinvl17456; SI-NEXT:    v_mov_b32_e32 v0, v27457; SI-NEXT:    s_waitcnt expcnt(0)7458; SI-NEXT:    s_setpc_b64 s[30:31]7459;7460; VI-LABEL: global_atomic_umax_i32_ret_offset__amdgpu_no_remote_memory:7461; VI:       ; %bb.0:7462; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7463; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v07464; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc7465; VI-NEXT:    flat_atomic_umax v0, v[0:1], v2 glc7466; VI-NEXT:    s_waitcnt vmcnt(0)7467; VI-NEXT:    buffer_wbinvl1_vol7468; VI-NEXT:    s_setpc_b64 s[30:31]7469;7470; GFX9-LABEL: global_atomic_umax_i32_ret_offset__amdgpu_no_remote_memory:7471; GFX9:       ; %bb.0:7472; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7473; GFX9-NEXT:    global_atomic_umax v0, v[0:1], v2, off offset:16 glc7474; GFX9-NEXT:    s_waitcnt vmcnt(0)7475; GFX9-NEXT:    buffer_wbinvl1_vol7476; GFX9-NEXT:    s_setpc_b64 s[30:31]7477  %gep = getelementptr i32, ptr addrspace(1) %out, i64 47478  %result = atomicrmw umax ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !07479  ret i32 %result7480}7481 7482; ---------------------------------------------------------------------7483; atomicrmw umin7484; ---------------------------------------------------------------------7485 7486define void @global_atomic_umin_i32_noret(ptr addrspace(1) %ptr, i32 %in) {7487; SI-LABEL: global_atomic_umin_i32_noret:7488; SI:       ; %bb.0:7489; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7490; SI-NEXT:    s_mov_b32 s6, 07491; SI-NEXT:    s_mov_b32 s7, 0xf0007492; SI-NEXT:    s_mov_b32 s4, s67493; SI-NEXT:    s_mov_b32 s5, s67494; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr647495; SI-NEXT:    s_mov_b64 s[8:9], 07496; SI-NEXT:  .LBB110_1: ; %atomicrmw.start7497; SI-NEXT:    ; =>This Inner Loop Header: Depth=17498; SI-NEXT:    s_waitcnt vmcnt(0)7499; SI-NEXT:    v_min_u32_e32 v3, v4, v27500; SI-NEXT:    s_waitcnt expcnt(0)7501; SI-NEXT:    v_mov_b32_e32 v6, v47502; SI-NEXT:    v_mov_b32_e32 v5, v37503; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc7504; SI-NEXT:    s_waitcnt vmcnt(0)7505; SI-NEXT:    buffer_wbinvl17506; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v47507; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]7508; SI-NEXT:    v_mov_b32_e32 v4, v57509; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]7510; SI-NEXT:    s_cbranch_execnz .LBB110_17511; SI-NEXT:  ; %bb.2: ; %atomicrmw.end7512; SI-NEXT:    s_or_b64 exec, exec, s[8:9]7513; SI-NEXT:    s_waitcnt expcnt(0)7514; SI-NEXT:    s_setpc_b64 s[30:31]7515;7516; VI-LABEL: global_atomic_umin_i32_noret:7517; VI:       ; %bb.0:7518; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7519; VI-NEXT:    flat_load_dword v4, v[0:1]7520; VI-NEXT:    s_mov_b64 s[4:5], 07521; VI-NEXT:  .LBB110_1: ; %atomicrmw.start7522; VI-NEXT:    ; =>This Inner Loop Header: Depth=17523; VI-NEXT:    s_waitcnt vmcnt(0)7524; VI-NEXT:    v_min_u32_e32 v3, v4, v27525; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc7526; VI-NEXT:    s_waitcnt vmcnt(0)7527; VI-NEXT:    buffer_wbinvl1_vol7528; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v47529; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]7530; VI-NEXT:    v_mov_b32_e32 v4, v37531; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]7532; VI-NEXT:    s_cbranch_execnz .LBB110_17533; VI-NEXT:  ; %bb.2: ; %atomicrmw.end7534; VI-NEXT:    s_or_b64 exec, exec, s[4:5]7535; VI-NEXT:    s_setpc_b64 s[30:31]7536;7537; GFX9-LABEL: global_atomic_umin_i32_noret:7538; GFX9:       ; %bb.0:7539; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7540; GFX9-NEXT:    global_load_dword v4, v[0:1], off7541; GFX9-NEXT:    s_mov_b64 s[4:5], 07542; GFX9-NEXT:  .LBB110_1: ; %atomicrmw.start7543; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=17544; GFX9-NEXT:    s_waitcnt vmcnt(0)7545; GFX9-NEXT:    v_min_u32_e32 v3, v4, v27546; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc7547; GFX9-NEXT:    s_waitcnt vmcnt(0)7548; GFX9-NEXT:    buffer_wbinvl1_vol7549; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v47550; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]7551; GFX9-NEXT:    v_mov_b32_e32 v4, v37552; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]7553; GFX9-NEXT:    s_cbranch_execnz .LBB110_17554; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end7555; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]7556; GFX9-NEXT:    s_setpc_b64 s[30:31]7557  %tmp0 = atomicrmw umin ptr addrspace(1) %ptr, i32 %in seq_cst7558  ret void7559}7560 7561define void @global_atomic_umin_i32_noret_offset(ptr addrspace(1) %out, i32 %in) {7562; SI-LABEL: global_atomic_umin_i32_noret_offset:7563; SI:       ; %bb.0:7564; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7565; SI-NEXT:    s_mov_b32 s6, 07566; SI-NEXT:    s_mov_b32 s7, 0xf0007567; SI-NEXT:    s_mov_b32 s4, s67568; SI-NEXT:    s_mov_b32 s5, s67569; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:167570; SI-NEXT:    s_mov_b64 s[8:9], 07571; SI-NEXT:  .LBB111_1: ; %atomicrmw.start7572; SI-NEXT:    ; =>This Inner Loop Header: Depth=17573; SI-NEXT:    s_waitcnt vmcnt(0)7574; SI-NEXT:    v_min_u32_e32 v3, v4, v27575; SI-NEXT:    s_waitcnt expcnt(0)7576; SI-NEXT:    v_mov_b32_e32 v6, v47577; SI-NEXT:    v_mov_b32_e32 v5, v37578; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:16 glc7579; SI-NEXT:    s_waitcnt vmcnt(0)7580; SI-NEXT:    buffer_wbinvl17581; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v47582; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]7583; SI-NEXT:    v_mov_b32_e32 v4, v57584; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]7585; SI-NEXT:    s_cbranch_execnz .LBB111_17586; SI-NEXT:  ; %bb.2: ; %atomicrmw.end7587; SI-NEXT:    s_or_b64 exec, exec, s[8:9]7588; SI-NEXT:    s_waitcnt expcnt(0)7589; SI-NEXT:    s_setpc_b64 s[30:31]7590;7591; VI-LABEL: global_atomic_umin_i32_noret_offset:7592; VI:       ; %bb.0:7593; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7594; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v07595; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc7596; VI-NEXT:    flat_load_dword v4, v[0:1]7597; VI-NEXT:    s_mov_b64 s[4:5], 07598; VI-NEXT:  .LBB111_1: ; %atomicrmw.start7599; VI-NEXT:    ; =>This Inner Loop Header: Depth=17600; VI-NEXT:    s_waitcnt vmcnt(0)7601; VI-NEXT:    v_min_u32_e32 v3, v4, v27602; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc7603; VI-NEXT:    s_waitcnt vmcnt(0)7604; VI-NEXT:    buffer_wbinvl1_vol7605; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v47606; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]7607; VI-NEXT:    v_mov_b32_e32 v4, v37608; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]7609; VI-NEXT:    s_cbranch_execnz .LBB111_17610; VI-NEXT:  ; %bb.2: ; %atomicrmw.end7611; VI-NEXT:    s_or_b64 exec, exec, s[4:5]7612; VI-NEXT:    s_setpc_b64 s[30:31]7613;7614; GFX9-LABEL: global_atomic_umin_i32_noret_offset:7615; GFX9:       ; %bb.0:7616; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7617; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:167618; GFX9-NEXT:    s_mov_b64 s[4:5], 07619; GFX9-NEXT:  .LBB111_1: ; %atomicrmw.start7620; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=17621; GFX9-NEXT:    s_waitcnt vmcnt(0)7622; GFX9-NEXT:    v_min_u32_e32 v3, v4, v27623; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc7624; GFX9-NEXT:    s_waitcnt vmcnt(0)7625; GFX9-NEXT:    buffer_wbinvl1_vol7626; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v47627; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]7628; GFX9-NEXT:    v_mov_b32_e32 v4, v37629; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]7630; GFX9-NEXT:    s_cbranch_execnz .LBB111_17631; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end7632; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]7633; GFX9-NEXT:    s_setpc_b64 s[30:31]7634  %gep = getelementptr i32, ptr addrspace(1) %out, i32 47635  %tmp0 = atomicrmw umin ptr addrspace(1) %gep, i32 %in seq_cst7636  ret void7637}7638 7639define i32 @global_atomic_umin_i32_ret(ptr addrspace(1) %ptr, i32 %in) {7640; SI-LABEL: global_atomic_umin_i32_ret:7641; SI:       ; %bb.0:7642; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7643; SI-NEXT:    s_mov_b32 s6, 07644; SI-NEXT:    s_mov_b32 s7, 0xf0007645; SI-NEXT:    s_mov_b32 s4, s67646; SI-NEXT:    s_mov_b32 s5, s67647; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr647648; SI-NEXT:    s_mov_b64 s[8:9], 07649; SI-NEXT:  .LBB112_1: ; %atomicrmw.start7650; SI-NEXT:    ; =>This Inner Loop Header: Depth=17651; SI-NEXT:    s_waitcnt vmcnt(0)7652; SI-NEXT:    v_mov_b32_e32 v5, v37653; SI-NEXT:    s_waitcnt expcnt(0)7654; SI-NEXT:    v_min_u32_e32 v4, v5, v27655; SI-NEXT:    v_mov_b32_e32 v3, v47656; SI-NEXT:    v_mov_b32_e32 v4, v57657; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc7658; SI-NEXT:    s_waitcnt vmcnt(0)7659; SI-NEXT:    buffer_wbinvl17660; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v57661; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]7662; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]7663; SI-NEXT:    s_cbranch_execnz .LBB112_17664; SI-NEXT:  ; %bb.2: ; %atomicrmw.end7665; SI-NEXT:    s_or_b64 exec, exec, s[8:9]7666; SI-NEXT:    v_mov_b32_e32 v0, v37667; SI-NEXT:    s_waitcnt expcnt(0)7668; SI-NEXT:    s_setpc_b64 s[30:31]7669;7670; VI-LABEL: global_atomic_umin_i32_ret:7671; VI:       ; %bb.0:7672; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7673; VI-NEXT:    flat_load_dword v3, v[0:1]7674; VI-NEXT:    s_mov_b64 s[4:5], 07675; VI-NEXT:  .LBB112_1: ; %atomicrmw.start7676; VI-NEXT:    ; =>This Inner Loop Header: Depth=17677; VI-NEXT:    s_waitcnt vmcnt(0)7678; VI-NEXT:    v_mov_b32_e32 v4, v37679; VI-NEXT:    v_min_u32_e32 v3, v4, v27680; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc7681; VI-NEXT:    s_waitcnt vmcnt(0)7682; VI-NEXT:    buffer_wbinvl1_vol7683; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v47684; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]7685; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]7686; VI-NEXT:    s_cbranch_execnz .LBB112_17687; VI-NEXT:  ; %bb.2: ; %atomicrmw.end7688; VI-NEXT:    s_or_b64 exec, exec, s[4:5]7689; VI-NEXT:    v_mov_b32_e32 v0, v37690; VI-NEXT:    s_setpc_b64 s[30:31]7691;7692; GFX9-LABEL: global_atomic_umin_i32_ret:7693; GFX9:       ; %bb.0:7694; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7695; GFX9-NEXT:    global_load_dword v3, v[0:1], off7696; GFX9-NEXT:    s_mov_b64 s[4:5], 07697; GFX9-NEXT:  .LBB112_1: ; %atomicrmw.start7698; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=17699; GFX9-NEXT:    s_waitcnt vmcnt(0)7700; GFX9-NEXT:    v_mov_b32_e32 v4, v37701; GFX9-NEXT:    v_min_u32_e32 v3, v4, v27702; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc7703; GFX9-NEXT:    s_waitcnt vmcnt(0)7704; GFX9-NEXT:    buffer_wbinvl1_vol7705; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v47706; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]7707; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]7708; GFX9-NEXT:    s_cbranch_execnz .LBB112_17709; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end7710; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]7711; GFX9-NEXT:    v_mov_b32_e32 v0, v37712; GFX9-NEXT:    s_setpc_b64 s[30:31]7713  %result = atomicrmw umin ptr addrspace(1) %ptr, i32 %in seq_cst7714  ret i32 %result7715}7716 7717define i32 @global_atomic_umin_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {7718; SI-LABEL: global_atomic_umin_i32_ret_offset:7719; SI:       ; %bb.0:7720; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7721; SI-NEXT:    s_mov_b32 s6, 07722; SI-NEXT:    s_mov_b32 s7, 0xf0007723; SI-NEXT:    s_mov_b32 s4, s67724; SI-NEXT:    s_mov_b32 s5, s67725; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:167726; SI-NEXT:    s_mov_b64 s[8:9], 07727; SI-NEXT:  .LBB113_1: ; %atomicrmw.start7728; SI-NEXT:    ; =>This Inner Loop Header: Depth=17729; SI-NEXT:    s_waitcnt vmcnt(0)7730; SI-NEXT:    v_mov_b32_e32 v5, v37731; SI-NEXT:    s_waitcnt expcnt(0)7732; SI-NEXT:    v_min_u32_e32 v4, v5, v27733; SI-NEXT:    v_mov_b32_e32 v3, v47734; SI-NEXT:    v_mov_b32_e32 v4, v57735; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:16 glc7736; SI-NEXT:    s_waitcnt vmcnt(0)7737; SI-NEXT:    buffer_wbinvl17738; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v57739; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]7740; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]7741; SI-NEXT:    s_cbranch_execnz .LBB113_17742; SI-NEXT:  ; %bb.2: ; %atomicrmw.end7743; SI-NEXT:    s_or_b64 exec, exec, s[8:9]7744; SI-NEXT:    v_mov_b32_e32 v0, v37745; SI-NEXT:    s_waitcnt expcnt(0)7746; SI-NEXT:    s_setpc_b64 s[30:31]7747;7748; VI-LABEL: global_atomic_umin_i32_ret_offset:7749; VI:       ; %bb.0:7750; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7751; VI-NEXT:    v_add_u32_e32 v3, vcc, 16, v07752; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc7753; VI-NEXT:    flat_load_dword v0, v[3:4]7754; VI-NEXT:    s_mov_b64 s[4:5], 07755; VI-NEXT:  .LBB113_1: ; %atomicrmw.start7756; VI-NEXT:    ; =>This Inner Loop Header: Depth=17757; VI-NEXT:    s_waitcnt vmcnt(0)7758; VI-NEXT:    v_mov_b32_e32 v1, v07759; VI-NEXT:    v_min_u32_e32 v0, v1, v27760; VI-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc7761; VI-NEXT:    s_waitcnt vmcnt(0)7762; VI-NEXT:    buffer_wbinvl1_vol7763; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v17764; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]7765; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]7766; VI-NEXT:    s_cbranch_execnz .LBB113_17767; VI-NEXT:  ; %bb.2: ; %atomicrmw.end7768; VI-NEXT:    s_or_b64 exec, exec, s[4:5]7769; VI-NEXT:    s_setpc_b64 s[30:31]7770;7771; GFX9-LABEL: global_atomic_umin_i32_ret_offset:7772; GFX9:       ; %bb.0:7773; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7774; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:167775; GFX9-NEXT:    s_mov_b64 s[4:5], 07776; GFX9-NEXT:  .LBB113_1: ; %atomicrmw.start7777; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=17778; GFX9-NEXT:    s_waitcnt vmcnt(0)7779; GFX9-NEXT:    v_mov_b32_e32 v4, v37780; GFX9-NEXT:    v_min_u32_e32 v3, v4, v27781; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc7782; GFX9-NEXT:    s_waitcnt vmcnt(0)7783; GFX9-NEXT:    buffer_wbinvl1_vol7784; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v47785; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]7786; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]7787; GFX9-NEXT:    s_cbranch_execnz .LBB113_17788; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end7789; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]7790; GFX9-NEXT:    v_mov_b32_e32 v0, v37791; GFX9-NEXT:    s_setpc_b64 s[30:31]7792  %gep = getelementptr i32, ptr addrspace(1) %out, i32 47793  %result = atomicrmw umin ptr addrspace(1) %gep, i32 %in seq_cst7794  ret i32 %result7795}7796 7797define amdgpu_gfx void @global_atomic_umin_i32_noret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {7798; SI-LABEL: global_atomic_umin_i32_noret_scalar:7799; SI:       ; %bb.0:7800; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7801; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -17802; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill7803; SI-NEXT:    s_mov_b64 exec, s[34:35]7804; SI-NEXT:    s_waitcnt expcnt(0)7805; SI-NEXT:    v_writelane_b32 v4, s6, 07806; SI-NEXT:    v_writelane_b32 v4, s7, 17807; SI-NEXT:    s_mov_b32 s34, s67808; SI-NEXT:    s_mov_b32 s7, 0xf0007809; SI-NEXT:    s_mov_b32 s6, -17810; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 07811; SI-NEXT:    s_mov_b64 s[36:37], 07812; SI-NEXT:  .LBB114_1: ; %atomicrmw.start7813; SI-NEXT:    ; =>This Inner Loop Header: Depth=17814; SI-NEXT:    s_waitcnt vmcnt(0)7815; SI-NEXT:    v_min_u32_e32 v0, s34, v17816; SI-NEXT:    s_waitcnt expcnt(0)7817; SI-NEXT:    v_mov_b32_e32 v3, v17818; SI-NEXT:    v_mov_b32_e32 v2, v07819; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc7820; SI-NEXT:    s_waitcnt vmcnt(0)7821; SI-NEXT:    buffer_wbinvl17822; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v17823; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]7824; SI-NEXT:    v_mov_b32_e32 v1, v27825; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]7826; SI-NEXT:    s_cbranch_execnz .LBB114_17827; SI-NEXT:  ; %bb.2: ; %atomicrmw.end7828; SI-NEXT:    s_or_b64 exec, exec, s[36:37]7829; SI-NEXT:    v_readlane_b32 s7, v4, 17830; SI-NEXT:    v_readlane_b32 s6, v4, 07831; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -17832; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload7833; SI-NEXT:    s_mov_b64 exec, s[34:35]7834; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)7835; SI-NEXT:    s_setpc_b64 s[30:31]7836;7837; VI-LABEL: global_atomic_umin_i32_noret_scalar:7838; VI:       ; %bb.0:7839; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7840; VI-NEXT:    v_mov_b32_e32 v0, s47841; VI-NEXT:    v_mov_b32_e32 v1, s57842; VI-NEXT:    flat_load_dword v3, v[0:1]7843; VI-NEXT:    s_mov_b64 s[34:35], 07844; VI-NEXT:  .LBB114_1: ; %atomicrmw.start7845; VI-NEXT:    ; =>This Inner Loop Header: Depth=17846; VI-NEXT:    s_waitcnt vmcnt(0)7847; VI-NEXT:    v_min_u32_e32 v2, s6, v37848; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc7849; VI-NEXT:    s_waitcnt vmcnt(0)7850; VI-NEXT:    buffer_wbinvl1_vol7851; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v37852; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]7853; VI-NEXT:    v_mov_b32_e32 v3, v27854; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]7855; VI-NEXT:    s_cbranch_execnz .LBB114_17856; VI-NEXT:  ; %bb.2: ; %atomicrmw.end7857; VI-NEXT:    s_or_b64 exec, exec, s[34:35]7858; VI-NEXT:    s_setpc_b64 s[30:31]7859;7860; GFX9-LABEL: global_atomic_umin_i32_noret_scalar:7861; GFX9:       ; %bb.0:7862; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7863; GFX9-NEXT:    v_mov_b32_e32 v2, 07864; GFX9-NEXT:    global_load_dword v1, v2, s[4:5]7865; GFX9-NEXT:    s_mov_b64 s[34:35], 07866; GFX9-NEXT:  .LBB114_1: ; %atomicrmw.start7867; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=17868; GFX9-NEXT:    s_waitcnt vmcnt(0)7869; GFX9-NEXT:    v_min_u32_e32 v0, s6, v17870; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] glc7871; GFX9-NEXT:    s_waitcnt vmcnt(0)7872; GFX9-NEXT:    buffer_wbinvl1_vol7873; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v17874; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]7875; GFX9-NEXT:    v_mov_b32_e32 v1, v07876; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]7877; GFX9-NEXT:    s_cbranch_execnz .LBB114_17878; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end7879; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]7880; GFX9-NEXT:    s_setpc_b64 s[30:31]7881  %tmp0 = atomicrmw umin ptr addrspace(1) %ptr, i32 %in seq_cst7882  ret void7883}7884 7885define amdgpu_gfx void @global_atomic_umin_i32_noret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {7886; SI-LABEL: global_atomic_umin_i32_noret_offset_scalar:7887; SI:       ; %bb.0:7888; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7889; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -17890; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill7891; SI-NEXT:    s_mov_b64 exec, s[34:35]7892; SI-NEXT:    s_waitcnt expcnt(0)7893; SI-NEXT:    v_writelane_b32 v4, s6, 07894; SI-NEXT:    v_writelane_b32 v4, s7, 17895; SI-NEXT:    s_mov_b32 s34, s67896; SI-NEXT:    s_mov_b32 s7, 0xf0007897; SI-NEXT:    s_mov_b32 s6, -17898; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:167899; SI-NEXT:    s_mov_b64 s[36:37], 07900; SI-NEXT:  .LBB115_1: ; %atomicrmw.start7901; SI-NEXT:    ; =>This Inner Loop Header: Depth=17902; SI-NEXT:    s_waitcnt vmcnt(0)7903; SI-NEXT:    v_min_u32_e32 v0, s34, v17904; SI-NEXT:    s_waitcnt expcnt(0)7905; SI-NEXT:    v_mov_b32_e32 v3, v17906; SI-NEXT:    v_mov_b32_e32 v2, v07907; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 offset:16 glc7908; SI-NEXT:    s_waitcnt vmcnt(0)7909; SI-NEXT:    buffer_wbinvl17910; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v17911; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]7912; SI-NEXT:    v_mov_b32_e32 v1, v27913; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]7914; SI-NEXT:    s_cbranch_execnz .LBB115_17915; SI-NEXT:  ; %bb.2: ; %atomicrmw.end7916; SI-NEXT:    s_or_b64 exec, exec, s[36:37]7917; SI-NEXT:    v_readlane_b32 s7, v4, 17918; SI-NEXT:    v_readlane_b32 s6, v4, 07919; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -17920; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload7921; SI-NEXT:    s_mov_b64 exec, s[34:35]7922; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)7923; SI-NEXT:    s_setpc_b64 s[30:31]7924;7925; VI-LABEL: global_atomic_umin_i32_noret_offset_scalar:7926; VI:       ; %bb.0:7927; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7928; VI-NEXT:    s_add_u32 s34, s4, 167929; VI-NEXT:    s_addc_u32 s35, s5, 07930; VI-NEXT:    v_mov_b32_e32 v0, s347931; VI-NEXT:    v_mov_b32_e32 v1, s357932; VI-NEXT:    flat_load_dword v3, v[0:1]7933; VI-NEXT:    s_mov_b64 s[34:35], 07934; VI-NEXT:  .LBB115_1: ; %atomicrmw.start7935; VI-NEXT:    ; =>This Inner Loop Header: Depth=17936; VI-NEXT:    s_waitcnt vmcnt(0)7937; VI-NEXT:    v_min_u32_e32 v2, s6, v37938; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc7939; VI-NEXT:    s_waitcnt vmcnt(0)7940; VI-NEXT:    buffer_wbinvl1_vol7941; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v37942; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]7943; VI-NEXT:    v_mov_b32_e32 v3, v27944; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]7945; VI-NEXT:    s_cbranch_execnz .LBB115_17946; VI-NEXT:  ; %bb.2: ; %atomicrmw.end7947; VI-NEXT:    s_or_b64 exec, exec, s[34:35]7948; VI-NEXT:    s_setpc_b64 s[30:31]7949;7950; GFX9-LABEL: global_atomic_umin_i32_noret_offset_scalar:7951; GFX9:       ; %bb.0:7952; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7953; GFX9-NEXT:    v_mov_b32_e32 v2, 07954; GFX9-NEXT:    global_load_dword v1, v2, s[4:5] offset:167955; GFX9-NEXT:    s_mov_b64 s[34:35], 07956; GFX9-NEXT:  .LBB115_1: ; %atomicrmw.start7957; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=17958; GFX9-NEXT:    s_waitcnt vmcnt(0)7959; GFX9-NEXT:    v_min_u32_e32 v0, s6, v17960; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 glc7961; GFX9-NEXT:    s_waitcnt vmcnt(0)7962; GFX9-NEXT:    buffer_wbinvl1_vol7963; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v17964; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]7965; GFX9-NEXT:    v_mov_b32_e32 v1, v07966; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]7967; GFX9-NEXT:    s_cbranch_execnz .LBB115_17968; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end7969; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]7970; GFX9-NEXT:    s_setpc_b64 s[30:31]7971  %gep = getelementptr i32, ptr addrspace(1) %out, i32 47972  %tmp0 = atomicrmw umin ptr addrspace(1) %gep, i32 %in seq_cst7973  ret void7974}7975 7976define amdgpu_gfx i32 @global_atomic_umin_i32_ret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {7977; SI-LABEL: global_atomic_umin_i32_ret_scalar:7978; SI:       ; %bb.0:7979; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7980; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -17981; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill7982; SI-NEXT:    s_mov_b64 exec, s[34:35]7983; SI-NEXT:    s_waitcnt expcnt(0)7984; SI-NEXT:    v_writelane_b32 v3, s6, 07985; SI-NEXT:    v_writelane_b32 v3, s7, 17986; SI-NEXT:    s_mov_b32 s34, s67987; SI-NEXT:    s_mov_b32 s7, 0xf0007988; SI-NEXT:    s_mov_b32 s6, -17989; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 07990; SI-NEXT:    s_mov_b64 s[36:37], 07991; SI-NEXT:  .LBB116_1: ; %atomicrmw.start7992; SI-NEXT:    ; =>This Inner Loop Header: Depth=17993; SI-NEXT:    s_waitcnt vmcnt(0)7994; SI-NEXT:    v_mov_b32_e32 v2, v07995; SI-NEXT:    s_waitcnt expcnt(0)7996; SI-NEXT:    v_min_u32_e32 v1, s34, v27997; SI-NEXT:    v_mov_b32_e32 v0, v17998; SI-NEXT:    v_mov_b32_e32 v1, v27999; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc8000; SI-NEXT:    s_waitcnt vmcnt(0)8001; SI-NEXT:    buffer_wbinvl18002; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v28003; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]8004; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]8005; SI-NEXT:    s_cbranch_execnz .LBB116_18006; SI-NEXT:  ; %bb.2: ; %atomicrmw.end8007; SI-NEXT:    s_or_b64 exec, exec, s[36:37]8008; SI-NEXT:    v_readlane_b32 s7, v3, 18009; SI-NEXT:    v_readlane_b32 s6, v3, 08010; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -18011; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload8012; SI-NEXT:    s_mov_b64 exec, s[34:35]8013; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)8014; SI-NEXT:    s_setpc_b64 s[30:31]8015;8016; VI-LABEL: global_atomic_umin_i32_ret_scalar:8017; VI:       ; %bb.0:8018; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8019; VI-NEXT:    v_mov_b32_e32 v0, s48020; VI-NEXT:    v_mov_b32_e32 v1, s58021; VI-NEXT:    flat_load_dword v0, v[0:1]8022; VI-NEXT:    v_mov_b32_e32 v1, s48023; VI-NEXT:    s_mov_b64 s[34:35], 08024; VI-NEXT:    v_mov_b32_e32 v2, s58025; VI-NEXT:  .LBB116_1: ; %atomicrmw.start8026; VI-NEXT:    ; =>This Inner Loop Header: Depth=18027; VI-NEXT:    s_waitcnt vmcnt(0)8028; VI-NEXT:    v_mov_b32_e32 v4, v08029; VI-NEXT:    v_min_u32_e32 v3, s6, v48030; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc8031; VI-NEXT:    s_waitcnt vmcnt(0)8032; VI-NEXT:    buffer_wbinvl1_vol8033; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v48034; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]8035; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]8036; VI-NEXT:    s_cbranch_execnz .LBB116_18037; VI-NEXT:  ; %bb.2: ; %atomicrmw.end8038; VI-NEXT:    s_or_b64 exec, exec, s[34:35]8039; VI-NEXT:    s_setpc_b64 s[30:31]8040;8041; GFX9-LABEL: global_atomic_umin_i32_ret_scalar:8042; GFX9:       ; %bb.0:8043; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8044; GFX9-NEXT:    v_mov_b32_e32 v1, 08045; GFX9-NEXT:    global_load_dword v0, v1, s[4:5]8046; GFX9-NEXT:    s_mov_b64 s[34:35], 08047; GFX9-NEXT:  .LBB116_1: ; %atomicrmw.start8048; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=18049; GFX9-NEXT:    s_waitcnt vmcnt(0)8050; GFX9-NEXT:    v_mov_b32_e32 v3, v08051; GFX9-NEXT:    v_min_u32_e32 v2, s6, v38052; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] glc8053; GFX9-NEXT:    s_waitcnt vmcnt(0)8054; GFX9-NEXT:    buffer_wbinvl1_vol8055; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v38056; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]8057; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]8058; GFX9-NEXT:    s_cbranch_execnz .LBB116_18059; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end8060; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]8061; GFX9-NEXT:    s_setpc_b64 s[30:31]8062  %result = atomicrmw umin ptr addrspace(1) %ptr, i32 %in seq_cst8063  ret i32 %result8064}8065 8066define amdgpu_gfx i32 @global_atomic_umin_i32_ret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {8067; SI-LABEL: global_atomic_umin_i32_ret_offset_scalar:8068; SI:       ; %bb.0:8069; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8070; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -18071; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill8072; SI-NEXT:    s_mov_b64 exec, s[34:35]8073; SI-NEXT:    s_waitcnt expcnt(0)8074; SI-NEXT:    v_writelane_b32 v3, s6, 08075; SI-NEXT:    v_writelane_b32 v3, s7, 18076; SI-NEXT:    s_mov_b32 s34, s68077; SI-NEXT:    s_mov_b32 s7, 0xf0008078; SI-NEXT:    s_mov_b32 s6, -18079; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:168080; SI-NEXT:    s_mov_b64 s[36:37], 08081; SI-NEXT:  .LBB117_1: ; %atomicrmw.start8082; SI-NEXT:    ; =>This Inner Loop Header: Depth=18083; SI-NEXT:    s_waitcnt vmcnt(0)8084; SI-NEXT:    v_mov_b32_e32 v2, v08085; SI-NEXT:    s_waitcnt expcnt(0)8086; SI-NEXT:    v_min_u32_e32 v1, s34, v28087; SI-NEXT:    v_mov_b32_e32 v0, v18088; SI-NEXT:    v_mov_b32_e32 v1, v28089; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc8090; SI-NEXT:    s_waitcnt vmcnt(0)8091; SI-NEXT:    buffer_wbinvl18092; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v28093; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]8094; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]8095; SI-NEXT:    s_cbranch_execnz .LBB117_18096; SI-NEXT:  ; %bb.2: ; %atomicrmw.end8097; SI-NEXT:    s_or_b64 exec, exec, s[36:37]8098; SI-NEXT:    v_readlane_b32 s7, v3, 18099; SI-NEXT:    v_readlane_b32 s6, v3, 08100; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -18101; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload8102; SI-NEXT:    s_mov_b64 exec, s[34:35]8103; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)8104; SI-NEXT:    s_setpc_b64 s[30:31]8105;8106; VI-LABEL: global_atomic_umin_i32_ret_offset_scalar:8107; VI:       ; %bb.0:8108; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8109; VI-NEXT:    s_add_u32 s34, s4, 168110; VI-NEXT:    s_addc_u32 s35, s5, 08111; VI-NEXT:    v_mov_b32_e32 v1, s348112; VI-NEXT:    v_mov_b32_e32 v2, s358113; VI-NEXT:    flat_load_dword v0, v[1:2]8114; VI-NEXT:    s_mov_b64 s[34:35], 08115; VI-NEXT:  .LBB117_1: ; %atomicrmw.start8116; VI-NEXT:    ; =>This Inner Loop Header: Depth=18117; VI-NEXT:    s_waitcnt vmcnt(0)8118; VI-NEXT:    v_mov_b32_e32 v4, v08119; VI-NEXT:    v_min_u32_e32 v3, s6, v48120; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc8121; VI-NEXT:    s_waitcnt vmcnt(0)8122; VI-NEXT:    buffer_wbinvl1_vol8123; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v48124; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]8125; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]8126; VI-NEXT:    s_cbranch_execnz .LBB117_18127; VI-NEXT:  ; %bb.2: ; %atomicrmw.end8128; VI-NEXT:    s_or_b64 exec, exec, s[34:35]8129; VI-NEXT:    s_setpc_b64 s[30:31]8130;8131; GFX9-LABEL: global_atomic_umin_i32_ret_offset_scalar:8132; GFX9:       ; %bb.0:8133; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8134; GFX9-NEXT:    v_mov_b32_e32 v1, 08135; GFX9-NEXT:    global_load_dword v0, v1, s[4:5] offset:168136; GFX9-NEXT:    s_mov_b64 s[34:35], 08137; GFX9-NEXT:  .LBB117_1: ; %atomicrmw.start8138; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=18139; GFX9-NEXT:    s_waitcnt vmcnt(0)8140; GFX9-NEXT:    v_mov_b32_e32 v3, v08141; GFX9-NEXT:    v_min_u32_e32 v2, s6, v38142; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] offset:16 glc8143; GFX9-NEXT:    s_waitcnt vmcnt(0)8144; GFX9-NEXT:    buffer_wbinvl1_vol8145; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v38146; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]8147; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]8148; GFX9-NEXT:    s_cbranch_execnz .LBB117_18149; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end8150; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]8151; GFX9-NEXT:    s_setpc_b64 s[30:31]8152  %gep = getelementptr i32, ptr addrspace(1) %out, i32 48153  %result = atomicrmw umin ptr addrspace(1) %gep, i32 %in seq_cst8154  ret i32 %result8155}8156 8157define void @global_atomic_umin_i32_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {8158; SI-LABEL: global_atomic_umin_i32_noret_offset__amdgpu_no_remote_memory:8159; SI:       ; %bb.0:8160; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8161; SI-NEXT:    s_mov_b32 s6, 08162; SI-NEXT:    s_mov_b32 s7, 0xf0008163; SI-NEXT:    s_mov_b32 s4, s68164; SI-NEXT:    s_mov_b32 s5, s68165; SI-NEXT:    buffer_atomic_umin v2, v[0:1], s[4:7], 0 addr64 offset:168166; SI-NEXT:    s_waitcnt vmcnt(0)8167; SI-NEXT:    buffer_wbinvl18168; SI-NEXT:    s_waitcnt expcnt(0)8169; SI-NEXT:    s_setpc_b64 s[30:31]8170;8171; VI-LABEL: global_atomic_umin_i32_noret_offset__amdgpu_no_remote_memory:8172; VI:       ; %bb.0:8173; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8174; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v08175; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc8176; VI-NEXT:    flat_atomic_umin v[0:1], v28177; VI-NEXT:    s_waitcnt vmcnt(0)8178; VI-NEXT:    buffer_wbinvl1_vol8179; VI-NEXT:    s_setpc_b64 s[30:31]8180;8181; GFX9-LABEL: global_atomic_umin_i32_noret_offset__amdgpu_no_remote_memory:8182; GFX9:       ; %bb.0:8183; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8184; GFX9-NEXT:    global_atomic_umin v[0:1], v2, off offset:168185; GFX9-NEXT:    s_waitcnt vmcnt(0)8186; GFX9-NEXT:    buffer_wbinvl1_vol8187; GFX9-NEXT:    s_setpc_b64 s[30:31]8188  %gep = getelementptr i32, ptr addrspace(1) %out, i64 48189  %tmp0 = atomicrmw umin ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !08190  ret void8191}8192 8193define i32 @global_atomic_umin_i32_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {8194; SI-LABEL: global_atomic_umin_i32_ret_offset__amdgpu_no_remote_memory:8195; SI:       ; %bb.0:8196; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8197; SI-NEXT:    s_mov_b32 s6, 08198; SI-NEXT:    s_mov_b32 s7, 0xf0008199; SI-NEXT:    s_mov_b32 s4, s68200; SI-NEXT:    s_mov_b32 s5, s68201; SI-NEXT:    buffer_atomic_umin v2, v[0:1], s[4:7], 0 addr64 offset:16 glc8202; SI-NEXT:    s_waitcnt vmcnt(0)8203; SI-NEXT:    buffer_wbinvl18204; SI-NEXT:    v_mov_b32_e32 v0, v28205; SI-NEXT:    s_waitcnt expcnt(0)8206; SI-NEXT:    s_setpc_b64 s[30:31]8207;8208; VI-LABEL: global_atomic_umin_i32_ret_offset__amdgpu_no_remote_memory:8209; VI:       ; %bb.0:8210; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8211; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v08212; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc8213; VI-NEXT:    flat_atomic_umin v0, v[0:1], v2 glc8214; VI-NEXT:    s_waitcnt vmcnt(0)8215; VI-NEXT:    buffer_wbinvl1_vol8216; VI-NEXT:    s_setpc_b64 s[30:31]8217;8218; GFX9-LABEL: global_atomic_umin_i32_ret_offset__amdgpu_no_remote_memory:8219; GFX9:       ; %bb.0:8220; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8221; GFX9-NEXT:    global_atomic_umin v0, v[0:1], v2, off offset:16 glc8222; GFX9-NEXT:    s_waitcnt vmcnt(0)8223; GFX9-NEXT:    buffer_wbinvl1_vol8224; GFX9-NEXT:    s_setpc_b64 s[30:31]8225  %gep = getelementptr i32, ptr addrspace(1) %out, i64 48226  %result = atomicrmw umin ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !08227  ret i32 %result8228}8229 8230; ---------------------------------------------------------------------8231; atomicrmw min8232; ---------------------------------------------------------------------8233 8234define void @global_atomic_min_i32_noret(ptr addrspace(1) %ptr, i32 %in) {8235; SI-LABEL: global_atomic_min_i32_noret:8236; SI:       ; %bb.0:8237; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8238; SI-NEXT:    s_mov_b32 s6, 08239; SI-NEXT:    s_mov_b32 s7, 0xf0008240; SI-NEXT:    s_mov_b32 s4, s68241; SI-NEXT:    s_mov_b32 s5, s68242; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr648243; SI-NEXT:    s_mov_b64 s[8:9], 08244; SI-NEXT:  .LBB120_1: ; %atomicrmw.start8245; SI-NEXT:    ; =>This Inner Loop Header: Depth=18246; SI-NEXT:    s_waitcnt vmcnt(0)8247; SI-NEXT:    v_min_i32_e32 v3, v4, v28248; SI-NEXT:    s_waitcnt expcnt(0)8249; SI-NEXT:    v_mov_b32_e32 v6, v48250; SI-NEXT:    v_mov_b32_e32 v5, v38251; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc8252; SI-NEXT:    s_waitcnt vmcnt(0)8253; SI-NEXT:    buffer_wbinvl18254; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v48255; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]8256; SI-NEXT:    v_mov_b32_e32 v4, v58257; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]8258; SI-NEXT:    s_cbranch_execnz .LBB120_18259; SI-NEXT:  ; %bb.2: ; %atomicrmw.end8260; SI-NEXT:    s_or_b64 exec, exec, s[8:9]8261; SI-NEXT:    s_waitcnt expcnt(0)8262; SI-NEXT:    s_setpc_b64 s[30:31]8263;8264; VI-LABEL: global_atomic_min_i32_noret:8265; VI:       ; %bb.0:8266; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8267; VI-NEXT:    flat_load_dword v4, v[0:1]8268; VI-NEXT:    s_mov_b64 s[4:5], 08269; VI-NEXT:  .LBB120_1: ; %atomicrmw.start8270; VI-NEXT:    ; =>This Inner Loop Header: Depth=18271; VI-NEXT:    s_waitcnt vmcnt(0)8272; VI-NEXT:    v_min_i32_e32 v3, v4, v28273; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc8274; VI-NEXT:    s_waitcnt vmcnt(0)8275; VI-NEXT:    buffer_wbinvl1_vol8276; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v48277; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]8278; VI-NEXT:    v_mov_b32_e32 v4, v38279; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]8280; VI-NEXT:    s_cbranch_execnz .LBB120_18281; VI-NEXT:  ; %bb.2: ; %atomicrmw.end8282; VI-NEXT:    s_or_b64 exec, exec, s[4:5]8283; VI-NEXT:    s_setpc_b64 s[30:31]8284;8285; GFX9-LABEL: global_atomic_min_i32_noret:8286; GFX9:       ; %bb.0:8287; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8288; GFX9-NEXT:    global_load_dword v4, v[0:1], off8289; GFX9-NEXT:    s_mov_b64 s[4:5], 08290; GFX9-NEXT:  .LBB120_1: ; %atomicrmw.start8291; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=18292; GFX9-NEXT:    s_waitcnt vmcnt(0)8293; GFX9-NEXT:    v_min_i32_e32 v3, v4, v28294; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc8295; GFX9-NEXT:    s_waitcnt vmcnt(0)8296; GFX9-NEXT:    buffer_wbinvl1_vol8297; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v48298; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]8299; GFX9-NEXT:    v_mov_b32_e32 v4, v38300; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]8301; GFX9-NEXT:    s_cbranch_execnz .LBB120_18302; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end8303; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]8304; GFX9-NEXT:    s_setpc_b64 s[30:31]8305  %tmp0 = atomicrmw min ptr addrspace(1) %ptr, i32 %in seq_cst8306  ret void8307}8308 8309define void @global_atomic_min_i32_noret_offset(ptr addrspace(1) %out, i32 %in) {8310; SI-LABEL: global_atomic_min_i32_noret_offset:8311; SI:       ; %bb.0:8312; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8313; SI-NEXT:    s_mov_b32 s6, 08314; SI-NEXT:    s_mov_b32 s7, 0xf0008315; SI-NEXT:    s_mov_b32 s4, s68316; SI-NEXT:    s_mov_b32 s5, s68317; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:168318; SI-NEXT:    s_mov_b64 s[8:9], 08319; SI-NEXT:  .LBB121_1: ; %atomicrmw.start8320; SI-NEXT:    ; =>This Inner Loop Header: Depth=18321; SI-NEXT:    s_waitcnt vmcnt(0)8322; SI-NEXT:    v_min_i32_e32 v3, v4, v28323; SI-NEXT:    s_waitcnt expcnt(0)8324; SI-NEXT:    v_mov_b32_e32 v6, v48325; SI-NEXT:    v_mov_b32_e32 v5, v38326; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:16 glc8327; SI-NEXT:    s_waitcnt vmcnt(0)8328; SI-NEXT:    buffer_wbinvl18329; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v48330; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]8331; SI-NEXT:    v_mov_b32_e32 v4, v58332; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]8333; SI-NEXT:    s_cbranch_execnz .LBB121_18334; SI-NEXT:  ; %bb.2: ; %atomicrmw.end8335; SI-NEXT:    s_or_b64 exec, exec, s[8:9]8336; SI-NEXT:    s_waitcnt expcnt(0)8337; SI-NEXT:    s_setpc_b64 s[30:31]8338;8339; VI-LABEL: global_atomic_min_i32_noret_offset:8340; VI:       ; %bb.0:8341; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8342; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v08343; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc8344; VI-NEXT:    flat_load_dword v4, v[0:1]8345; VI-NEXT:    s_mov_b64 s[4:5], 08346; VI-NEXT:  .LBB121_1: ; %atomicrmw.start8347; VI-NEXT:    ; =>This Inner Loop Header: Depth=18348; VI-NEXT:    s_waitcnt vmcnt(0)8349; VI-NEXT:    v_min_i32_e32 v3, v4, v28350; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc8351; VI-NEXT:    s_waitcnt vmcnt(0)8352; VI-NEXT:    buffer_wbinvl1_vol8353; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v48354; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]8355; VI-NEXT:    v_mov_b32_e32 v4, v38356; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]8357; VI-NEXT:    s_cbranch_execnz .LBB121_18358; VI-NEXT:  ; %bb.2: ; %atomicrmw.end8359; VI-NEXT:    s_or_b64 exec, exec, s[4:5]8360; VI-NEXT:    s_setpc_b64 s[30:31]8361;8362; GFX9-LABEL: global_atomic_min_i32_noret_offset:8363; GFX9:       ; %bb.0:8364; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8365; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:168366; GFX9-NEXT:    s_mov_b64 s[4:5], 08367; GFX9-NEXT:  .LBB121_1: ; %atomicrmw.start8368; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=18369; GFX9-NEXT:    s_waitcnt vmcnt(0)8370; GFX9-NEXT:    v_min_i32_e32 v3, v4, v28371; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc8372; GFX9-NEXT:    s_waitcnt vmcnt(0)8373; GFX9-NEXT:    buffer_wbinvl1_vol8374; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v48375; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]8376; GFX9-NEXT:    v_mov_b32_e32 v4, v38377; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]8378; GFX9-NEXT:    s_cbranch_execnz .LBB121_18379; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end8380; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]8381; GFX9-NEXT:    s_setpc_b64 s[30:31]8382  %gep = getelementptr i32, ptr addrspace(1) %out, i32 48383  %tmp0 = atomicrmw min ptr addrspace(1) %gep, i32 %in seq_cst8384  ret void8385}8386 8387define i32 @global_atomic_min_i32_ret(ptr addrspace(1) %ptr, i32 %in) {8388; SI-LABEL: global_atomic_min_i32_ret:8389; SI:       ; %bb.0:8390; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8391; SI-NEXT:    s_mov_b32 s6, 08392; SI-NEXT:    s_mov_b32 s7, 0xf0008393; SI-NEXT:    s_mov_b32 s4, s68394; SI-NEXT:    s_mov_b32 s5, s68395; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr648396; SI-NEXT:    s_mov_b64 s[8:9], 08397; SI-NEXT:  .LBB122_1: ; %atomicrmw.start8398; SI-NEXT:    ; =>This Inner Loop Header: Depth=18399; SI-NEXT:    s_waitcnt vmcnt(0)8400; SI-NEXT:    v_mov_b32_e32 v5, v38401; SI-NEXT:    s_waitcnt expcnt(0)8402; SI-NEXT:    v_min_i32_e32 v4, v5, v28403; SI-NEXT:    v_mov_b32_e32 v3, v48404; SI-NEXT:    v_mov_b32_e32 v4, v58405; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc8406; SI-NEXT:    s_waitcnt vmcnt(0)8407; SI-NEXT:    buffer_wbinvl18408; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v58409; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]8410; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]8411; SI-NEXT:    s_cbranch_execnz .LBB122_18412; SI-NEXT:  ; %bb.2: ; %atomicrmw.end8413; SI-NEXT:    s_or_b64 exec, exec, s[8:9]8414; SI-NEXT:    v_mov_b32_e32 v0, v38415; SI-NEXT:    s_waitcnt expcnt(0)8416; SI-NEXT:    s_setpc_b64 s[30:31]8417;8418; VI-LABEL: global_atomic_min_i32_ret:8419; VI:       ; %bb.0:8420; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8421; VI-NEXT:    flat_load_dword v3, v[0:1]8422; VI-NEXT:    s_mov_b64 s[4:5], 08423; VI-NEXT:  .LBB122_1: ; %atomicrmw.start8424; VI-NEXT:    ; =>This Inner Loop Header: Depth=18425; VI-NEXT:    s_waitcnt vmcnt(0)8426; VI-NEXT:    v_mov_b32_e32 v4, v38427; VI-NEXT:    v_min_i32_e32 v3, v4, v28428; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc8429; VI-NEXT:    s_waitcnt vmcnt(0)8430; VI-NEXT:    buffer_wbinvl1_vol8431; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v48432; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]8433; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]8434; VI-NEXT:    s_cbranch_execnz .LBB122_18435; VI-NEXT:  ; %bb.2: ; %atomicrmw.end8436; VI-NEXT:    s_or_b64 exec, exec, s[4:5]8437; VI-NEXT:    v_mov_b32_e32 v0, v38438; VI-NEXT:    s_setpc_b64 s[30:31]8439;8440; GFX9-LABEL: global_atomic_min_i32_ret:8441; GFX9:       ; %bb.0:8442; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8443; GFX9-NEXT:    global_load_dword v3, v[0:1], off8444; GFX9-NEXT:    s_mov_b64 s[4:5], 08445; GFX9-NEXT:  .LBB122_1: ; %atomicrmw.start8446; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=18447; GFX9-NEXT:    s_waitcnt vmcnt(0)8448; GFX9-NEXT:    v_mov_b32_e32 v4, v38449; GFX9-NEXT:    v_min_i32_e32 v3, v4, v28450; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc8451; GFX9-NEXT:    s_waitcnt vmcnt(0)8452; GFX9-NEXT:    buffer_wbinvl1_vol8453; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v48454; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]8455; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]8456; GFX9-NEXT:    s_cbranch_execnz .LBB122_18457; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end8458; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]8459; GFX9-NEXT:    v_mov_b32_e32 v0, v38460; GFX9-NEXT:    s_setpc_b64 s[30:31]8461  %result = atomicrmw min ptr addrspace(1) %ptr, i32 %in seq_cst8462  ret i32 %result8463}8464 8465define i32 @global_atomic_min_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {8466; SI-LABEL: global_atomic_min_i32_ret_offset:8467; SI:       ; %bb.0:8468; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8469; SI-NEXT:    s_mov_b32 s6, 08470; SI-NEXT:    s_mov_b32 s7, 0xf0008471; SI-NEXT:    s_mov_b32 s4, s68472; SI-NEXT:    s_mov_b32 s5, s68473; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:168474; SI-NEXT:    s_mov_b64 s[8:9], 08475; SI-NEXT:  .LBB123_1: ; %atomicrmw.start8476; SI-NEXT:    ; =>This Inner Loop Header: Depth=18477; SI-NEXT:    s_waitcnt vmcnt(0)8478; SI-NEXT:    v_mov_b32_e32 v5, v38479; SI-NEXT:    s_waitcnt expcnt(0)8480; SI-NEXT:    v_min_i32_e32 v4, v5, v28481; SI-NEXT:    v_mov_b32_e32 v3, v48482; SI-NEXT:    v_mov_b32_e32 v4, v58483; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:16 glc8484; SI-NEXT:    s_waitcnt vmcnt(0)8485; SI-NEXT:    buffer_wbinvl18486; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v58487; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]8488; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]8489; SI-NEXT:    s_cbranch_execnz .LBB123_18490; SI-NEXT:  ; %bb.2: ; %atomicrmw.end8491; SI-NEXT:    s_or_b64 exec, exec, s[8:9]8492; SI-NEXT:    v_mov_b32_e32 v0, v38493; SI-NEXT:    s_waitcnt expcnt(0)8494; SI-NEXT:    s_setpc_b64 s[30:31]8495;8496; VI-LABEL: global_atomic_min_i32_ret_offset:8497; VI:       ; %bb.0:8498; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8499; VI-NEXT:    v_add_u32_e32 v3, vcc, 16, v08500; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc8501; VI-NEXT:    flat_load_dword v0, v[3:4]8502; VI-NEXT:    s_mov_b64 s[4:5], 08503; VI-NEXT:  .LBB123_1: ; %atomicrmw.start8504; VI-NEXT:    ; =>This Inner Loop Header: Depth=18505; VI-NEXT:    s_waitcnt vmcnt(0)8506; VI-NEXT:    v_mov_b32_e32 v1, v08507; VI-NEXT:    v_min_i32_e32 v0, v1, v28508; VI-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc8509; VI-NEXT:    s_waitcnt vmcnt(0)8510; VI-NEXT:    buffer_wbinvl1_vol8511; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v18512; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]8513; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]8514; VI-NEXT:    s_cbranch_execnz .LBB123_18515; VI-NEXT:  ; %bb.2: ; %atomicrmw.end8516; VI-NEXT:    s_or_b64 exec, exec, s[4:5]8517; VI-NEXT:    s_setpc_b64 s[30:31]8518;8519; GFX9-LABEL: global_atomic_min_i32_ret_offset:8520; GFX9:       ; %bb.0:8521; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8522; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:168523; GFX9-NEXT:    s_mov_b64 s[4:5], 08524; GFX9-NEXT:  .LBB123_1: ; %atomicrmw.start8525; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=18526; GFX9-NEXT:    s_waitcnt vmcnt(0)8527; GFX9-NEXT:    v_mov_b32_e32 v4, v38528; GFX9-NEXT:    v_min_i32_e32 v3, v4, v28529; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc8530; GFX9-NEXT:    s_waitcnt vmcnt(0)8531; GFX9-NEXT:    buffer_wbinvl1_vol8532; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v48533; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]8534; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]8535; GFX9-NEXT:    s_cbranch_execnz .LBB123_18536; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end8537; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]8538; GFX9-NEXT:    v_mov_b32_e32 v0, v38539; GFX9-NEXT:    s_setpc_b64 s[30:31]8540  %gep = getelementptr i32, ptr addrspace(1) %out, i32 48541  %result = atomicrmw min ptr addrspace(1) %gep, i32 %in seq_cst8542  ret i32 %result8543}8544 8545define amdgpu_gfx void @global_atomic_min_i32_noret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {8546; SI-LABEL: global_atomic_min_i32_noret_scalar:8547; SI:       ; %bb.0:8548; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8549; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -18550; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill8551; SI-NEXT:    s_mov_b64 exec, s[34:35]8552; SI-NEXT:    s_waitcnt expcnt(0)8553; SI-NEXT:    v_writelane_b32 v4, s6, 08554; SI-NEXT:    v_writelane_b32 v4, s7, 18555; SI-NEXT:    s_mov_b32 s34, s68556; SI-NEXT:    s_mov_b32 s7, 0xf0008557; SI-NEXT:    s_mov_b32 s6, -18558; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 08559; SI-NEXT:    s_mov_b64 s[36:37], 08560; SI-NEXT:  .LBB124_1: ; %atomicrmw.start8561; SI-NEXT:    ; =>This Inner Loop Header: Depth=18562; SI-NEXT:    s_waitcnt vmcnt(0)8563; SI-NEXT:    v_min_i32_e32 v0, s34, v18564; SI-NEXT:    s_waitcnt expcnt(0)8565; SI-NEXT:    v_mov_b32_e32 v3, v18566; SI-NEXT:    v_mov_b32_e32 v2, v08567; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc8568; SI-NEXT:    s_waitcnt vmcnt(0)8569; SI-NEXT:    buffer_wbinvl18570; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v18571; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]8572; SI-NEXT:    v_mov_b32_e32 v1, v28573; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]8574; SI-NEXT:    s_cbranch_execnz .LBB124_18575; SI-NEXT:  ; %bb.2: ; %atomicrmw.end8576; SI-NEXT:    s_or_b64 exec, exec, s[36:37]8577; SI-NEXT:    v_readlane_b32 s7, v4, 18578; SI-NEXT:    v_readlane_b32 s6, v4, 08579; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -18580; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload8581; SI-NEXT:    s_mov_b64 exec, s[34:35]8582; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)8583; SI-NEXT:    s_setpc_b64 s[30:31]8584;8585; VI-LABEL: global_atomic_min_i32_noret_scalar:8586; VI:       ; %bb.0:8587; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8588; VI-NEXT:    v_mov_b32_e32 v0, s48589; VI-NEXT:    v_mov_b32_e32 v1, s58590; VI-NEXT:    flat_load_dword v3, v[0:1]8591; VI-NEXT:    s_mov_b64 s[34:35], 08592; VI-NEXT:  .LBB124_1: ; %atomicrmw.start8593; VI-NEXT:    ; =>This Inner Loop Header: Depth=18594; VI-NEXT:    s_waitcnt vmcnt(0)8595; VI-NEXT:    v_min_i32_e32 v2, s6, v38596; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc8597; VI-NEXT:    s_waitcnt vmcnt(0)8598; VI-NEXT:    buffer_wbinvl1_vol8599; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v38600; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]8601; VI-NEXT:    v_mov_b32_e32 v3, v28602; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]8603; VI-NEXT:    s_cbranch_execnz .LBB124_18604; VI-NEXT:  ; %bb.2: ; %atomicrmw.end8605; VI-NEXT:    s_or_b64 exec, exec, s[34:35]8606; VI-NEXT:    s_setpc_b64 s[30:31]8607;8608; GFX9-LABEL: global_atomic_min_i32_noret_scalar:8609; GFX9:       ; %bb.0:8610; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8611; GFX9-NEXT:    v_mov_b32_e32 v2, 08612; GFX9-NEXT:    global_load_dword v1, v2, s[4:5]8613; GFX9-NEXT:    s_mov_b64 s[34:35], 08614; GFX9-NEXT:  .LBB124_1: ; %atomicrmw.start8615; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=18616; GFX9-NEXT:    s_waitcnt vmcnt(0)8617; GFX9-NEXT:    v_min_i32_e32 v0, s6, v18618; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] glc8619; GFX9-NEXT:    s_waitcnt vmcnt(0)8620; GFX9-NEXT:    buffer_wbinvl1_vol8621; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v18622; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]8623; GFX9-NEXT:    v_mov_b32_e32 v1, v08624; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]8625; GFX9-NEXT:    s_cbranch_execnz .LBB124_18626; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end8627; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]8628; GFX9-NEXT:    s_setpc_b64 s[30:31]8629  %tmp0 = atomicrmw min ptr addrspace(1) %ptr, i32 %in seq_cst8630  ret void8631}8632 8633define amdgpu_gfx void @global_atomic_min_i32_noret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {8634; SI-LABEL: global_atomic_min_i32_noret_offset_scalar:8635; SI:       ; %bb.0:8636; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8637; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -18638; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill8639; SI-NEXT:    s_mov_b64 exec, s[34:35]8640; SI-NEXT:    s_waitcnt expcnt(0)8641; SI-NEXT:    v_writelane_b32 v4, s6, 08642; SI-NEXT:    v_writelane_b32 v4, s7, 18643; SI-NEXT:    s_mov_b32 s34, s68644; SI-NEXT:    s_mov_b32 s7, 0xf0008645; SI-NEXT:    s_mov_b32 s6, -18646; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:168647; SI-NEXT:    s_mov_b64 s[36:37], 08648; SI-NEXT:  .LBB125_1: ; %atomicrmw.start8649; SI-NEXT:    ; =>This Inner Loop Header: Depth=18650; SI-NEXT:    s_waitcnt vmcnt(0)8651; SI-NEXT:    v_min_i32_e32 v0, s34, v18652; SI-NEXT:    s_waitcnt expcnt(0)8653; SI-NEXT:    v_mov_b32_e32 v3, v18654; SI-NEXT:    v_mov_b32_e32 v2, v08655; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 offset:16 glc8656; SI-NEXT:    s_waitcnt vmcnt(0)8657; SI-NEXT:    buffer_wbinvl18658; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v18659; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]8660; SI-NEXT:    v_mov_b32_e32 v1, v28661; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]8662; SI-NEXT:    s_cbranch_execnz .LBB125_18663; SI-NEXT:  ; %bb.2: ; %atomicrmw.end8664; SI-NEXT:    s_or_b64 exec, exec, s[36:37]8665; SI-NEXT:    v_readlane_b32 s7, v4, 18666; SI-NEXT:    v_readlane_b32 s6, v4, 08667; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -18668; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload8669; SI-NEXT:    s_mov_b64 exec, s[34:35]8670; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)8671; SI-NEXT:    s_setpc_b64 s[30:31]8672;8673; VI-LABEL: global_atomic_min_i32_noret_offset_scalar:8674; VI:       ; %bb.0:8675; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8676; VI-NEXT:    s_add_u32 s34, s4, 168677; VI-NEXT:    s_addc_u32 s35, s5, 08678; VI-NEXT:    v_mov_b32_e32 v0, s348679; VI-NEXT:    v_mov_b32_e32 v1, s358680; VI-NEXT:    flat_load_dword v3, v[0:1]8681; VI-NEXT:    s_mov_b64 s[34:35], 08682; VI-NEXT:  .LBB125_1: ; %atomicrmw.start8683; VI-NEXT:    ; =>This Inner Loop Header: Depth=18684; VI-NEXT:    s_waitcnt vmcnt(0)8685; VI-NEXT:    v_min_i32_e32 v2, s6, v38686; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc8687; VI-NEXT:    s_waitcnt vmcnt(0)8688; VI-NEXT:    buffer_wbinvl1_vol8689; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v38690; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]8691; VI-NEXT:    v_mov_b32_e32 v3, v28692; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]8693; VI-NEXT:    s_cbranch_execnz .LBB125_18694; VI-NEXT:  ; %bb.2: ; %atomicrmw.end8695; VI-NEXT:    s_or_b64 exec, exec, s[34:35]8696; VI-NEXT:    s_setpc_b64 s[30:31]8697;8698; GFX9-LABEL: global_atomic_min_i32_noret_offset_scalar:8699; GFX9:       ; %bb.0:8700; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8701; GFX9-NEXT:    v_mov_b32_e32 v2, 08702; GFX9-NEXT:    global_load_dword v1, v2, s[4:5] offset:168703; GFX9-NEXT:    s_mov_b64 s[34:35], 08704; GFX9-NEXT:  .LBB125_1: ; %atomicrmw.start8705; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=18706; GFX9-NEXT:    s_waitcnt vmcnt(0)8707; GFX9-NEXT:    v_min_i32_e32 v0, s6, v18708; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 glc8709; GFX9-NEXT:    s_waitcnt vmcnt(0)8710; GFX9-NEXT:    buffer_wbinvl1_vol8711; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v18712; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]8713; GFX9-NEXT:    v_mov_b32_e32 v1, v08714; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]8715; GFX9-NEXT:    s_cbranch_execnz .LBB125_18716; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end8717; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]8718; GFX9-NEXT:    s_setpc_b64 s[30:31]8719  %gep = getelementptr i32, ptr addrspace(1) %out, i32 48720  %tmp0 = atomicrmw min ptr addrspace(1) %gep, i32 %in seq_cst8721  ret void8722}8723 8724define amdgpu_gfx i32 @global_atomic_min_i32_ret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {8725; SI-LABEL: global_atomic_min_i32_ret_scalar:8726; SI:       ; %bb.0:8727; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8728; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -18729; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill8730; SI-NEXT:    s_mov_b64 exec, s[34:35]8731; SI-NEXT:    s_waitcnt expcnt(0)8732; SI-NEXT:    v_writelane_b32 v3, s6, 08733; SI-NEXT:    v_writelane_b32 v3, s7, 18734; SI-NEXT:    s_mov_b32 s34, s68735; SI-NEXT:    s_mov_b32 s7, 0xf0008736; SI-NEXT:    s_mov_b32 s6, -18737; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 08738; SI-NEXT:    s_mov_b64 s[36:37], 08739; SI-NEXT:  .LBB126_1: ; %atomicrmw.start8740; SI-NEXT:    ; =>This Inner Loop Header: Depth=18741; SI-NEXT:    s_waitcnt vmcnt(0)8742; SI-NEXT:    v_mov_b32_e32 v2, v08743; SI-NEXT:    s_waitcnt expcnt(0)8744; SI-NEXT:    v_min_i32_e32 v1, s34, v28745; SI-NEXT:    v_mov_b32_e32 v0, v18746; SI-NEXT:    v_mov_b32_e32 v1, v28747; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc8748; SI-NEXT:    s_waitcnt vmcnt(0)8749; SI-NEXT:    buffer_wbinvl18750; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v28751; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]8752; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]8753; SI-NEXT:    s_cbranch_execnz .LBB126_18754; SI-NEXT:  ; %bb.2: ; %atomicrmw.end8755; SI-NEXT:    s_or_b64 exec, exec, s[36:37]8756; SI-NEXT:    v_readlane_b32 s7, v3, 18757; SI-NEXT:    v_readlane_b32 s6, v3, 08758; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -18759; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload8760; SI-NEXT:    s_mov_b64 exec, s[34:35]8761; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)8762; SI-NEXT:    s_setpc_b64 s[30:31]8763;8764; VI-LABEL: global_atomic_min_i32_ret_scalar:8765; VI:       ; %bb.0:8766; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8767; VI-NEXT:    v_mov_b32_e32 v0, s48768; VI-NEXT:    v_mov_b32_e32 v1, s58769; VI-NEXT:    flat_load_dword v0, v[0:1]8770; VI-NEXT:    v_mov_b32_e32 v1, s48771; VI-NEXT:    s_mov_b64 s[34:35], 08772; VI-NEXT:    v_mov_b32_e32 v2, s58773; VI-NEXT:  .LBB126_1: ; %atomicrmw.start8774; VI-NEXT:    ; =>This Inner Loop Header: Depth=18775; VI-NEXT:    s_waitcnt vmcnt(0)8776; VI-NEXT:    v_mov_b32_e32 v4, v08777; VI-NEXT:    v_min_i32_e32 v3, s6, v48778; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc8779; VI-NEXT:    s_waitcnt vmcnt(0)8780; VI-NEXT:    buffer_wbinvl1_vol8781; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v48782; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]8783; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]8784; VI-NEXT:    s_cbranch_execnz .LBB126_18785; VI-NEXT:  ; %bb.2: ; %atomicrmw.end8786; VI-NEXT:    s_or_b64 exec, exec, s[34:35]8787; VI-NEXT:    s_setpc_b64 s[30:31]8788;8789; GFX9-LABEL: global_atomic_min_i32_ret_scalar:8790; GFX9:       ; %bb.0:8791; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8792; GFX9-NEXT:    v_mov_b32_e32 v1, 08793; GFX9-NEXT:    global_load_dword v0, v1, s[4:5]8794; GFX9-NEXT:    s_mov_b64 s[34:35], 08795; GFX9-NEXT:  .LBB126_1: ; %atomicrmw.start8796; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=18797; GFX9-NEXT:    s_waitcnt vmcnt(0)8798; GFX9-NEXT:    v_mov_b32_e32 v3, v08799; GFX9-NEXT:    v_min_i32_e32 v2, s6, v38800; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] glc8801; GFX9-NEXT:    s_waitcnt vmcnt(0)8802; GFX9-NEXT:    buffer_wbinvl1_vol8803; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v38804; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]8805; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]8806; GFX9-NEXT:    s_cbranch_execnz .LBB126_18807; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end8808; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]8809; GFX9-NEXT:    s_setpc_b64 s[30:31]8810  %result = atomicrmw min ptr addrspace(1) %ptr, i32 %in seq_cst8811  ret i32 %result8812}8813 8814define amdgpu_gfx i32 @global_atomic_min_i32_ret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {8815; SI-LABEL: global_atomic_min_i32_ret_offset_scalar:8816; SI:       ; %bb.0:8817; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8818; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -18819; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill8820; SI-NEXT:    s_mov_b64 exec, s[34:35]8821; SI-NEXT:    s_waitcnt expcnt(0)8822; SI-NEXT:    v_writelane_b32 v3, s6, 08823; SI-NEXT:    v_writelane_b32 v3, s7, 18824; SI-NEXT:    s_mov_b32 s34, s68825; SI-NEXT:    s_mov_b32 s7, 0xf0008826; SI-NEXT:    s_mov_b32 s6, -18827; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:168828; SI-NEXT:    s_mov_b64 s[36:37], 08829; SI-NEXT:  .LBB127_1: ; %atomicrmw.start8830; SI-NEXT:    ; =>This Inner Loop Header: Depth=18831; SI-NEXT:    s_waitcnt vmcnt(0)8832; SI-NEXT:    v_mov_b32_e32 v2, v08833; SI-NEXT:    s_waitcnt expcnt(0)8834; SI-NEXT:    v_min_i32_e32 v1, s34, v28835; SI-NEXT:    v_mov_b32_e32 v0, v18836; SI-NEXT:    v_mov_b32_e32 v1, v28837; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc8838; SI-NEXT:    s_waitcnt vmcnt(0)8839; SI-NEXT:    buffer_wbinvl18840; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v28841; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]8842; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]8843; SI-NEXT:    s_cbranch_execnz .LBB127_18844; SI-NEXT:  ; %bb.2: ; %atomicrmw.end8845; SI-NEXT:    s_or_b64 exec, exec, s[36:37]8846; SI-NEXT:    v_readlane_b32 s7, v3, 18847; SI-NEXT:    v_readlane_b32 s6, v3, 08848; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -18849; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload8850; SI-NEXT:    s_mov_b64 exec, s[34:35]8851; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)8852; SI-NEXT:    s_setpc_b64 s[30:31]8853;8854; VI-LABEL: global_atomic_min_i32_ret_offset_scalar:8855; VI:       ; %bb.0:8856; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8857; VI-NEXT:    s_add_u32 s34, s4, 168858; VI-NEXT:    s_addc_u32 s35, s5, 08859; VI-NEXT:    v_mov_b32_e32 v1, s348860; VI-NEXT:    v_mov_b32_e32 v2, s358861; VI-NEXT:    flat_load_dword v0, v[1:2]8862; VI-NEXT:    s_mov_b64 s[34:35], 08863; VI-NEXT:  .LBB127_1: ; %atomicrmw.start8864; VI-NEXT:    ; =>This Inner Loop Header: Depth=18865; VI-NEXT:    s_waitcnt vmcnt(0)8866; VI-NEXT:    v_mov_b32_e32 v4, v08867; VI-NEXT:    v_min_i32_e32 v3, s6, v48868; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc8869; VI-NEXT:    s_waitcnt vmcnt(0)8870; VI-NEXT:    buffer_wbinvl1_vol8871; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v48872; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]8873; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]8874; VI-NEXT:    s_cbranch_execnz .LBB127_18875; VI-NEXT:  ; %bb.2: ; %atomicrmw.end8876; VI-NEXT:    s_or_b64 exec, exec, s[34:35]8877; VI-NEXT:    s_setpc_b64 s[30:31]8878;8879; GFX9-LABEL: global_atomic_min_i32_ret_offset_scalar:8880; GFX9:       ; %bb.0:8881; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8882; GFX9-NEXT:    v_mov_b32_e32 v1, 08883; GFX9-NEXT:    global_load_dword v0, v1, s[4:5] offset:168884; GFX9-NEXT:    s_mov_b64 s[34:35], 08885; GFX9-NEXT:  .LBB127_1: ; %atomicrmw.start8886; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=18887; GFX9-NEXT:    s_waitcnt vmcnt(0)8888; GFX9-NEXT:    v_mov_b32_e32 v3, v08889; GFX9-NEXT:    v_min_i32_e32 v2, s6, v38890; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] offset:16 glc8891; GFX9-NEXT:    s_waitcnt vmcnt(0)8892; GFX9-NEXT:    buffer_wbinvl1_vol8893; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v38894; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]8895; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]8896; GFX9-NEXT:    s_cbranch_execnz .LBB127_18897; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end8898; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]8899; GFX9-NEXT:    s_setpc_b64 s[30:31]8900  %gep = getelementptr i32, ptr addrspace(1) %out, i32 48901  %result = atomicrmw min ptr addrspace(1) %gep, i32 %in seq_cst8902  ret i32 %result8903}8904 8905define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i32 %index) {8906; SI-LABEL: atomic_min_i32_addr64_offset:8907; SI:       ; %bb.0: ; %entry8908; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x98909; SI-NEXT:    s_waitcnt lgkmcnt(0)8910; SI-NEXT:    s_ashr_i32 s5, s3, 318911; SI-NEXT:    s_mov_b32 s4, s38912; SI-NEXT:    s_lshl_b64 s[4:5], s[4:5], 28913; SI-NEXT:    s_add_u32 s4, s0, s48914; SI-NEXT:    s_addc_u32 s5, s1, s58915; SI-NEXT:    s_load_dword s3, s[4:5], 0x48916; SI-NEXT:    s_mov_b64 s[0:1], 08917; SI-NEXT:    s_mov_b32 s7, 0xf0008918; SI-NEXT:    s_waitcnt lgkmcnt(0)8919; SI-NEXT:    v_mov_b32_e32 v1, s38920; SI-NEXT:    s_mov_b32 s6, -18921; SI-NEXT:  .LBB128_1: ; %atomicrmw.start8922; SI-NEXT:    ; =>This Inner Loop Header: Depth=18923; SI-NEXT:    v_min_i32_e32 v0, s2, v18924; SI-NEXT:    s_waitcnt expcnt(0)8925; SI-NEXT:    v_mov_b32_e32 v3, v18926; SI-NEXT:    v_mov_b32_e32 v2, v08927; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 offset:16 glc8928; SI-NEXT:    s_waitcnt vmcnt(0)8929; SI-NEXT:    buffer_wbinvl18930; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v18931; SI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]8932; SI-NEXT:    v_mov_b32_e32 v1, v28933; SI-NEXT:    s_andn2_b64 exec, exec, s[0:1]8934; SI-NEXT:    s_cbranch_execnz .LBB128_18935; SI-NEXT:  ; %bb.2: ; %atomicrmw.end8936; SI-NEXT:    s_endpgm8937;8938; VI-LABEL: atomic_min_i32_addr64_offset:8939; VI:       ; %bb.0: ; %entry8940; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x248941; VI-NEXT:    s_waitcnt lgkmcnt(0)8942; VI-NEXT:    s_ashr_i32 s5, s3, 318943; VI-NEXT:    s_mov_b32 s4, s38944; VI-NEXT:    s_lshl_b64 s[4:5], s[4:5], 28945; VI-NEXT:    s_add_u32 s4, s0, s48946; VI-NEXT:    s_addc_u32 s5, s1, s58947; VI-NEXT:    s_load_dword s3, s[4:5], 0x108948; VI-NEXT:    s_add_u32 s4, s4, 168949; VI-NEXT:    s_addc_u32 s5, s5, 08950; VI-NEXT:    v_mov_b32_e32 v0, s48951; VI-NEXT:    s_mov_b64 s[0:1], 08952; VI-NEXT:    s_waitcnt lgkmcnt(0)8953; VI-NEXT:    v_mov_b32_e32 v3, s38954; VI-NEXT:    v_mov_b32_e32 v1, s58955; VI-NEXT:  .LBB128_1: ; %atomicrmw.start8956; VI-NEXT:    ; =>This Inner Loop Header: Depth=18957; VI-NEXT:    v_min_i32_e32 v2, s2, v38958; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc8959; VI-NEXT:    s_waitcnt vmcnt(0)8960; VI-NEXT:    buffer_wbinvl1_vol8961; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v38962; VI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]8963; VI-NEXT:    v_mov_b32_e32 v3, v28964; VI-NEXT:    s_andn2_b64 exec, exec, s[0:1]8965; VI-NEXT:    s_cbranch_execnz .LBB128_18966; VI-NEXT:  ; %bb.2: ; %atomicrmw.end8967; VI-NEXT:    s_endpgm8968;8969; GFX9-LABEL: atomic_min_i32_addr64_offset:8970; GFX9:       ; %bb.0: ; %entry8971; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x248972; GFX9-NEXT:    v_mov_b32_e32 v2, 08973; GFX9-NEXT:    s_waitcnt lgkmcnt(0)8974; GFX9-NEXT:    s_ashr_i32 s5, s3, 318975; GFX9-NEXT:    s_mov_b32 s4, s38976; GFX9-NEXT:    s_lshl_b64 s[4:5], s[4:5], 28977; GFX9-NEXT:    s_add_u32 s0, s0, s48978; GFX9-NEXT:    s_addc_u32 s1, s1, s58979; GFX9-NEXT:    s_load_dword s3, s[0:1], 0x108980; GFX9-NEXT:    s_mov_b64 s[4:5], 08981; GFX9-NEXT:    s_waitcnt lgkmcnt(0)8982; GFX9-NEXT:    v_mov_b32_e32 v1, s38983; GFX9-NEXT:  .LBB128_1: ; %atomicrmw.start8984; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=18985; GFX9-NEXT:    v_min_i32_e32 v0, s2, v18986; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc8987; GFX9-NEXT:    s_waitcnt vmcnt(0)8988; GFX9-NEXT:    buffer_wbinvl1_vol8989; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v18990; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]8991; GFX9-NEXT:    v_mov_b32_e32 v1, v08992; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]8993; GFX9-NEXT:    s_cbranch_execnz .LBB128_18994; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end8995; GFX9-NEXT:    s_endpgm8996entry:8997  %ptr = getelementptr i32, ptr addrspace(1) %out, i32 %index8998  %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 48999  %tmp0 = atomicrmw min ptr addrspace(1) %gep, i32 %in seq_cst9000  ret void9001}9002 9003define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i32 %index) {9004; SI-LABEL: atomic_min_i32_ret_addr64_offset:9005; SI:       ; %bb.0: ; %entry9006; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd9007; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x99008; SI-NEXT:    s_waitcnt lgkmcnt(0)9009; SI-NEXT:    s_ashr_i32 s5, s9, 319010; SI-NEXT:    s_mov_b32 s4, s99011; SI-NEXT:    s_lshl_b64 s[4:5], s[4:5], 29012; SI-NEXT:    s_add_u32 s4, s0, s49013; SI-NEXT:    s_addc_u32 s5, s1, s59014; SI-NEXT:    s_load_dword s6, s[4:5], 0x49015; SI-NEXT:    s_mov_b64 s[0:1], 09016; SI-NEXT:    s_mov_b32 s7, 0xf0009017; SI-NEXT:    s_waitcnt lgkmcnt(0)9018; SI-NEXT:    v_mov_b32_e32 v1, s69019; SI-NEXT:    s_mov_b32 s6, -19020; SI-NEXT:  .LBB129_1: ; %atomicrmw.start9021; SI-NEXT:    ; =>This Inner Loop Header: Depth=19022; SI-NEXT:    v_min_i32_e32 v0, s8, v19023; SI-NEXT:    s_waitcnt expcnt(0)9024; SI-NEXT:    v_mov_b32_e32 v3, v19025; SI-NEXT:    v_mov_b32_e32 v2, v09026; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 offset:16 glc9027; SI-NEXT:    s_waitcnt vmcnt(0)9028; SI-NEXT:    buffer_wbinvl19029; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v19030; SI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]9031; SI-NEXT:    v_mov_b32_e32 v1, v29032; SI-NEXT:    s_andn2_b64 exec, exec, s[0:1]9033; SI-NEXT:    s_cbranch_execnz .LBB129_19034; SI-NEXT:  ; %bb.2: ; %atomicrmw.end9035; SI-NEXT:    s_or_b64 exec, exec, s[0:1]9036; SI-NEXT:    s_mov_b32 s7, 0xf0009037; SI-NEXT:    s_mov_b32 s6, -19038; SI-NEXT:    s_mov_b32 s4, s29039; SI-NEXT:    s_mov_b32 s5, s39040; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 09041; SI-NEXT:    s_endpgm9042;9043; VI-LABEL: atomic_min_i32_ret_addr64_offset:9044; VI:       ; %bb.0: ; %entry9045; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x349046; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x249047; VI-NEXT:    s_waitcnt lgkmcnt(0)9048; VI-NEXT:    s_ashr_i32 s5, s7, 319049; VI-NEXT:    s_mov_b32 s4, s79050; VI-NEXT:    s_lshl_b64 s[4:5], s[4:5], 29051; VI-NEXT:    s_add_u32 s4, s0, s49052; VI-NEXT:    s_addc_u32 s5, s1, s59053; VI-NEXT:    s_load_dword s7, s[4:5], 0x109054; VI-NEXT:    s_add_u32 s4, s4, 169055; VI-NEXT:    s_addc_u32 s5, s5, 09056; VI-NEXT:    v_mov_b32_e32 v0, s49057; VI-NEXT:    s_mov_b64 s[0:1], 09058; VI-NEXT:    s_waitcnt lgkmcnt(0)9059; VI-NEXT:    v_mov_b32_e32 v2, s79060; VI-NEXT:    v_mov_b32_e32 v1, s59061; VI-NEXT:  .LBB129_1: ; %atomicrmw.start9062; VI-NEXT:    ; =>This Inner Loop Header: Depth=19063; VI-NEXT:    v_mov_b32_e32 v3, v29064; VI-NEXT:    v_min_i32_e32 v2, s6, v39065; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc9066; VI-NEXT:    s_waitcnt vmcnt(0)9067; VI-NEXT:    buffer_wbinvl1_vol9068; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v39069; VI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]9070; VI-NEXT:    s_andn2_b64 exec, exec, s[0:1]9071; VI-NEXT:    s_cbranch_execnz .LBB129_19072; VI-NEXT:  ; %bb.2: ; %atomicrmw.end9073; VI-NEXT:    s_or_b64 exec, exec, s[0:1]9074; VI-NEXT:    v_mov_b32_e32 v0, s29075; VI-NEXT:    v_mov_b32_e32 v1, s39076; VI-NEXT:    flat_store_dword v[0:1], v29077; VI-NEXT:    s_endpgm9078;9079; GFX9-LABEL: atomic_min_i32_ret_addr64_offset:9080; GFX9:       ; %bb.0: ; %entry9081; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x349082; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x249083; GFX9-NEXT:    v_mov_b32_e32 v1, 09084; GFX9-NEXT:    s_waitcnt lgkmcnt(0)9085; GFX9-NEXT:    s_ashr_i32 s5, s7, 319086; GFX9-NEXT:    s_mov_b32 s4, s79087; GFX9-NEXT:    s_lshl_b64 s[4:5], s[4:5], 29088; GFX9-NEXT:    s_add_u32 s0, s0, s49089; GFX9-NEXT:    s_addc_u32 s1, s1, s59090; GFX9-NEXT:    s_load_dword s7, s[0:1], 0x109091; GFX9-NEXT:    s_mov_b64 s[4:5], 09092; GFX9-NEXT:    s_waitcnt lgkmcnt(0)9093; GFX9-NEXT:    v_mov_b32_e32 v0, s79094; GFX9-NEXT:  .LBB129_1: ; %atomicrmw.start9095; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=19096; GFX9-NEXT:    v_mov_b32_e32 v3, v09097; GFX9-NEXT:    v_min_i32_e32 v2, s6, v39098; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[0:1] offset:16 glc9099; GFX9-NEXT:    s_waitcnt vmcnt(0)9100; GFX9-NEXT:    buffer_wbinvl1_vol9101; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v39102; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]9103; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]9104; GFX9-NEXT:    s_cbranch_execnz .LBB129_19105; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end9106; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]9107; GFX9-NEXT:    v_mov_b32_e32 v1, 09108; GFX9-NEXT:    global_store_dword v1, v0, s[2:3]9109; GFX9-NEXT:    s_endpgm9110entry:9111  %ptr = getelementptr i32, ptr addrspace(1) %out, i32 %index9112  %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 49113  %tmp0 = atomicrmw min ptr addrspace(1) %gep, i32 %in seq_cst9114  store i32 %tmp0, ptr addrspace(1) %out29115  ret void9116}9117 9118define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {9119; SI-LABEL: atomic_min_i32:9120; SI:       ; %bb.0: ; %entry9121; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x99122; SI-NEXT:    s_load_dword s6, s[4:5], 0xb9123; SI-NEXT:    s_waitcnt lgkmcnt(0)9124; SI-NEXT:    s_load_dword s2, s[0:1], 0x09125; SI-NEXT:    s_mov_b64 s[4:5], 09126; SI-NEXT:    s_mov_b32 s3, 0xf0009127; SI-NEXT:    s_waitcnt lgkmcnt(0)9128; SI-NEXT:    v_mov_b32_e32 v1, s29129; SI-NEXT:    s_mov_b32 s2, -19130; SI-NEXT:  .LBB130_1: ; %atomicrmw.start9131; SI-NEXT:    ; =>This Inner Loop Header: Depth=19132; SI-NEXT:    v_min_i32_e32 v0, s6, v19133; SI-NEXT:    s_waitcnt expcnt(0)9134; SI-NEXT:    v_mov_b32_e32 v3, v19135; SI-NEXT:    v_mov_b32_e32 v2, v09136; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc9137; SI-NEXT:    s_waitcnt vmcnt(0)9138; SI-NEXT:    buffer_wbinvl19139; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v19140; SI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]9141; SI-NEXT:    v_mov_b32_e32 v1, v29142; SI-NEXT:    s_andn2_b64 exec, exec, s[4:5]9143; SI-NEXT:    s_cbranch_execnz .LBB130_19144; SI-NEXT:  ; %bb.2: ; %atomicrmw.end9145; SI-NEXT:    s_endpgm9146;9147; VI-LABEL: atomic_min_i32:9148; VI:       ; %bb.0: ; %entry9149; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x249150; VI-NEXT:    s_load_dword s2, s[4:5], 0x2c9151; VI-NEXT:    s_mov_b64 s[0:1], 09152; VI-NEXT:    s_waitcnt lgkmcnt(0)9153; VI-NEXT:    s_load_dword s3, s[6:7], 0x09154; VI-NEXT:    v_mov_b32_e32 v0, s69155; VI-NEXT:    v_mov_b32_e32 v1, s79156; VI-NEXT:    s_waitcnt lgkmcnt(0)9157; VI-NEXT:    v_mov_b32_e32 v3, s39158; VI-NEXT:  .LBB130_1: ; %atomicrmw.start9159; VI-NEXT:    ; =>This Inner Loop Header: Depth=19160; VI-NEXT:    v_min_i32_e32 v2, s2, v39161; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc9162; VI-NEXT:    s_waitcnt vmcnt(0)9163; VI-NEXT:    buffer_wbinvl1_vol9164; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v39165; VI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]9166; VI-NEXT:    v_mov_b32_e32 v3, v29167; VI-NEXT:    s_andn2_b64 exec, exec, s[0:1]9168; VI-NEXT:    s_cbranch_execnz .LBB130_19169; VI-NEXT:  ; %bb.2: ; %atomicrmw.end9170; VI-NEXT:    s_endpgm9171;9172; GFX9-LABEL: atomic_min_i32:9173; GFX9:       ; %bb.0: ; %entry9174; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x249175; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c9176; GFX9-NEXT:    s_mov_b64 s[2:3], 09177; GFX9-NEXT:    v_mov_b32_e32 v2, 09178; GFX9-NEXT:    s_waitcnt lgkmcnt(0)9179; GFX9-NEXT:    s_load_dword s4, s[0:1], 0x09180; GFX9-NEXT:    s_waitcnt lgkmcnt(0)9181; GFX9-NEXT:    v_mov_b32_e32 v1, s49182; GFX9-NEXT:  .LBB130_1: ; %atomicrmw.start9183; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=19184; GFX9-NEXT:    v_min_i32_e32 v0, s6, v19185; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc9186; GFX9-NEXT:    s_waitcnt vmcnt(0)9187; GFX9-NEXT:    buffer_wbinvl1_vol9188; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v19189; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]9190; GFX9-NEXT:    v_mov_b32_e32 v1, v09191; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]9192; GFX9-NEXT:    s_cbranch_execnz .LBB130_19193; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end9194; GFX9-NEXT:    s_endpgm9195entry:9196  %tmp0 = atomicrmw min ptr addrspace(1) %out, i32 %in seq_cst9197  ret void9198}9199 9200define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i32 %index) {9201; SI-LABEL: atomic_min_i32_ret_addr64:9202; SI:       ; %bb.0: ; %entry9203; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xd9204; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x99205; SI-NEXT:    s_waitcnt lgkmcnt(0)9206; SI-NEXT:    s_ashr_i32 s5, s9, 319207; SI-NEXT:    s_mov_b32 s4, s99208; SI-NEXT:    s_lshl_b64 s[4:5], s[4:5], 29209; SI-NEXT:    s_add_u32 s4, s0, s49210; SI-NEXT:    s_addc_u32 s5, s1, s59211; SI-NEXT:    s_load_dword s6, s[4:5], 0x09212; SI-NEXT:    s_mov_b64 s[0:1], 09213; SI-NEXT:    s_mov_b32 s7, 0xf0009214; SI-NEXT:    s_waitcnt lgkmcnt(0)9215; SI-NEXT:    v_mov_b32_e32 v1, s69216; SI-NEXT:    s_mov_b32 s6, -19217; SI-NEXT:  .LBB131_1: ; %atomicrmw.start9218; SI-NEXT:    ; =>This Inner Loop Header: Depth=19219; SI-NEXT:    v_min_i32_e32 v0, s8, v19220; SI-NEXT:    s_waitcnt expcnt(0)9221; SI-NEXT:    v_mov_b32_e32 v3, v19222; SI-NEXT:    v_mov_b32_e32 v2, v09223; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc9224; SI-NEXT:    s_waitcnt vmcnt(0)9225; SI-NEXT:    buffer_wbinvl19226; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v19227; SI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]9228; SI-NEXT:    v_mov_b32_e32 v1, v29229; SI-NEXT:    s_andn2_b64 exec, exec, s[0:1]9230; SI-NEXT:    s_cbranch_execnz .LBB131_19231; SI-NEXT:  ; %bb.2: ; %atomicrmw.end9232; SI-NEXT:    s_or_b64 exec, exec, s[0:1]9233; SI-NEXT:    s_mov_b32 s7, 0xf0009234; SI-NEXT:    s_mov_b32 s6, -19235; SI-NEXT:    s_mov_b32 s4, s29236; SI-NEXT:    s_mov_b32 s5, s39237; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 09238; SI-NEXT:    s_endpgm9239;9240; VI-LABEL: atomic_min_i32_ret_addr64:9241; VI:       ; %bb.0: ; %entry9242; VI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x349243; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x249244; VI-NEXT:    s_waitcnt lgkmcnt(0)9245; VI-NEXT:    s_ashr_i32 s5, s7, 319246; VI-NEXT:    s_mov_b32 s4, s79247; VI-NEXT:    s_lshl_b64 s[4:5], s[4:5], 29248; VI-NEXT:    s_add_u32 s4, s0, s49249; VI-NEXT:    s_addc_u32 s5, s1, s59250; VI-NEXT:    s_load_dword s7, s[4:5], 0x09251; VI-NEXT:    v_mov_b32_e32 v0, s49252; VI-NEXT:    s_mov_b64 s[0:1], 09253; VI-NEXT:    v_mov_b32_e32 v1, s59254; VI-NEXT:    s_waitcnt lgkmcnt(0)9255; VI-NEXT:    v_mov_b32_e32 v2, s79256; VI-NEXT:  .LBB131_1: ; %atomicrmw.start9257; VI-NEXT:    ; =>This Inner Loop Header: Depth=19258; VI-NEXT:    v_mov_b32_e32 v3, v29259; VI-NEXT:    v_min_i32_e32 v2, s6, v39260; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc9261; VI-NEXT:    s_waitcnt vmcnt(0)9262; VI-NEXT:    buffer_wbinvl1_vol9263; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v39264; VI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]9265; VI-NEXT:    s_andn2_b64 exec, exec, s[0:1]9266; VI-NEXT:    s_cbranch_execnz .LBB131_19267; VI-NEXT:  ; %bb.2: ; %atomicrmw.end9268; VI-NEXT:    s_or_b64 exec, exec, s[0:1]9269; VI-NEXT:    v_mov_b32_e32 v0, s29270; VI-NEXT:    v_mov_b32_e32 v1, s39271; VI-NEXT:    flat_store_dword v[0:1], v29272; VI-NEXT:    s_endpgm9273;9274; GFX9-LABEL: atomic_min_i32_ret_addr64:9275; GFX9:       ; %bb.0: ; %entry9276; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x349277; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x249278; GFX9-NEXT:    v_mov_b32_e32 v1, 09279; GFX9-NEXT:    s_waitcnt lgkmcnt(0)9280; GFX9-NEXT:    s_ashr_i32 s5, s7, 319281; GFX9-NEXT:    s_mov_b32 s4, s79282; GFX9-NEXT:    s_lshl_b64 s[4:5], s[4:5], 29283; GFX9-NEXT:    s_add_u32 s0, s0, s49284; GFX9-NEXT:    s_addc_u32 s1, s1, s59285; GFX9-NEXT:    s_load_dword s7, s[0:1], 0x09286; GFX9-NEXT:    s_mov_b64 s[4:5], 09287; GFX9-NEXT:    s_waitcnt lgkmcnt(0)9288; GFX9-NEXT:    v_mov_b32_e32 v0, s79289; GFX9-NEXT:  .LBB131_1: ; %atomicrmw.start9290; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=19291; GFX9-NEXT:    v_mov_b32_e32 v3, v09292; GFX9-NEXT:    v_min_i32_e32 v2, s6, v39293; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[0:1] glc9294; GFX9-NEXT:    s_waitcnt vmcnt(0)9295; GFX9-NEXT:    buffer_wbinvl1_vol9296; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v39297; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]9298; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]9299; GFX9-NEXT:    s_cbranch_execnz .LBB131_19300; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end9301; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]9302; GFX9-NEXT:    v_mov_b32_e32 v1, 09303; GFX9-NEXT:    global_store_dword v1, v0, s[2:3]9304; GFX9-NEXT:    s_endpgm9305entry:9306  %ptr = getelementptr i32, ptr addrspace(1) %out, i32 %index9307  %tmp0 = atomicrmw min ptr addrspace(1) %ptr, i32 %in seq_cst9308  store i32 %tmp0, ptr addrspace(1) %out29309  ret void9310}9311 9312define void @global_atomic_min_i32_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {9313; SI-LABEL: global_atomic_min_i32_noret_offset__amdgpu_no_remote_memory:9314; SI:       ; %bb.0:9315; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9316; SI-NEXT:    s_mov_b32 s6, 09317; SI-NEXT:    s_mov_b32 s7, 0xf0009318; SI-NEXT:    s_mov_b32 s4, s69319; SI-NEXT:    s_mov_b32 s5, s69320; SI-NEXT:    buffer_atomic_smin v2, v[0:1], s[4:7], 0 addr64 offset:169321; SI-NEXT:    s_waitcnt vmcnt(0)9322; SI-NEXT:    buffer_wbinvl19323; SI-NEXT:    s_waitcnt expcnt(0)9324; SI-NEXT:    s_setpc_b64 s[30:31]9325;9326; VI-LABEL: global_atomic_min_i32_noret_offset__amdgpu_no_remote_memory:9327; VI:       ; %bb.0:9328; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9329; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v09330; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc9331; VI-NEXT:    flat_atomic_smin v[0:1], v29332; VI-NEXT:    s_waitcnt vmcnt(0)9333; VI-NEXT:    buffer_wbinvl1_vol9334; VI-NEXT:    s_setpc_b64 s[30:31]9335;9336; GFX9-LABEL: global_atomic_min_i32_noret_offset__amdgpu_no_remote_memory:9337; GFX9:       ; %bb.0:9338; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9339; GFX9-NEXT:    global_atomic_smin v[0:1], v2, off offset:169340; GFX9-NEXT:    s_waitcnt vmcnt(0)9341; GFX9-NEXT:    buffer_wbinvl1_vol9342; GFX9-NEXT:    s_setpc_b64 s[30:31]9343  %gep = getelementptr i32, ptr addrspace(1) %out, i64 49344  %tmp0 = atomicrmw min ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !09345  ret void9346}9347 9348define i32 @global_atomic_min_i32_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {9349; SI-LABEL: global_atomic_min_i32_ret_offset__amdgpu_no_remote_memory:9350; SI:       ; %bb.0:9351; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9352; SI-NEXT:    s_mov_b32 s6, 09353; SI-NEXT:    s_mov_b32 s7, 0xf0009354; SI-NEXT:    s_mov_b32 s4, s69355; SI-NEXT:    s_mov_b32 s5, s69356; SI-NEXT:    buffer_atomic_smin v2, v[0:1], s[4:7], 0 addr64 offset:16 glc9357; SI-NEXT:    s_waitcnt vmcnt(0)9358; SI-NEXT:    buffer_wbinvl19359; SI-NEXT:    v_mov_b32_e32 v0, v29360; SI-NEXT:    s_waitcnt expcnt(0)9361; SI-NEXT:    s_setpc_b64 s[30:31]9362;9363; VI-LABEL: global_atomic_min_i32_ret_offset__amdgpu_no_remote_memory:9364; VI:       ; %bb.0:9365; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9366; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v09367; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc9368; VI-NEXT:    flat_atomic_smin v0, v[0:1], v2 glc9369; VI-NEXT:    s_waitcnt vmcnt(0)9370; VI-NEXT:    buffer_wbinvl1_vol9371; VI-NEXT:    s_setpc_b64 s[30:31]9372;9373; GFX9-LABEL: global_atomic_min_i32_ret_offset__amdgpu_no_remote_memory:9374; GFX9:       ; %bb.0:9375; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9376; GFX9-NEXT:    global_atomic_smin v0, v[0:1], v2, off offset:16 glc9377; GFX9-NEXT:    s_waitcnt vmcnt(0)9378; GFX9-NEXT:    buffer_wbinvl1_vol9379; GFX9-NEXT:    s_setpc_b64 s[30:31]9380  %gep = getelementptr i32, ptr addrspace(1) %out, i64 49381  %result = atomicrmw min ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !09382  ret i32 %result9383}9384 9385; ---------------------------------------------------------------------9386; atomicrmw uinc_wrap9387; ---------------------------------------------------------------------9388 9389define void @global_atomic_uinc_wrap_i32_noret(ptr addrspace(1) %ptr, i32 %in) {9390; SI-LABEL: global_atomic_uinc_wrap_i32_noret:9391; SI:       ; %bb.0:9392; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9393; SI-NEXT:    s_mov_b32 s6, 09394; SI-NEXT:    s_mov_b32 s7, 0xf0009395; SI-NEXT:    s_mov_b32 s4, s69396; SI-NEXT:    s_mov_b32 s5, s69397; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr649398; SI-NEXT:    s_mov_b64 s[8:9], 09399; SI-NEXT:  .LBB134_1: ; %atomicrmw.start9400; SI-NEXT:    ; =>This Inner Loop Header: Depth=19401; SI-NEXT:    s_waitcnt vmcnt(0)9402; SI-NEXT:    v_add_i32_e32 v3, vcc, 1, v49403; SI-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v29404; SI-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc9405; SI-NEXT:    s_waitcnt expcnt(0)9406; SI-NEXT:    v_mov_b32_e32 v6, v49407; SI-NEXT:    v_mov_b32_e32 v5, v39408; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc9409; SI-NEXT:    s_waitcnt vmcnt(0)9410; SI-NEXT:    buffer_wbinvl19411; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v49412; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]9413; SI-NEXT:    v_mov_b32_e32 v4, v59414; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]9415; SI-NEXT:    s_cbranch_execnz .LBB134_19416; SI-NEXT:  ; %bb.2: ; %atomicrmw.end9417; SI-NEXT:    s_or_b64 exec, exec, s[8:9]9418; SI-NEXT:    s_waitcnt expcnt(0)9419; SI-NEXT:    s_setpc_b64 s[30:31]9420;9421; VI-LABEL: global_atomic_uinc_wrap_i32_noret:9422; VI:       ; %bb.0:9423; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9424; VI-NEXT:    flat_load_dword v4, v[0:1]9425; VI-NEXT:    s_mov_b64 s[4:5], 09426; VI-NEXT:  .LBB134_1: ; %atomicrmw.start9427; VI-NEXT:    ; =>This Inner Loop Header: Depth=19428; VI-NEXT:    s_waitcnt vmcnt(0)9429; VI-NEXT:    v_add_u32_e32 v3, vcc, 1, v49430; VI-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v29431; VI-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc9432; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc9433; VI-NEXT:    s_waitcnt vmcnt(0)9434; VI-NEXT:    buffer_wbinvl1_vol9435; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v49436; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]9437; VI-NEXT:    v_mov_b32_e32 v4, v39438; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]9439; VI-NEXT:    s_cbranch_execnz .LBB134_19440; VI-NEXT:  ; %bb.2: ; %atomicrmw.end9441; VI-NEXT:    s_or_b64 exec, exec, s[4:5]9442; VI-NEXT:    s_setpc_b64 s[30:31]9443;9444; GFX9-LABEL: global_atomic_uinc_wrap_i32_noret:9445; GFX9:       ; %bb.0:9446; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9447; GFX9-NEXT:    global_load_dword v4, v[0:1], off9448; GFX9-NEXT:    s_mov_b64 s[4:5], 09449; GFX9-NEXT:  .LBB134_1: ; %atomicrmw.start9450; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=19451; GFX9-NEXT:    s_waitcnt vmcnt(0)9452; GFX9-NEXT:    v_add_u32_e32 v3, 1, v49453; GFX9-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v29454; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc9455; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc9456; GFX9-NEXT:    s_waitcnt vmcnt(0)9457; GFX9-NEXT:    buffer_wbinvl1_vol9458; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v49459; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]9460; GFX9-NEXT:    v_mov_b32_e32 v4, v39461; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]9462; GFX9-NEXT:    s_cbranch_execnz .LBB134_19463; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end9464; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]9465; GFX9-NEXT:    s_setpc_b64 s[30:31]9466  %tmp0 = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i32 %in seq_cst9467  ret void9468}9469 9470define void @global_atomic_uinc_wrap_i32_noret_offset(ptr addrspace(1) %out, i32 %in) {9471; SI-LABEL: global_atomic_uinc_wrap_i32_noret_offset:9472; SI:       ; %bb.0:9473; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9474; SI-NEXT:    s_mov_b32 s6, 09475; SI-NEXT:    s_mov_b32 s7, 0xf0009476; SI-NEXT:    s_mov_b32 s4, s69477; SI-NEXT:    s_mov_b32 s5, s69478; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:169479; SI-NEXT:    s_mov_b64 s[8:9], 09480; SI-NEXT:  .LBB135_1: ; %atomicrmw.start9481; SI-NEXT:    ; =>This Inner Loop Header: Depth=19482; SI-NEXT:    s_waitcnt vmcnt(0)9483; SI-NEXT:    v_add_i32_e32 v3, vcc, 1, v49484; SI-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v29485; SI-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc9486; SI-NEXT:    s_waitcnt expcnt(0)9487; SI-NEXT:    v_mov_b32_e32 v6, v49488; SI-NEXT:    v_mov_b32_e32 v5, v39489; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:16 glc9490; SI-NEXT:    s_waitcnt vmcnt(0)9491; SI-NEXT:    buffer_wbinvl19492; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v49493; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]9494; SI-NEXT:    v_mov_b32_e32 v4, v59495; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]9496; SI-NEXT:    s_cbranch_execnz .LBB135_19497; SI-NEXT:  ; %bb.2: ; %atomicrmw.end9498; SI-NEXT:    s_or_b64 exec, exec, s[8:9]9499; SI-NEXT:    s_waitcnt expcnt(0)9500; SI-NEXT:    s_setpc_b64 s[30:31]9501;9502; VI-LABEL: global_atomic_uinc_wrap_i32_noret_offset:9503; VI:       ; %bb.0:9504; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9505; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v09506; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc9507; VI-NEXT:    flat_load_dword v4, v[0:1]9508; VI-NEXT:    s_mov_b64 s[4:5], 09509; VI-NEXT:  .LBB135_1: ; %atomicrmw.start9510; VI-NEXT:    ; =>This Inner Loop Header: Depth=19511; VI-NEXT:    s_waitcnt vmcnt(0)9512; VI-NEXT:    v_add_u32_e32 v3, vcc, 1, v49513; VI-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v29514; VI-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc9515; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc9516; VI-NEXT:    s_waitcnt vmcnt(0)9517; VI-NEXT:    buffer_wbinvl1_vol9518; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v49519; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]9520; VI-NEXT:    v_mov_b32_e32 v4, v39521; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]9522; VI-NEXT:    s_cbranch_execnz .LBB135_19523; VI-NEXT:  ; %bb.2: ; %atomicrmw.end9524; VI-NEXT:    s_or_b64 exec, exec, s[4:5]9525; VI-NEXT:    s_setpc_b64 s[30:31]9526;9527; GFX9-LABEL: global_atomic_uinc_wrap_i32_noret_offset:9528; GFX9:       ; %bb.0:9529; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9530; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:169531; GFX9-NEXT:    s_mov_b64 s[4:5], 09532; GFX9-NEXT:  .LBB135_1: ; %atomicrmw.start9533; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=19534; GFX9-NEXT:    s_waitcnt vmcnt(0)9535; GFX9-NEXT:    v_add_u32_e32 v3, 1, v49536; GFX9-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v29537; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc9538; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc9539; GFX9-NEXT:    s_waitcnt vmcnt(0)9540; GFX9-NEXT:    buffer_wbinvl1_vol9541; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v49542; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]9543; GFX9-NEXT:    v_mov_b32_e32 v4, v39544; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]9545; GFX9-NEXT:    s_cbranch_execnz .LBB135_19546; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end9547; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]9548; GFX9-NEXT:    s_setpc_b64 s[30:31]9549  %gep = getelementptr i32, ptr addrspace(1) %out, i32 49550  %tmp0 = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 %in seq_cst9551  ret void9552}9553 9554define i32 @global_atomic_uinc_wrap_i32_ret(ptr addrspace(1) %ptr, i32 %in) {9555; SI-LABEL: global_atomic_uinc_wrap_i32_ret:9556; SI:       ; %bb.0:9557; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9558; SI-NEXT:    s_mov_b32 s6, 09559; SI-NEXT:    s_mov_b32 s7, 0xf0009560; SI-NEXT:    s_mov_b32 s4, s69561; SI-NEXT:    s_mov_b32 s5, s69562; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr649563; SI-NEXT:    s_mov_b64 s[8:9], 09564; SI-NEXT:  .LBB136_1: ; %atomicrmw.start9565; SI-NEXT:    ; =>This Inner Loop Header: Depth=19566; SI-NEXT:    s_waitcnt vmcnt(0)9567; SI-NEXT:    v_mov_b32_e32 v5, v39568; SI-NEXT:    s_waitcnt expcnt(0)9569; SI-NEXT:    v_add_i32_e32 v3, vcc, 1, v59570; SI-NEXT:    v_cmp_lt_u32_e32 vcc, v5, v29571; SI-NEXT:    v_cndmask_b32_e32 v4, 0, v3, vcc9572; SI-NEXT:    v_mov_b32_e32 v3, v49573; SI-NEXT:    v_mov_b32_e32 v4, v59574; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc9575; SI-NEXT:    s_waitcnt vmcnt(0)9576; SI-NEXT:    buffer_wbinvl19577; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v59578; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]9579; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]9580; SI-NEXT:    s_cbranch_execnz .LBB136_19581; SI-NEXT:  ; %bb.2: ; %atomicrmw.end9582; SI-NEXT:    s_or_b64 exec, exec, s[8:9]9583; SI-NEXT:    v_mov_b32_e32 v0, v39584; SI-NEXT:    s_waitcnt expcnt(0)9585; SI-NEXT:    s_setpc_b64 s[30:31]9586;9587; VI-LABEL: global_atomic_uinc_wrap_i32_ret:9588; VI:       ; %bb.0:9589; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9590; VI-NEXT:    flat_load_dword v3, v[0:1]9591; VI-NEXT:    s_mov_b64 s[4:5], 09592; VI-NEXT:  .LBB136_1: ; %atomicrmw.start9593; VI-NEXT:    ; =>This Inner Loop Header: Depth=19594; VI-NEXT:    s_waitcnt vmcnt(0)9595; VI-NEXT:    v_mov_b32_e32 v4, v39596; VI-NEXT:    v_add_u32_e32 v3, vcc, 1, v49597; VI-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v29598; VI-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc9599; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc9600; VI-NEXT:    s_waitcnt vmcnt(0)9601; VI-NEXT:    buffer_wbinvl1_vol9602; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v49603; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]9604; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]9605; VI-NEXT:    s_cbranch_execnz .LBB136_19606; VI-NEXT:  ; %bb.2: ; %atomicrmw.end9607; VI-NEXT:    s_or_b64 exec, exec, s[4:5]9608; VI-NEXT:    v_mov_b32_e32 v0, v39609; VI-NEXT:    s_setpc_b64 s[30:31]9610;9611; GFX9-LABEL: global_atomic_uinc_wrap_i32_ret:9612; GFX9:       ; %bb.0:9613; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9614; GFX9-NEXT:    global_load_dword v3, v[0:1], off9615; GFX9-NEXT:    s_mov_b64 s[4:5], 09616; GFX9-NEXT:  .LBB136_1: ; %atomicrmw.start9617; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=19618; GFX9-NEXT:    s_waitcnt vmcnt(0)9619; GFX9-NEXT:    v_mov_b32_e32 v4, v39620; GFX9-NEXT:    v_add_u32_e32 v3, 1, v49621; GFX9-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v29622; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc9623; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc9624; GFX9-NEXT:    s_waitcnt vmcnt(0)9625; GFX9-NEXT:    buffer_wbinvl1_vol9626; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v49627; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]9628; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]9629; GFX9-NEXT:    s_cbranch_execnz .LBB136_19630; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end9631; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]9632; GFX9-NEXT:    v_mov_b32_e32 v0, v39633; GFX9-NEXT:    s_setpc_b64 s[30:31]9634  %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i32 %in seq_cst9635  ret i32 %result9636}9637 9638define i32 @global_atomic_uinc_wrap_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {9639; SI-LABEL: global_atomic_uinc_wrap_i32_ret_offset:9640; SI:       ; %bb.0:9641; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9642; SI-NEXT:    s_mov_b32 s6, 09643; SI-NEXT:    s_mov_b32 s7, 0xf0009644; SI-NEXT:    s_mov_b32 s4, s69645; SI-NEXT:    s_mov_b32 s5, s69646; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:169647; SI-NEXT:    s_mov_b64 s[8:9], 09648; SI-NEXT:  .LBB137_1: ; %atomicrmw.start9649; SI-NEXT:    ; =>This Inner Loop Header: Depth=19650; SI-NEXT:    s_waitcnt vmcnt(0)9651; SI-NEXT:    v_mov_b32_e32 v5, v39652; SI-NEXT:    s_waitcnt expcnt(0)9653; SI-NEXT:    v_add_i32_e32 v3, vcc, 1, v59654; SI-NEXT:    v_cmp_lt_u32_e32 vcc, v5, v29655; SI-NEXT:    v_cndmask_b32_e32 v4, 0, v3, vcc9656; SI-NEXT:    v_mov_b32_e32 v3, v49657; SI-NEXT:    v_mov_b32_e32 v4, v59658; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:16 glc9659; SI-NEXT:    s_waitcnt vmcnt(0)9660; SI-NEXT:    buffer_wbinvl19661; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v59662; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]9663; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]9664; SI-NEXT:    s_cbranch_execnz .LBB137_19665; SI-NEXT:  ; %bb.2: ; %atomicrmw.end9666; SI-NEXT:    s_or_b64 exec, exec, s[8:9]9667; SI-NEXT:    v_mov_b32_e32 v0, v39668; SI-NEXT:    s_waitcnt expcnt(0)9669; SI-NEXT:    s_setpc_b64 s[30:31]9670;9671; VI-LABEL: global_atomic_uinc_wrap_i32_ret_offset:9672; VI:       ; %bb.0:9673; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9674; VI-NEXT:    v_add_u32_e32 v3, vcc, 16, v09675; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc9676; VI-NEXT:    flat_load_dword v0, v[3:4]9677; VI-NEXT:    s_mov_b64 s[4:5], 09678; VI-NEXT:  .LBB137_1: ; %atomicrmw.start9679; VI-NEXT:    ; =>This Inner Loop Header: Depth=19680; VI-NEXT:    s_waitcnt vmcnt(0)9681; VI-NEXT:    v_mov_b32_e32 v1, v09682; VI-NEXT:    v_add_u32_e32 v0, vcc, 1, v19683; VI-NEXT:    v_cmp_lt_u32_e32 vcc, v1, v29684; VI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc9685; VI-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc9686; VI-NEXT:    s_waitcnt vmcnt(0)9687; VI-NEXT:    buffer_wbinvl1_vol9688; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v19689; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]9690; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]9691; VI-NEXT:    s_cbranch_execnz .LBB137_19692; VI-NEXT:  ; %bb.2: ; %atomicrmw.end9693; VI-NEXT:    s_or_b64 exec, exec, s[4:5]9694; VI-NEXT:    s_setpc_b64 s[30:31]9695;9696; GFX9-LABEL: global_atomic_uinc_wrap_i32_ret_offset:9697; GFX9:       ; %bb.0:9698; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9699; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:169700; GFX9-NEXT:    s_mov_b64 s[4:5], 09701; GFX9-NEXT:  .LBB137_1: ; %atomicrmw.start9702; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=19703; GFX9-NEXT:    s_waitcnt vmcnt(0)9704; GFX9-NEXT:    v_mov_b32_e32 v4, v39705; GFX9-NEXT:    v_add_u32_e32 v3, 1, v49706; GFX9-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v29707; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc9708; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc9709; GFX9-NEXT:    s_waitcnt vmcnt(0)9710; GFX9-NEXT:    buffer_wbinvl1_vol9711; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v49712; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]9713; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]9714; GFX9-NEXT:    s_cbranch_execnz .LBB137_19715; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end9716; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]9717; GFX9-NEXT:    v_mov_b32_e32 v0, v39718; GFX9-NEXT:    s_setpc_b64 s[30:31]9719  %gep = getelementptr i32, ptr addrspace(1) %out, i32 49720  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 %in seq_cst9721  ret i32 %result9722}9723 9724define amdgpu_gfx void @global_atomic_uinc_wrap_i32_noret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {9725; SI-LABEL: global_atomic_uinc_wrap_i32_noret_scalar:9726; SI:       ; %bb.0:9727; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9728; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -19729; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill9730; SI-NEXT:    s_mov_b64 exec, s[34:35]9731; SI-NEXT:    s_waitcnt expcnt(0)9732; SI-NEXT:    v_writelane_b32 v4, s6, 09733; SI-NEXT:    v_writelane_b32 v4, s7, 19734; SI-NEXT:    s_mov_b32 s34, s69735; SI-NEXT:    s_mov_b32 s7, 0xf0009736; SI-NEXT:    s_mov_b32 s6, -19737; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 09738; SI-NEXT:    s_mov_b64 s[36:37], 09739; SI-NEXT:  .LBB138_1: ; %atomicrmw.start9740; SI-NEXT:    ; =>This Inner Loop Header: Depth=19741; SI-NEXT:    s_waitcnt vmcnt(0)9742; SI-NEXT:    v_add_i32_e32 v0, vcc, 1, v19743; SI-NEXT:    v_cmp_gt_u32_e32 vcc, s34, v19744; SI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc9745; SI-NEXT:    s_waitcnt expcnt(0)9746; SI-NEXT:    v_mov_b32_e32 v3, v19747; SI-NEXT:    v_mov_b32_e32 v2, v09748; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc9749; SI-NEXT:    s_waitcnt vmcnt(0)9750; SI-NEXT:    buffer_wbinvl19751; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v19752; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]9753; SI-NEXT:    v_mov_b32_e32 v1, v29754; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]9755; SI-NEXT:    s_cbranch_execnz .LBB138_19756; SI-NEXT:  ; %bb.2: ; %atomicrmw.end9757; SI-NEXT:    s_or_b64 exec, exec, s[36:37]9758; SI-NEXT:    v_readlane_b32 s7, v4, 19759; SI-NEXT:    v_readlane_b32 s6, v4, 09760; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -19761; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload9762; SI-NEXT:    s_mov_b64 exec, s[34:35]9763; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)9764; SI-NEXT:    s_setpc_b64 s[30:31]9765;9766; VI-LABEL: global_atomic_uinc_wrap_i32_noret_scalar:9767; VI:       ; %bb.0:9768; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9769; VI-NEXT:    v_mov_b32_e32 v0, s49770; VI-NEXT:    v_mov_b32_e32 v1, s59771; VI-NEXT:    flat_load_dword v3, v[0:1]9772; VI-NEXT:    s_mov_b64 s[34:35], 09773; VI-NEXT:  .LBB138_1: ; %atomicrmw.start9774; VI-NEXT:    ; =>This Inner Loop Header: Depth=19775; VI-NEXT:    s_waitcnt vmcnt(0)9776; VI-NEXT:    v_add_u32_e32 v2, vcc, 1, v39777; VI-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v39778; VI-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc9779; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc9780; VI-NEXT:    s_waitcnt vmcnt(0)9781; VI-NEXT:    buffer_wbinvl1_vol9782; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v39783; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]9784; VI-NEXT:    v_mov_b32_e32 v3, v29785; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]9786; VI-NEXT:    s_cbranch_execnz .LBB138_19787; VI-NEXT:  ; %bb.2: ; %atomicrmw.end9788; VI-NEXT:    s_or_b64 exec, exec, s[34:35]9789; VI-NEXT:    s_setpc_b64 s[30:31]9790;9791; GFX9-LABEL: global_atomic_uinc_wrap_i32_noret_scalar:9792; GFX9:       ; %bb.0:9793; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9794; GFX9-NEXT:    v_mov_b32_e32 v2, 09795; GFX9-NEXT:    global_load_dword v1, v2, s[4:5]9796; GFX9-NEXT:    s_mov_b64 s[34:35], 09797; GFX9-NEXT:  .LBB138_1: ; %atomicrmw.start9798; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=19799; GFX9-NEXT:    s_waitcnt vmcnt(0)9800; GFX9-NEXT:    v_add_u32_e32 v0, 1, v19801; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v19802; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc9803; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] glc9804; GFX9-NEXT:    s_waitcnt vmcnt(0)9805; GFX9-NEXT:    buffer_wbinvl1_vol9806; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v19807; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]9808; GFX9-NEXT:    v_mov_b32_e32 v1, v09809; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]9810; GFX9-NEXT:    s_cbranch_execnz .LBB138_19811; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end9812; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]9813; GFX9-NEXT:    s_setpc_b64 s[30:31]9814  %tmp0 = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i32 %in seq_cst9815  ret void9816}9817 9818define amdgpu_gfx void @global_atomic_uinc_wrap_i32_noret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {9819; SI-LABEL: global_atomic_uinc_wrap_i32_noret_offset_scalar:9820; SI:       ; %bb.0:9821; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9822; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -19823; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill9824; SI-NEXT:    s_mov_b64 exec, s[34:35]9825; SI-NEXT:    s_waitcnt expcnt(0)9826; SI-NEXT:    v_writelane_b32 v4, s6, 09827; SI-NEXT:    v_writelane_b32 v4, s7, 19828; SI-NEXT:    s_mov_b32 s34, s69829; SI-NEXT:    s_mov_b32 s7, 0xf0009830; SI-NEXT:    s_mov_b32 s6, -19831; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:169832; SI-NEXT:    s_mov_b64 s[36:37], 09833; SI-NEXT:  .LBB139_1: ; %atomicrmw.start9834; SI-NEXT:    ; =>This Inner Loop Header: Depth=19835; SI-NEXT:    s_waitcnt vmcnt(0)9836; SI-NEXT:    v_add_i32_e32 v0, vcc, 1, v19837; SI-NEXT:    v_cmp_gt_u32_e32 vcc, s34, v19838; SI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc9839; SI-NEXT:    s_waitcnt expcnt(0)9840; SI-NEXT:    v_mov_b32_e32 v3, v19841; SI-NEXT:    v_mov_b32_e32 v2, v09842; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 offset:16 glc9843; SI-NEXT:    s_waitcnt vmcnt(0)9844; SI-NEXT:    buffer_wbinvl19845; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v19846; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]9847; SI-NEXT:    v_mov_b32_e32 v1, v29848; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]9849; SI-NEXT:    s_cbranch_execnz .LBB139_19850; SI-NEXT:  ; %bb.2: ; %atomicrmw.end9851; SI-NEXT:    s_or_b64 exec, exec, s[36:37]9852; SI-NEXT:    v_readlane_b32 s7, v4, 19853; SI-NEXT:    v_readlane_b32 s6, v4, 09854; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -19855; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload9856; SI-NEXT:    s_mov_b64 exec, s[34:35]9857; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)9858; SI-NEXT:    s_setpc_b64 s[30:31]9859;9860; VI-LABEL: global_atomic_uinc_wrap_i32_noret_offset_scalar:9861; VI:       ; %bb.0:9862; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9863; VI-NEXT:    s_add_u32 s34, s4, 169864; VI-NEXT:    s_addc_u32 s35, s5, 09865; VI-NEXT:    v_mov_b32_e32 v0, s349866; VI-NEXT:    v_mov_b32_e32 v1, s359867; VI-NEXT:    flat_load_dword v3, v[0:1]9868; VI-NEXT:    s_mov_b64 s[34:35], 09869; VI-NEXT:  .LBB139_1: ; %atomicrmw.start9870; VI-NEXT:    ; =>This Inner Loop Header: Depth=19871; VI-NEXT:    s_waitcnt vmcnt(0)9872; VI-NEXT:    v_add_u32_e32 v2, vcc, 1, v39873; VI-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v39874; VI-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc9875; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc9876; VI-NEXT:    s_waitcnt vmcnt(0)9877; VI-NEXT:    buffer_wbinvl1_vol9878; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v39879; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]9880; VI-NEXT:    v_mov_b32_e32 v3, v29881; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]9882; VI-NEXT:    s_cbranch_execnz .LBB139_19883; VI-NEXT:  ; %bb.2: ; %atomicrmw.end9884; VI-NEXT:    s_or_b64 exec, exec, s[34:35]9885; VI-NEXT:    s_setpc_b64 s[30:31]9886;9887; GFX9-LABEL: global_atomic_uinc_wrap_i32_noret_offset_scalar:9888; GFX9:       ; %bb.0:9889; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9890; GFX9-NEXT:    v_mov_b32_e32 v2, 09891; GFX9-NEXT:    global_load_dword v1, v2, s[4:5] offset:169892; GFX9-NEXT:    s_mov_b64 s[34:35], 09893; GFX9-NEXT:  .LBB139_1: ; %atomicrmw.start9894; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=19895; GFX9-NEXT:    s_waitcnt vmcnt(0)9896; GFX9-NEXT:    v_add_u32_e32 v0, 1, v19897; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v19898; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc9899; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 glc9900; GFX9-NEXT:    s_waitcnt vmcnt(0)9901; GFX9-NEXT:    buffer_wbinvl1_vol9902; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v19903; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]9904; GFX9-NEXT:    v_mov_b32_e32 v1, v09905; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]9906; GFX9-NEXT:    s_cbranch_execnz .LBB139_19907; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end9908; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]9909; GFX9-NEXT:    s_setpc_b64 s[30:31]9910  %gep = getelementptr i32, ptr addrspace(1) %out, i32 49911  %tmp0 = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 %in seq_cst9912  ret void9913}9914 9915define amdgpu_gfx i32 @global_atomic_uinc_wrap_i32_ret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {9916; SI-LABEL: global_atomic_uinc_wrap_i32_ret_scalar:9917; SI:       ; %bb.0:9918; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9919; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -19920; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill9921; SI-NEXT:    s_mov_b64 exec, s[34:35]9922; SI-NEXT:    s_waitcnt expcnt(0)9923; SI-NEXT:    v_writelane_b32 v3, s6, 09924; SI-NEXT:    v_writelane_b32 v3, s7, 19925; SI-NEXT:    s_mov_b32 s34, s69926; SI-NEXT:    s_mov_b32 s7, 0xf0009927; SI-NEXT:    s_mov_b32 s6, -19928; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 09929; SI-NEXT:    s_mov_b64 s[36:37], 09930; SI-NEXT:  .LBB140_1: ; %atomicrmw.start9931; SI-NEXT:    ; =>This Inner Loop Header: Depth=19932; SI-NEXT:    s_waitcnt vmcnt(0)9933; SI-NEXT:    v_mov_b32_e32 v2, v09934; SI-NEXT:    s_waitcnt expcnt(0)9935; SI-NEXT:    v_add_i32_e32 v0, vcc, 1, v29936; SI-NEXT:    v_cmp_gt_u32_e32 vcc, s34, v29937; SI-NEXT:    v_cndmask_b32_e32 v1, 0, v0, vcc9938; SI-NEXT:    v_mov_b32_e32 v0, v19939; SI-NEXT:    v_mov_b32_e32 v1, v29940; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc9941; SI-NEXT:    s_waitcnt vmcnt(0)9942; SI-NEXT:    buffer_wbinvl19943; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v29944; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]9945; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]9946; SI-NEXT:    s_cbranch_execnz .LBB140_19947; SI-NEXT:  ; %bb.2: ; %atomicrmw.end9948; SI-NEXT:    s_or_b64 exec, exec, s[36:37]9949; SI-NEXT:    v_readlane_b32 s7, v3, 19950; SI-NEXT:    v_readlane_b32 s6, v3, 09951; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -19952; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload9953; SI-NEXT:    s_mov_b64 exec, s[34:35]9954; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)9955; SI-NEXT:    s_setpc_b64 s[30:31]9956;9957; VI-LABEL: global_atomic_uinc_wrap_i32_ret_scalar:9958; VI:       ; %bb.0:9959; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9960; VI-NEXT:    v_mov_b32_e32 v0, s49961; VI-NEXT:    v_mov_b32_e32 v1, s59962; VI-NEXT:    flat_load_dword v0, v[0:1]9963; VI-NEXT:    v_mov_b32_e32 v1, s49964; VI-NEXT:    s_mov_b64 s[34:35], 09965; VI-NEXT:    v_mov_b32_e32 v2, s59966; VI-NEXT:  .LBB140_1: ; %atomicrmw.start9967; VI-NEXT:    ; =>This Inner Loop Header: Depth=19968; VI-NEXT:    s_waitcnt vmcnt(0)9969; VI-NEXT:    v_mov_b32_e32 v4, v09970; VI-NEXT:    v_add_u32_e32 v0, vcc, 1, v49971; VI-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v49972; VI-NEXT:    v_cndmask_b32_e32 v3, 0, v0, vcc9973; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc9974; VI-NEXT:    s_waitcnt vmcnt(0)9975; VI-NEXT:    buffer_wbinvl1_vol9976; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v49977; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]9978; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]9979; VI-NEXT:    s_cbranch_execnz .LBB140_19980; VI-NEXT:  ; %bb.2: ; %atomicrmw.end9981; VI-NEXT:    s_or_b64 exec, exec, s[34:35]9982; VI-NEXT:    s_setpc_b64 s[30:31]9983;9984; GFX9-LABEL: global_atomic_uinc_wrap_i32_ret_scalar:9985; GFX9:       ; %bb.0:9986; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9987; GFX9-NEXT:    v_mov_b32_e32 v1, 09988; GFX9-NEXT:    global_load_dword v0, v1, s[4:5]9989; GFX9-NEXT:    s_mov_b64 s[34:35], 09990; GFX9-NEXT:  .LBB140_1: ; %atomicrmw.start9991; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=19992; GFX9-NEXT:    s_waitcnt vmcnt(0)9993; GFX9-NEXT:    v_mov_b32_e32 v3, v09994; GFX9-NEXT:    v_add_u32_e32 v0, 1, v39995; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v39996; GFX9-NEXT:    v_cndmask_b32_e32 v2, 0, v0, vcc9997; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] glc9998; GFX9-NEXT:    s_waitcnt vmcnt(0)9999; GFX9-NEXT:    buffer_wbinvl1_vol10000; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v310001; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]10002; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]10003; GFX9-NEXT:    s_cbranch_execnz .LBB140_110004; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end10005; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]10006; GFX9-NEXT:    s_setpc_b64 s[30:31]10007  %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i32 %in seq_cst10008  ret i32 %result10009}10010 10011define amdgpu_gfx i32 @global_atomic_uinc_wrap_i32_ret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {10012; SI-LABEL: global_atomic_uinc_wrap_i32_ret_offset_scalar:10013; SI:       ; %bb.0:10014; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10015; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -110016; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill10017; SI-NEXT:    s_mov_b64 exec, s[34:35]10018; SI-NEXT:    s_waitcnt expcnt(0)10019; SI-NEXT:    v_writelane_b32 v3, s6, 010020; SI-NEXT:    v_writelane_b32 v3, s7, 110021; SI-NEXT:    s_mov_b32 s34, s610022; SI-NEXT:    s_mov_b32 s7, 0xf00010023; SI-NEXT:    s_mov_b32 s6, -110024; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:1610025; SI-NEXT:    s_mov_b64 s[36:37], 010026; SI-NEXT:  .LBB141_1: ; %atomicrmw.start10027; SI-NEXT:    ; =>This Inner Loop Header: Depth=110028; SI-NEXT:    s_waitcnt vmcnt(0)10029; SI-NEXT:    v_mov_b32_e32 v2, v010030; SI-NEXT:    s_waitcnt expcnt(0)10031; SI-NEXT:    v_add_i32_e32 v0, vcc, 1, v210032; SI-NEXT:    v_cmp_gt_u32_e32 vcc, s34, v210033; SI-NEXT:    v_cndmask_b32_e32 v1, 0, v0, vcc10034; SI-NEXT:    v_mov_b32_e32 v0, v110035; SI-NEXT:    v_mov_b32_e32 v1, v210036; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc10037; SI-NEXT:    s_waitcnt vmcnt(0)10038; SI-NEXT:    buffer_wbinvl110039; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v210040; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]10041; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]10042; SI-NEXT:    s_cbranch_execnz .LBB141_110043; SI-NEXT:  ; %bb.2: ; %atomicrmw.end10044; SI-NEXT:    s_or_b64 exec, exec, s[36:37]10045; SI-NEXT:    v_readlane_b32 s7, v3, 110046; SI-NEXT:    v_readlane_b32 s6, v3, 010047; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -110048; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload10049; SI-NEXT:    s_mov_b64 exec, s[34:35]10050; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)10051; SI-NEXT:    s_setpc_b64 s[30:31]10052;10053; VI-LABEL: global_atomic_uinc_wrap_i32_ret_offset_scalar:10054; VI:       ; %bb.0:10055; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10056; VI-NEXT:    s_add_u32 s34, s4, 1610057; VI-NEXT:    s_addc_u32 s35, s5, 010058; VI-NEXT:    v_mov_b32_e32 v1, s3410059; VI-NEXT:    v_mov_b32_e32 v2, s3510060; VI-NEXT:    flat_load_dword v0, v[1:2]10061; VI-NEXT:    s_mov_b64 s[34:35], 010062; VI-NEXT:  .LBB141_1: ; %atomicrmw.start10063; VI-NEXT:    ; =>This Inner Loop Header: Depth=110064; VI-NEXT:    s_waitcnt vmcnt(0)10065; VI-NEXT:    v_mov_b32_e32 v4, v010066; VI-NEXT:    v_add_u32_e32 v0, vcc, 1, v410067; VI-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v410068; VI-NEXT:    v_cndmask_b32_e32 v3, 0, v0, vcc10069; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc10070; VI-NEXT:    s_waitcnt vmcnt(0)10071; VI-NEXT:    buffer_wbinvl1_vol10072; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v410073; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]10074; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]10075; VI-NEXT:    s_cbranch_execnz .LBB141_110076; VI-NEXT:  ; %bb.2: ; %atomicrmw.end10077; VI-NEXT:    s_or_b64 exec, exec, s[34:35]10078; VI-NEXT:    s_setpc_b64 s[30:31]10079;10080; GFX9-LABEL: global_atomic_uinc_wrap_i32_ret_offset_scalar:10081; GFX9:       ; %bb.0:10082; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10083; GFX9-NEXT:    v_mov_b32_e32 v1, 010084; GFX9-NEXT:    global_load_dword v0, v1, s[4:5] offset:1610085; GFX9-NEXT:    s_mov_b64 s[34:35], 010086; GFX9-NEXT:  .LBB141_1: ; %atomicrmw.start10087; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=110088; GFX9-NEXT:    s_waitcnt vmcnt(0)10089; GFX9-NEXT:    v_mov_b32_e32 v3, v010090; GFX9-NEXT:    v_add_u32_e32 v0, 1, v310091; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v310092; GFX9-NEXT:    v_cndmask_b32_e32 v2, 0, v0, vcc10093; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] offset:16 glc10094; GFX9-NEXT:    s_waitcnt vmcnt(0)10095; GFX9-NEXT:    buffer_wbinvl1_vol10096; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v310097; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]10098; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]10099; GFX9-NEXT:    s_cbranch_execnz .LBB141_110100; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end10101; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]10102; GFX9-NEXT:    s_setpc_b64 s[30:31]10103  %gep = getelementptr i32, ptr addrspace(1) %out, i32 410104  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 %in seq_cst10105  ret i32 %result10106}10107 10108define void @global_atomic_uinc_wrap_i32_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {10109; SI-LABEL: global_atomic_uinc_wrap_i32_noret_offset__amdgpu_no_remote_memory:10110; SI:       ; %bb.0:10111; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10112; SI-NEXT:    s_mov_b32 s6, 010113; SI-NEXT:    s_mov_b32 s7, 0xf00010114; SI-NEXT:    s_mov_b32 s4, s610115; SI-NEXT:    s_mov_b32 s5, s610116; SI-NEXT:    buffer_atomic_inc v2, v[0:1], s[4:7], 0 addr64 offset:1610117; SI-NEXT:    s_waitcnt vmcnt(0)10118; SI-NEXT:    buffer_wbinvl110119; SI-NEXT:    s_waitcnt expcnt(0)10120; SI-NEXT:    s_setpc_b64 s[30:31]10121;10122; VI-LABEL: global_atomic_uinc_wrap_i32_noret_offset__amdgpu_no_remote_memory:10123; VI:       ; %bb.0:10124; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10125; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v010126; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc10127; VI-NEXT:    flat_atomic_inc v[0:1], v210128; VI-NEXT:    s_waitcnt vmcnt(0)10129; VI-NEXT:    buffer_wbinvl1_vol10130; VI-NEXT:    s_setpc_b64 s[30:31]10131;10132; GFX9-LABEL: global_atomic_uinc_wrap_i32_noret_offset__amdgpu_no_remote_memory:10133; GFX9:       ; %bb.0:10134; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10135; GFX9-NEXT:    global_atomic_inc v[0:1], v2, off offset:1610136; GFX9-NEXT:    s_waitcnt vmcnt(0)10137; GFX9-NEXT:    buffer_wbinvl1_vol10138; GFX9-NEXT:    s_setpc_b64 s[30:31]10139  %gep = getelementptr i32, ptr addrspace(1) %out, i64 410140  %tmp0 = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !010141  ret void10142}10143 10144define i32 @global_atomic_uinc_wrap_i32_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {10145; SI-LABEL: global_atomic_uinc_wrap_i32_ret_offset__amdgpu_no_remote_memory:10146; SI:       ; %bb.0:10147; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10148; SI-NEXT:    s_mov_b32 s6, 010149; SI-NEXT:    s_mov_b32 s7, 0xf00010150; SI-NEXT:    s_mov_b32 s4, s610151; SI-NEXT:    s_mov_b32 s5, s610152; SI-NEXT:    buffer_atomic_inc v2, v[0:1], s[4:7], 0 addr64 offset:16 glc10153; SI-NEXT:    s_waitcnt vmcnt(0)10154; SI-NEXT:    buffer_wbinvl110155; SI-NEXT:    v_mov_b32_e32 v0, v210156; SI-NEXT:    s_waitcnt expcnt(0)10157; SI-NEXT:    s_setpc_b64 s[30:31]10158;10159; VI-LABEL: global_atomic_uinc_wrap_i32_ret_offset__amdgpu_no_remote_memory:10160; VI:       ; %bb.0:10161; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10162; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v010163; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc10164; VI-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc10165; VI-NEXT:    s_waitcnt vmcnt(0)10166; VI-NEXT:    buffer_wbinvl1_vol10167; VI-NEXT:    s_setpc_b64 s[30:31]10168;10169; GFX9-LABEL: global_atomic_uinc_wrap_i32_ret_offset__amdgpu_no_remote_memory:10170; GFX9:       ; %bb.0:10171; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10172; GFX9-NEXT:    global_atomic_inc v0, v[0:1], v2, off offset:16 glc10173; GFX9-NEXT:    s_waitcnt vmcnt(0)10174; GFX9-NEXT:    buffer_wbinvl1_vol10175; GFX9-NEXT:    s_setpc_b64 s[30:31]10176  %gep = getelementptr i32, ptr addrspace(1) %out, i64 410177  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !010178  ret i32 %result10179}10180 10181; ---------------------------------------------------------------------10182; atomicrmw udec_wrap10183; ---------------------------------------------------------------------10184 10185define void @global_atomic_udec_wrap_i32_noret(ptr addrspace(1) %ptr, i32 %in) {10186; SI-LABEL: global_atomic_udec_wrap_i32_noret:10187; SI:       ; %bb.0:10188; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10189; SI-NEXT:    s_mov_b32 s10, 010190; SI-NEXT:    s_mov_b32 s11, 0xf00010191; SI-NEXT:    s_mov_b32 s8, s1010192; SI-NEXT:    s_mov_b32 s9, s1010193; SI-NEXT:    buffer_load_dword v4, v[0:1], s[8:11], 0 addr6410194; SI-NEXT:    s_mov_b64 s[6:7], 010195; SI-NEXT:  .LBB144_1: ; %atomicrmw.start10196; SI-NEXT:    ; =>This Inner Loop Header: Depth=110197; SI-NEXT:    s_waitcnt vmcnt(0)10198; SI-NEXT:    v_subrev_i32_e32 v3, vcc, 1, v410199; SI-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v210200; SI-NEXT:    s_or_b64 vcc, vcc, s[4:5]10201; SI-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc10202; SI-NEXT:    s_waitcnt expcnt(0)10203; SI-NEXT:    v_mov_b32_e32 v6, v410204; SI-NEXT:    v_mov_b32_e32 v5, v310205; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[8:11], 0 addr64 glc10206; SI-NEXT:    s_waitcnt vmcnt(0)10207; SI-NEXT:    buffer_wbinvl110208; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v410209; SI-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]10210; SI-NEXT:    v_mov_b32_e32 v4, v510211; SI-NEXT:    s_andn2_b64 exec, exec, s[6:7]10212; SI-NEXT:    s_cbranch_execnz .LBB144_110213; SI-NEXT:  ; %bb.2: ; %atomicrmw.end10214; SI-NEXT:    s_or_b64 exec, exec, s[6:7]10215; SI-NEXT:    s_waitcnt expcnt(0)10216; SI-NEXT:    s_setpc_b64 s[30:31]10217;10218; VI-LABEL: global_atomic_udec_wrap_i32_noret:10219; VI:       ; %bb.0:10220; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10221; VI-NEXT:    flat_load_dword v4, v[0:1]10222; VI-NEXT:    s_mov_b64 s[6:7], 010223; VI-NEXT:  .LBB144_1: ; %atomicrmw.start10224; VI-NEXT:    ; =>This Inner Loop Header: Depth=110225; VI-NEXT:    s_waitcnt vmcnt(0)10226; VI-NEXT:    v_subrev_u32_e32 v3, vcc, 1, v410227; VI-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v210228; VI-NEXT:    s_or_b64 vcc, vcc, s[4:5]10229; VI-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc10230; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc10231; VI-NEXT:    s_waitcnt vmcnt(0)10232; VI-NEXT:    buffer_wbinvl1_vol10233; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v410234; VI-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]10235; VI-NEXT:    v_mov_b32_e32 v4, v310236; VI-NEXT:    s_andn2_b64 exec, exec, s[6:7]10237; VI-NEXT:    s_cbranch_execnz .LBB144_110238; VI-NEXT:  ; %bb.2: ; %atomicrmw.end10239; VI-NEXT:    s_or_b64 exec, exec, s[6:7]10240; VI-NEXT:    s_setpc_b64 s[30:31]10241;10242; GFX9-LABEL: global_atomic_udec_wrap_i32_noret:10243; GFX9:       ; %bb.0:10244; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10245; GFX9-NEXT:    global_load_dword v4, v[0:1], off10246; GFX9-NEXT:    s_mov_b64 s[6:7], 010247; GFX9-NEXT:  .LBB144_1: ; %atomicrmw.start10248; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=110249; GFX9-NEXT:    s_waitcnt vmcnt(0)10250; GFX9-NEXT:    v_subrev_co_u32_e32 v3, vcc, 1, v410251; GFX9-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v210252; GFX9-NEXT:    s_or_b64 vcc, vcc, s[4:5]10253; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc10254; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc10255; GFX9-NEXT:    s_waitcnt vmcnt(0)10256; GFX9-NEXT:    buffer_wbinvl1_vol10257; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v410258; GFX9-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]10259; GFX9-NEXT:    v_mov_b32_e32 v4, v310260; GFX9-NEXT:    s_andn2_b64 exec, exec, s[6:7]10261; GFX9-NEXT:    s_cbranch_execnz .LBB144_110262; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end10263; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]10264; GFX9-NEXT:    s_setpc_b64 s[30:31]10265  %tmp0 = atomicrmw udec_wrap ptr addrspace(1) %ptr, i32 %in seq_cst10266  ret void10267}10268 10269define void @global_atomic_udec_wrap_i32_noret_offset(ptr addrspace(1) %out, i32 %in) {10270; SI-LABEL: global_atomic_udec_wrap_i32_noret_offset:10271; SI:       ; %bb.0:10272; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10273; SI-NEXT:    s_mov_b32 s10, 010274; SI-NEXT:    s_mov_b32 s11, 0xf00010275; SI-NEXT:    s_mov_b32 s8, s1010276; SI-NEXT:    s_mov_b32 s9, s1010277; SI-NEXT:    buffer_load_dword v4, v[0:1], s[8:11], 0 addr64 offset:1610278; SI-NEXT:    s_mov_b64 s[6:7], 010279; SI-NEXT:  .LBB145_1: ; %atomicrmw.start10280; SI-NEXT:    ; =>This Inner Loop Header: Depth=110281; SI-NEXT:    s_waitcnt vmcnt(0)10282; SI-NEXT:    v_subrev_i32_e32 v3, vcc, 1, v410283; SI-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v210284; SI-NEXT:    s_or_b64 vcc, vcc, s[4:5]10285; SI-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc10286; SI-NEXT:    s_waitcnt expcnt(0)10287; SI-NEXT:    v_mov_b32_e32 v6, v410288; SI-NEXT:    v_mov_b32_e32 v5, v310289; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[8:11], 0 addr64 offset:16 glc10290; SI-NEXT:    s_waitcnt vmcnt(0)10291; SI-NEXT:    buffer_wbinvl110292; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v410293; SI-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]10294; SI-NEXT:    v_mov_b32_e32 v4, v510295; SI-NEXT:    s_andn2_b64 exec, exec, s[6:7]10296; SI-NEXT:    s_cbranch_execnz .LBB145_110297; SI-NEXT:  ; %bb.2: ; %atomicrmw.end10298; SI-NEXT:    s_or_b64 exec, exec, s[6:7]10299; SI-NEXT:    s_waitcnt expcnt(0)10300; SI-NEXT:    s_setpc_b64 s[30:31]10301;10302; VI-LABEL: global_atomic_udec_wrap_i32_noret_offset:10303; VI:       ; %bb.0:10304; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10305; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v010306; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc10307; VI-NEXT:    flat_load_dword v4, v[0:1]10308; VI-NEXT:    s_mov_b64 s[6:7], 010309; VI-NEXT:  .LBB145_1: ; %atomicrmw.start10310; VI-NEXT:    ; =>This Inner Loop Header: Depth=110311; VI-NEXT:    s_waitcnt vmcnt(0)10312; VI-NEXT:    v_subrev_u32_e32 v3, vcc, 1, v410313; VI-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v210314; VI-NEXT:    s_or_b64 vcc, vcc, s[4:5]10315; VI-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc10316; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc10317; VI-NEXT:    s_waitcnt vmcnt(0)10318; VI-NEXT:    buffer_wbinvl1_vol10319; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v410320; VI-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]10321; VI-NEXT:    v_mov_b32_e32 v4, v310322; VI-NEXT:    s_andn2_b64 exec, exec, s[6:7]10323; VI-NEXT:    s_cbranch_execnz .LBB145_110324; VI-NEXT:  ; %bb.2: ; %atomicrmw.end10325; VI-NEXT:    s_or_b64 exec, exec, s[6:7]10326; VI-NEXT:    s_setpc_b64 s[30:31]10327;10328; GFX9-LABEL: global_atomic_udec_wrap_i32_noret_offset:10329; GFX9:       ; %bb.0:10330; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10331; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:1610332; GFX9-NEXT:    s_mov_b64 s[6:7], 010333; GFX9-NEXT:  .LBB145_1: ; %atomicrmw.start10334; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=110335; GFX9-NEXT:    s_waitcnt vmcnt(0)10336; GFX9-NEXT:    v_subrev_co_u32_e32 v3, vcc, 1, v410337; GFX9-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v210338; GFX9-NEXT:    s_or_b64 vcc, vcc, s[4:5]10339; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc10340; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc10341; GFX9-NEXT:    s_waitcnt vmcnt(0)10342; GFX9-NEXT:    buffer_wbinvl1_vol10343; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v410344; GFX9-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]10345; GFX9-NEXT:    v_mov_b32_e32 v4, v310346; GFX9-NEXT:    s_andn2_b64 exec, exec, s[6:7]10347; GFX9-NEXT:    s_cbranch_execnz .LBB145_110348; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end10349; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]10350; GFX9-NEXT:    s_setpc_b64 s[30:31]10351  %gep = getelementptr i32, ptr addrspace(1) %out, i32 410352  %tmp0 = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 %in seq_cst10353  ret void10354}10355 10356define i32 @global_atomic_udec_wrap_i32_ret(ptr addrspace(1) %ptr, i32 %in) {10357; SI-LABEL: global_atomic_udec_wrap_i32_ret:10358; SI:       ; %bb.0:10359; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10360; SI-NEXT:    s_mov_b32 s10, 010361; SI-NEXT:    s_mov_b32 s11, 0xf00010362; SI-NEXT:    s_mov_b32 s8, s1010363; SI-NEXT:    s_mov_b32 s9, s1010364; SI-NEXT:    buffer_load_dword v3, v[0:1], s[8:11], 0 addr6410365; SI-NEXT:    s_mov_b64 s[6:7], 010366; SI-NEXT:  .LBB146_1: ; %atomicrmw.start10367; SI-NEXT:    ; =>This Inner Loop Header: Depth=110368; SI-NEXT:    s_waitcnt vmcnt(0)10369; SI-NEXT:    v_mov_b32_e32 v5, v310370; SI-NEXT:    s_waitcnt expcnt(0)10371; SI-NEXT:    v_subrev_i32_e32 v3, vcc, 1, v510372; SI-NEXT:    v_cmp_gt_u32_e64 s[4:5], v5, v210373; SI-NEXT:    s_or_b64 vcc, vcc, s[4:5]10374; SI-NEXT:    v_cndmask_b32_e32 v4, v3, v2, vcc10375; SI-NEXT:    v_mov_b32_e32 v3, v410376; SI-NEXT:    v_mov_b32_e32 v4, v510377; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[8:11], 0 addr64 glc10378; SI-NEXT:    s_waitcnt vmcnt(0)10379; SI-NEXT:    buffer_wbinvl110380; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v510381; SI-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]10382; SI-NEXT:    s_andn2_b64 exec, exec, s[6:7]10383; SI-NEXT:    s_cbranch_execnz .LBB146_110384; SI-NEXT:  ; %bb.2: ; %atomicrmw.end10385; SI-NEXT:    s_or_b64 exec, exec, s[6:7]10386; SI-NEXT:    v_mov_b32_e32 v0, v310387; SI-NEXT:    s_waitcnt expcnt(0)10388; SI-NEXT:    s_setpc_b64 s[30:31]10389;10390; VI-LABEL: global_atomic_udec_wrap_i32_ret:10391; VI:       ; %bb.0:10392; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10393; VI-NEXT:    flat_load_dword v3, v[0:1]10394; VI-NEXT:    s_mov_b64 s[6:7], 010395; VI-NEXT:  .LBB146_1: ; %atomicrmw.start10396; VI-NEXT:    ; =>This Inner Loop Header: Depth=110397; VI-NEXT:    s_waitcnt vmcnt(0)10398; VI-NEXT:    v_mov_b32_e32 v4, v310399; VI-NEXT:    v_subrev_u32_e32 v3, vcc, 1, v410400; VI-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v210401; VI-NEXT:    s_or_b64 vcc, vcc, s[4:5]10402; VI-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc10403; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc10404; VI-NEXT:    s_waitcnt vmcnt(0)10405; VI-NEXT:    buffer_wbinvl1_vol10406; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v410407; VI-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]10408; VI-NEXT:    s_andn2_b64 exec, exec, s[6:7]10409; VI-NEXT:    s_cbranch_execnz .LBB146_110410; VI-NEXT:  ; %bb.2: ; %atomicrmw.end10411; VI-NEXT:    s_or_b64 exec, exec, s[6:7]10412; VI-NEXT:    v_mov_b32_e32 v0, v310413; VI-NEXT:    s_setpc_b64 s[30:31]10414;10415; GFX9-LABEL: global_atomic_udec_wrap_i32_ret:10416; GFX9:       ; %bb.0:10417; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10418; GFX9-NEXT:    global_load_dword v3, v[0:1], off10419; GFX9-NEXT:    s_mov_b64 s[6:7], 010420; GFX9-NEXT:  .LBB146_1: ; %atomicrmw.start10421; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=110422; GFX9-NEXT:    s_waitcnt vmcnt(0)10423; GFX9-NEXT:    v_mov_b32_e32 v4, v310424; GFX9-NEXT:    v_subrev_co_u32_e32 v3, vcc, 1, v410425; GFX9-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v210426; GFX9-NEXT:    s_or_b64 vcc, vcc, s[4:5]10427; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc10428; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc10429; GFX9-NEXT:    s_waitcnt vmcnt(0)10430; GFX9-NEXT:    buffer_wbinvl1_vol10431; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v410432; GFX9-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]10433; GFX9-NEXT:    s_andn2_b64 exec, exec, s[6:7]10434; GFX9-NEXT:    s_cbranch_execnz .LBB146_110435; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end10436; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]10437; GFX9-NEXT:    v_mov_b32_e32 v0, v310438; GFX9-NEXT:    s_setpc_b64 s[30:31]10439  %result = atomicrmw udec_wrap ptr addrspace(1) %ptr, i32 %in seq_cst10440  ret i32 %result10441}10442 10443define i32 @global_atomic_udec_wrap_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {10444; SI-LABEL: global_atomic_udec_wrap_i32_ret_offset:10445; SI:       ; %bb.0:10446; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10447; SI-NEXT:    s_mov_b32 s10, 010448; SI-NEXT:    s_mov_b32 s11, 0xf00010449; SI-NEXT:    s_mov_b32 s8, s1010450; SI-NEXT:    s_mov_b32 s9, s1010451; SI-NEXT:    buffer_load_dword v3, v[0:1], s[8:11], 0 addr64 offset:1610452; SI-NEXT:    s_mov_b64 s[6:7], 010453; SI-NEXT:  .LBB147_1: ; %atomicrmw.start10454; SI-NEXT:    ; =>This Inner Loop Header: Depth=110455; SI-NEXT:    s_waitcnt vmcnt(0)10456; SI-NEXT:    v_mov_b32_e32 v5, v310457; SI-NEXT:    s_waitcnt expcnt(0)10458; SI-NEXT:    v_subrev_i32_e32 v3, vcc, 1, v510459; SI-NEXT:    v_cmp_gt_u32_e64 s[4:5], v5, v210460; SI-NEXT:    s_or_b64 vcc, vcc, s[4:5]10461; SI-NEXT:    v_cndmask_b32_e32 v4, v3, v2, vcc10462; SI-NEXT:    v_mov_b32_e32 v3, v410463; SI-NEXT:    v_mov_b32_e32 v4, v510464; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[8:11], 0 addr64 offset:16 glc10465; SI-NEXT:    s_waitcnt vmcnt(0)10466; SI-NEXT:    buffer_wbinvl110467; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v510468; SI-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]10469; SI-NEXT:    s_andn2_b64 exec, exec, s[6:7]10470; SI-NEXT:    s_cbranch_execnz .LBB147_110471; SI-NEXT:  ; %bb.2: ; %atomicrmw.end10472; SI-NEXT:    s_or_b64 exec, exec, s[6:7]10473; SI-NEXT:    v_mov_b32_e32 v0, v310474; SI-NEXT:    s_waitcnt expcnt(0)10475; SI-NEXT:    s_setpc_b64 s[30:31]10476;10477; VI-LABEL: global_atomic_udec_wrap_i32_ret_offset:10478; VI:       ; %bb.0:10479; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10480; VI-NEXT:    v_add_u32_e32 v3, vcc, 16, v010481; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc10482; VI-NEXT:    flat_load_dword v0, v[3:4]10483; VI-NEXT:    s_mov_b64 s[6:7], 010484; VI-NEXT:  .LBB147_1: ; %atomicrmw.start10485; VI-NEXT:    ; =>This Inner Loop Header: Depth=110486; VI-NEXT:    s_waitcnt vmcnt(0)10487; VI-NEXT:    v_mov_b32_e32 v1, v010488; VI-NEXT:    v_subrev_u32_e32 v0, vcc, 1, v110489; VI-NEXT:    v_cmp_gt_u32_e64 s[4:5], v1, v210490; VI-NEXT:    s_or_b64 vcc, vcc, s[4:5]10491; VI-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc10492; VI-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc10493; VI-NEXT:    s_waitcnt vmcnt(0)10494; VI-NEXT:    buffer_wbinvl1_vol10495; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v110496; VI-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]10497; VI-NEXT:    s_andn2_b64 exec, exec, s[6:7]10498; VI-NEXT:    s_cbranch_execnz .LBB147_110499; VI-NEXT:  ; %bb.2: ; %atomicrmw.end10500; VI-NEXT:    s_or_b64 exec, exec, s[6:7]10501; VI-NEXT:    s_setpc_b64 s[30:31]10502;10503; GFX9-LABEL: global_atomic_udec_wrap_i32_ret_offset:10504; GFX9:       ; %bb.0:10505; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10506; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:1610507; GFX9-NEXT:    s_mov_b64 s[6:7], 010508; GFX9-NEXT:  .LBB147_1: ; %atomicrmw.start10509; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=110510; GFX9-NEXT:    s_waitcnt vmcnt(0)10511; GFX9-NEXT:    v_mov_b32_e32 v4, v310512; GFX9-NEXT:    v_subrev_co_u32_e32 v3, vcc, 1, v410513; GFX9-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v210514; GFX9-NEXT:    s_or_b64 vcc, vcc, s[4:5]10515; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc10516; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc10517; GFX9-NEXT:    s_waitcnt vmcnt(0)10518; GFX9-NEXT:    buffer_wbinvl1_vol10519; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v410520; GFX9-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]10521; GFX9-NEXT:    s_andn2_b64 exec, exec, s[6:7]10522; GFX9-NEXT:    s_cbranch_execnz .LBB147_110523; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end10524; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]10525; GFX9-NEXT:    v_mov_b32_e32 v0, v310526; GFX9-NEXT:    s_setpc_b64 s[30:31]10527  %gep = getelementptr i32, ptr addrspace(1) %out, i32 410528  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 %in seq_cst10529  ret i32 %result10530}10531 10532define amdgpu_gfx void @global_atomic_udec_wrap_i32_noret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {10533; SI-LABEL: global_atomic_udec_wrap_i32_noret_scalar:10534; SI:       ; %bb.0:10535; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10536; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -110537; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 ; 4-byte Folded Spill10538; SI-NEXT:    s_mov_b64 exec, s[34:35]10539; SI-NEXT:    s_waitcnt expcnt(0)10540; SI-NEXT:    v_writelane_b32 v5, s6, 010541; SI-NEXT:    v_writelane_b32 v5, s7, 110542; SI-NEXT:    s_mov_b32 s34, s610543; SI-NEXT:    s_mov_b32 s7, 0xf00010544; SI-NEXT:    s_mov_b32 s6, -110545; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 010546; SI-NEXT:    s_mov_b64 s[38:39], 010547; SI-NEXT:    v_mov_b32_e32 v2, s3410548; SI-NEXT:  .LBB148_1: ; %atomicrmw.start10549; SI-NEXT:    ; =>This Inner Loop Header: Depth=110550; SI-NEXT:    s_waitcnt vmcnt(0)10551; SI-NEXT:    v_subrev_i32_e32 v0, vcc, 1, v110552; SI-NEXT:    v_cmp_lt_u32_e64 s[36:37], s34, v110553; SI-NEXT:    s_or_b64 vcc, vcc, s[36:37]10554; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc10555; SI-NEXT:    s_waitcnt expcnt(0)10556; SI-NEXT:    v_mov_b32_e32 v4, v110557; SI-NEXT:    v_mov_b32_e32 v3, v010558; SI-NEXT:    buffer_atomic_cmpswap v[3:4], off, s[4:7], 0 glc10559; SI-NEXT:    s_waitcnt vmcnt(0)10560; SI-NEXT:    buffer_wbinvl110561; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v110562; SI-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]10563; SI-NEXT:    v_mov_b32_e32 v1, v310564; SI-NEXT:    s_andn2_b64 exec, exec, s[38:39]10565; SI-NEXT:    s_cbranch_execnz .LBB148_110566; SI-NEXT:  ; %bb.2: ; %atomicrmw.end10567; SI-NEXT:    s_or_b64 exec, exec, s[38:39]10568; SI-NEXT:    v_readlane_b32 s7, v5, 110569; SI-NEXT:    v_readlane_b32 s6, v5, 010570; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -110571; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 ; 4-byte Folded Reload10572; SI-NEXT:    s_mov_b64 exec, s[34:35]10573; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)10574; SI-NEXT:    s_setpc_b64 s[30:31]10575;10576; VI-LABEL: global_atomic_udec_wrap_i32_noret_scalar:10577; VI:       ; %bb.0:10578; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10579; VI-NEXT:    v_mov_b32_e32 v0, s410580; VI-NEXT:    v_mov_b32_e32 v1, s510581; VI-NEXT:    flat_load_dword v1, v[0:1]10582; VI-NEXT:    v_mov_b32_e32 v2, s410583; VI-NEXT:    s_mov_b64 s[36:37], 010584; VI-NEXT:    v_mov_b32_e32 v4, s610585; VI-NEXT:    v_mov_b32_e32 v3, s510586; VI-NEXT:  .LBB148_1: ; %atomicrmw.start10587; VI-NEXT:    ; =>This Inner Loop Header: Depth=110588; VI-NEXT:    s_waitcnt vmcnt(0)10589; VI-NEXT:    v_subrev_u32_e32 v0, vcc, 1, v110590; VI-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v110591; VI-NEXT:    s_or_b64 vcc, vcc, s[34:35]10592; VI-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc10593; VI-NEXT:    flat_atomic_cmpswap v0, v[2:3], v[0:1] glc10594; VI-NEXT:    s_waitcnt vmcnt(0)10595; VI-NEXT:    buffer_wbinvl1_vol10596; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v110597; VI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]10598; VI-NEXT:    v_mov_b32_e32 v1, v010599; VI-NEXT:    s_andn2_b64 exec, exec, s[36:37]10600; VI-NEXT:    s_cbranch_execnz .LBB148_110601; VI-NEXT:  ; %bb.2: ; %atomicrmw.end10602; VI-NEXT:    s_or_b64 exec, exec, s[36:37]10603; VI-NEXT:    s_setpc_b64 s[30:31]10604;10605; GFX9-LABEL: global_atomic_udec_wrap_i32_noret_scalar:10606; GFX9:       ; %bb.0:10607; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10608; GFX9-NEXT:    v_mov_b32_e32 v2, 010609; GFX9-NEXT:    global_load_dword v1, v2, s[4:5]10610; GFX9-NEXT:    s_mov_b64 s[36:37], 010611; GFX9-NEXT:    v_mov_b32_e32 v3, s610612; GFX9-NEXT:  .LBB148_1: ; %atomicrmw.start10613; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=110614; GFX9-NEXT:    s_waitcnt vmcnt(0)10615; GFX9-NEXT:    v_subrev_co_u32_e32 v0, vcc, 1, v110616; GFX9-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v110617; GFX9-NEXT:    s_or_b64 vcc, vcc, s[34:35]10618; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc10619; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] glc10620; GFX9-NEXT:    s_waitcnt vmcnt(0)10621; GFX9-NEXT:    buffer_wbinvl1_vol10622; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v110623; GFX9-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]10624; GFX9-NEXT:    v_mov_b32_e32 v1, v010625; GFX9-NEXT:    s_andn2_b64 exec, exec, s[36:37]10626; GFX9-NEXT:    s_cbranch_execnz .LBB148_110627; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end10628; GFX9-NEXT:    s_or_b64 exec, exec, s[36:37]10629; GFX9-NEXT:    s_setpc_b64 s[30:31]10630  %tmp0 = atomicrmw udec_wrap ptr addrspace(1) %ptr, i32 %in seq_cst10631  ret void10632}10633 10634define amdgpu_gfx void @global_atomic_udec_wrap_i32_noret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {10635; SI-LABEL: global_atomic_udec_wrap_i32_noret_offset_scalar:10636; SI:       ; %bb.0:10637; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10638; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -110639; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 ; 4-byte Folded Spill10640; SI-NEXT:    s_mov_b64 exec, s[34:35]10641; SI-NEXT:    s_waitcnt expcnt(0)10642; SI-NEXT:    v_writelane_b32 v5, s6, 010643; SI-NEXT:    v_writelane_b32 v5, s7, 110644; SI-NEXT:    s_mov_b32 s34, s610645; SI-NEXT:    s_mov_b32 s7, 0xf00010646; SI-NEXT:    s_mov_b32 s6, -110647; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:1610648; SI-NEXT:    s_mov_b64 s[38:39], 010649; SI-NEXT:    v_mov_b32_e32 v2, s3410650; SI-NEXT:  .LBB149_1: ; %atomicrmw.start10651; SI-NEXT:    ; =>This Inner Loop Header: Depth=110652; SI-NEXT:    s_waitcnt vmcnt(0)10653; SI-NEXT:    v_subrev_i32_e32 v0, vcc, 1, v110654; SI-NEXT:    v_cmp_lt_u32_e64 s[36:37], s34, v110655; SI-NEXT:    s_or_b64 vcc, vcc, s[36:37]10656; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc10657; SI-NEXT:    s_waitcnt expcnt(0)10658; SI-NEXT:    v_mov_b32_e32 v4, v110659; SI-NEXT:    v_mov_b32_e32 v3, v010660; SI-NEXT:    buffer_atomic_cmpswap v[3:4], off, s[4:7], 0 offset:16 glc10661; SI-NEXT:    s_waitcnt vmcnt(0)10662; SI-NEXT:    buffer_wbinvl110663; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v110664; SI-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]10665; SI-NEXT:    v_mov_b32_e32 v1, v310666; SI-NEXT:    s_andn2_b64 exec, exec, s[38:39]10667; SI-NEXT:    s_cbranch_execnz .LBB149_110668; SI-NEXT:  ; %bb.2: ; %atomicrmw.end10669; SI-NEXT:    s_or_b64 exec, exec, s[38:39]10670; SI-NEXT:    v_readlane_b32 s7, v5, 110671; SI-NEXT:    v_readlane_b32 s6, v5, 010672; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -110673; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 ; 4-byte Folded Reload10674; SI-NEXT:    s_mov_b64 exec, s[34:35]10675; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)10676; SI-NEXT:    s_setpc_b64 s[30:31]10677;10678; VI-LABEL: global_atomic_udec_wrap_i32_noret_offset_scalar:10679; VI:       ; %bb.0:10680; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10681; VI-NEXT:    s_add_u32 s34, s4, 1610682; VI-NEXT:    s_addc_u32 s35, s5, 010683; VI-NEXT:    v_mov_b32_e32 v0, s3410684; VI-NEXT:    v_mov_b32_e32 v1, s3510685; VI-NEXT:    flat_load_dword v3, v[0:1]10686; VI-NEXT:    s_mov_b64 s[36:37], 010687; VI-NEXT:    v_mov_b32_e32 v4, s610688; VI-NEXT:  .LBB149_1: ; %atomicrmw.start10689; VI-NEXT:    ; =>This Inner Loop Header: Depth=110690; VI-NEXT:    s_waitcnt vmcnt(0)10691; VI-NEXT:    v_subrev_u32_e32 v2, vcc, 1, v310692; VI-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v310693; VI-NEXT:    s_or_b64 vcc, vcc, s[34:35]10694; VI-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc10695; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc10696; VI-NEXT:    s_waitcnt vmcnt(0)10697; VI-NEXT:    buffer_wbinvl1_vol10698; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v310699; VI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]10700; VI-NEXT:    v_mov_b32_e32 v3, v210701; VI-NEXT:    s_andn2_b64 exec, exec, s[36:37]10702; VI-NEXT:    s_cbranch_execnz .LBB149_110703; VI-NEXT:  ; %bb.2: ; %atomicrmw.end10704; VI-NEXT:    s_or_b64 exec, exec, s[36:37]10705; VI-NEXT:    s_setpc_b64 s[30:31]10706;10707; GFX9-LABEL: global_atomic_udec_wrap_i32_noret_offset_scalar:10708; GFX9:       ; %bb.0:10709; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10710; GFX9-NEXT:    v_mov_b32_e32 v2, 010711; GFX9-NEXT:    global_load_dword v1, v2, s[4:5] offset:1610712; GFX9-NEXT:    s_mov_b64 s[36:37], 010713; GFX9-NEXT:    v_mov_b32_e32 v3, s610714; GFX9-NEXT:  .LBB149_1: ; %atomicrmw.start10715; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=110716; GFX9-NEXT:    s_waitcnt vmcnt(0)10717; GFX9-NEXT:    v_subrev_co_u32_e32 v0, vcc, 1, v110718; GFX9-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v110719; GFX9-NEXT:    s_or_b64 vcc, vcc, s[34:35]10720; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc10721; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 glc10722; GFX9-NEXT:    s_waitcnt vmcnt(0)10723; GFX9-NEXT:    buffer_wbinvl1_vol10724; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v110725; GFX9-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]10726; GFX9-NEXT:    v_mov_b32_e32 v1, v010727; GFX9-NEXT:    s_andn2_b64 exec, exec, s[36:37]10728; GFX9-NEXT:    s_cbranch_execnz .LBB149_110729; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end10730; GFX9-NEXT:    s_or_b64 exec, exec, s[36:37]10731; GFX9-NEXT:    s_setpc_b64 s[30:31]10732  %gep = getelementptr i32, ptr addrspace(1) %out, i32 410733  %tmp0 = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 %in seq_cst10734  ret void10735}10736 10737define amdgpu_gfx i32 @global_atomic_udec_wrap_i32_ret_scalar(ptr addrspace(1) inreg %ptr, i32 inreg %in) {10738; SI-LABEL: global_atomic_udec_wrap_i32_ret_scalar:10739; SI:       ; %bb.0:10740; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10741; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -110742; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 ; 4-byte Folded Spill10743; SI-NEXT:    s_mov_b64 exec, s[34:35]10744; SI-NEXT:    s_waitcnt expcnt(0)10745; SI-NEXT:    v_writelane_b32 v5, s6, 010746; SI-NEXT:    v_writelane_b32 v5, s7, 110747; SI-NEXT:    s_mov_b32 s34, s610748; SI-NEXT:    s_mov_b32 s7, 0xf00010749; SI-NEXT:    s_mov_b32 s6, -110750; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 010751; SI-NEXT:    s_mov_b64 s[38:39], 010752; SI-NEXT:    v_mov_b32_e32 v2, s3410753; SI-NEXT:  .LBB150_1: ; %atomicrmw.start10754; SI-NEXT:    ; =>This Inner Loop Header: Depth=110755; SI-NEXT:    s_waitcnt vmcnt(0)10756; SI-NEXT:    v_mov_b32_e32 v4, v010757; SI-NEXT:    s_waitcnt expcnt(0)10758; SI-NEXT:    v_subrev_i32_e32 v0, vcc, 1, v410759; SI-NEXT:    v_cmp_lt_u32_e64 s[36:37], s34, v410760; SI-NEXT:    s_or_b64 vcc, vcc, s[36:37]10761; SI-NEXT:    v_cndmask_b32_e32 v3, v0, v2, vcc10762; SI-NEXT:    v_mov_b32_e32 v0, v310763; SI-NEXT:    v_mov_b32_e32 v1, v410764; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc10765; SI-NEXT:    s_waitcnt vmcnt(0)10766; SI-NEXT:    buffer_wbinvl110767; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v410768; SI-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]10769; SI-NEXT:    s_andn2_b64 exec, exec, s[38:39]10770; SI-NEXT:    s_cbranch_execnz .LBB150_110771; SI-NEXT:  ; %bb.2: ; %atomicrmw.end10772; SI-NEXT:    s_or_b64 exec, exec, s[38:39]10773; SI-NEXT:    v_readlane_b32 s7, v5, 110774; SI-NEXT:    v_readlane_b32 s6, v5, 010775; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -110776; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 ; 4-byte Folded Reload10777; SI-NEXT:    s_mov_b64 exec, s[34:35]10778; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)10779; SI-NEXT:    s_setpc_b64 s[30:31]10780;10781; VI-LABEL: global_atomic_udec_wrap_i32_ret_scalar:10782; VI:       ; %bb.0:10783; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10784; VI-NEXT:    v_mov_b32_e32 v0, s410785; VI-NEXT:    v_mov_b32_e32 v1, s510786; VI-NEXT:    flat_load_dword v0, v[0:1]10787; VI-NEXT:    v_mov_b32_e32 v1, s410788; VI-NEXT:    s_mov_b64 s[36:37], 010789; VI-NEXT:    v_mov_b32_e32 v3, s610790; VI-NEXT:    v_mov_b32_e32 v2, s510791; VI-NEXT:  .LBB150_1: ; %atomicrmw.start10792; VI-NEXT:    ; =>This Inner Loop Header: Depth=110793; VI-NEXT:    s_waitcnt vmcnt(0)10794; VI-NEXT:    v_mov_b32_e32 v5, v010795; VI-NEXT:    v_subrev_u32_e32 v0, vcc, 1, v510796; VI-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v510797; VI-NEXT:    s_or_b64 vcc, vcc, s[34:35]10798; VI-NEXT:    v_cndmask_b32_e32 v4, v0, v3, vcc10799; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[4:5] glc10800; VI-NEXT:    s_waitcnt vmcnt(0)10801; VI-NEXT:    buffer_wbinvl1_vol10802; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v510803; VI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]10804; VI-NEXT:    s_andn2_b64 exec, exec, s[36:37]10805; VI-NEXT:    s_cbranch_execnz .LBB150_110806; VI-NEXT:  ; %bb.2: ; %atomicrmw.end10807; VI-NEXT:    s_or_b64 exec, exec, s[36:37]10808; VI-NEXT:    s_setpc_b64 s[30:31]10809;10810; GFX9-LABEL: global_atomic_udec_wrap_i32_ret_scalar:10811; GFX9:       ; %bb.0:10812; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10813; GFX9-NEXT:    v_mov_b32_e32 v1, 010814; GFX9-NEXT:    global_load_dword v0, v1, s[4:5]10815; GFX9-NEXT:    s_mov_b64 s[36:37], 010816; GFX9-NEXT:    v_mov_b32_e32 v2, s610817; GFX9-NEXT:  .LBB150_1: ; %atomicrmw.start10818; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=110819; GFX9-NEXT:    s_waitcnt vmcnt(0)10820; GFX9-NEXT:    v_mov_b32_e32 v4, v010821; GFX9-NEXT:    v_subrev_co_u32_e32 v0, vcc, 1, v410822; GFX9-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v410823; GFX9-NEXT:    s_or_b64 vcc, vcc, s[34:35]10824; GFX9-NEXT:    v_cndmask_b32_e32 v3, v0, v2, vcc10825; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[3:4], s[4:5] glc10826; GFX9-NEXT:    s_waitcnt vmcnt(0)10827; GFX9-NEXT:    buffer_wbinvl1_vol10828; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v410829; GFX9-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]10830; GFX9-NEXT:    s_andn2_b64 exec, exec, s[36:37]10831; GFX9-NEXT:    s_cbranch_execnz .LBB150_110832; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end10833; GFX9-NEXT:    s_or_b64 exec, exec, s[36:37]10834; GFX9-NEXT:    s_setpc_b64 s[30:31]10835  %result = atomicrmw udec_wrap ptr addrspace(1) %ptr, i32 %in seq_cst10836  ret i32 %result10837}10838 10839define amdgpu_gfx i32 @global_atomic_udec_wrap_i32_ret_offset_scalar(ptr addrspace(1) inreg %out, i32 inreg %in) {10840; SI-LABEL: global_atomic_udec_wrap_i32_ret_offset_scalar:10841; SI:       ; %bb.0:10842; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10843; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -110844; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 ; 4-byte Folded Spill10845; SI-NEXT:    s_mov_b64 exec, s[34:35]10846; SI-NEXT:    s_waitcnt expcnt(0)10847; SI-NEXT:    v_writelane_b32 v5, s6, 010848; SI-NEXT:    v_writelane_b32 v5, s7, 110849; SI-NEXT:    s_mov_b32 s34, s610850; SI-NEXT:    s_mov_b32 s7, 0xf00010851; SI-NEXT:    s_mov_b32 s6, -110852; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:1610853; SI-NEXT:    s_mov_b64 s[38:39], 010854; SI-NEXT:    v_mov_b32_e32 v2, s3410855; SI-NEXT:  .LBB151_1: ; %atomicrmw.start10856; SI-NEXT:    ; =>This Inner Loop Header: Depth=110857; SI-NEXT:    s_waitcnt vmcnt(0)10858; SI-NEXT:    v_mov_b32_e32 v4, v010859; SI-NEXT:    s_waitcnt expcnt(0)10860; SI-NEXT:    v_subrev_i32_e32 v0, vcc, 1, v410861; SI-NEXT:    v_cmp_lt_u32_e64 s[36:37], s34, v410862; SI-NEXT:    s_or_b64 vcc, vcc, s[36:37]10863; SI-NEXT:    v_cndmask_b32_e32 v3, v0, v2, vcc10864; SI-NEXT:    v_mov_b32_e32 v0, v310865; SI-NEXT:    v_mov_b32_e32 v1, v410866; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc10867; SI-NEXT:    s_waitcnt vmcnt(0)10868; SI-NEXT:    buffer_wbinvl110869; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v410870; SI-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]10871; SI-NEXT:    s_andn2_b64 exec, exec, s[38:39]10872; SI-NEXT:    s_cbranch_execnz .LBB151_110873; SI-NEXT:  ; %bb.2: ; %atomicrmw.end10874; SI-NEXT:    s_or_b64 exec, exec, s[38:39]10875; SI-NEXT:    v_readlane_b32 s7, v5, 110876; SI-NEXT:    v_readlane_b32 s6, v5, 010877; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -110878; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 ; 4-byte Folded Reload10879; SI-NEXT:    s_mov_b64 exec, s[34:35]10880; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)10881; SI-NEXT:    s_setpc_b64 s[30:31]10882;10883; VI-LABEL: global_atomic_udec_wrap_i32_ret_offset_scalar:10884; VI:       ; %bb.0:10885; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10886; VI-NEXT:    s_add_u32 s34, s4, 1610887; VI-NEXT:    s_addc_u32 s35, s5, 010888; VI-NEXT:    v_mov_b32_e32 v1, s3410889; VI-NEXT:    v_mov_b32_e32 v2, s3510890; VI-NEXT:    flat_load_dword v0, v[1:2]10891; VI-NEXT:    s_mov_b64 s[36:37], 010892; VI-NEXT:    v_mov_b32_e32 v3, s610893; VI-NEXT:  .LBB151_1: ; %atomicrmw.start10894; VI-NEXT:    ; =>This Inner Loop Header: Depth=110895; VI-NEXT:    s_waitcnt vmcnt(0)10896; VI-NEXT:    v_mov_b32_e32 v5, v010897; VI-NEXT:    v_subrev_u32_e32 v0, vcc, 1, v510898; VI-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v510899; VI-NEXT:    s_or_b64 vcc, vcc, s[34:35]10900; VI-NEXT:    v_cndmask_b32_e32 v4, v0, v3, vcc10901; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[4:5] glc10902; VI-NEXT:    s_waitcnt vmcnt(0)10903; VI-NEXT:    buffer_wbinvl1_vol10904; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v510905; VI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]10906; VI-NEXT:    s_andn2_b64 exec, exec, s[36:37]10907; VI-NEXT:    s_cbranch_execnz .LBB151_110908; VI-NEXT:  ; %bb.2: ; %atomicrmw.end10909; VI-NEXT:    s_or_b64 exec, exec, s[36:37]10910; VI-NEXT:    s_setpc_b64 s[30:31]10911;10912; GFX9-LABEL: global_atomic_udec_wrap_i32_ret_offset_scalar:10913; GFX9:       ; %bb.0:10914; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10915; GFX9-NEXT:    v_mov_b32_e32 v1, 010916; GFX9-NEXT:    global_load_dword v0, v1, s[4:5] offset:1610917; GFX9-NEXT:    s_mov_b64 s[36:37], 010918; GFX9-NEXT:    v_mov_b32_e32 v2, s610919; GFX9-NEXT:  .LBB151_1: ; %atomicrmw.start10920; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=110921; GFX9-NEXT:    s_waitcnt vmcnt(0)10922; GFX9-NEXT:    v_mov_b32_e32 v4, v010923; GFX9-NEXT:    v_subrev_co_u32_e32 v0, vcc, 1, v410924; GFX9-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v410925; GFX9-NEXT:    s_or_b64 vcc, vcc, s[34:35]10926; GFX9-NEXT:    v_cndmask_b32_e32 v3, v0, v2, vcc10927; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[3:4], s[4:5] offset:16 glc10928; GFX9-NEXT:    s_waitcnt vmcnt(0)10929; GFX9-NEXT:    buffer_wbinvl1_vol10930; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v410931; GFX9-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]10932; GFX9-NEXT:    s_andn2_b64 exec, exec, s[36:37]10933; GFX9-NEXT:    s_cbranch_execnz .LBB151_110934; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end10935; GFX9-NEXT:    s_or_b64 exec, exec, s[36:37]10936; GFX9-NEXT:    s_setpc_b64 s[30:31]10937  %gep = getelementptr i32, ptr addrspace(1) %out, i32 410938  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 %in seq_cst10939  ret i32 %result10940}10941 10942define void @global_atomic_udec_wrap_i32_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {10943; SI-LABEL: global_atomic_udec_wrap_i32_noret_offset__amdgpu_no_remote_memory:10944; SI:       ; %bb.0:10945; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10946; SI-NEXT:    s_mov_b32 s6, 010947; SI-NEXT:    s_mov_b32 s7, 0xf00010948; SI-NEXT:    s_mov_b32 s4, s610949; SI-NEXT:    s_mov_b32 s5, s610950; SI-NEXT:    buffer_atomic_dec v2, v[0:1], s[4:7], 0 addr64 offset:1610951; SI-NEXT:    s_waitcnt vmcnt(0)10952; SI-NEXT:    buffer_wbinvl110953; SI-NEXT:    s_waitcnt expcnt(0)10954; SI-NEXT:    s_setpc_b64 s[30:31]10955;10956; VI-LABEL: global_atomic_udec_wrap_i32_noret_offset__amdgpu_no_remote_memory:10957; VI:       ; %bb.0:10958; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10959; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v010960; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc10961; VI-NEXT:    flat_atomic_dec v[0:1], v210962; VI-NEXT:    s_waitcnt vmcnt(0)10963; VI-NEXT:    buffer_wbinvl1_vol10964; VI-NEXT:    s_setpc_b64 s[30:31]10965;10966; GFX9-LABEL: global_atomic_udec_wrap_i32_noret_offset__amdgpu_no_remote_memory:10967; GFX9:       ; %bb.0:10968; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10969; GFX9-NEXT:    global_atomic_dec v[0:1], v2, off offset:1610970; GFX9-NEXT:    s_waitcnt vmcnt(0)10971; GFX9-NEXT:    buffer_wbinvl1_vol10972; GFX9-NEXT:    s_setpc_b64 s[30:31]10973  %gep = getelementptr i32, ptr addrspace(1) %out, i64 410974  %tmp0 = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !010975  ret void10976}10977 10978define i32 @global_atomic_udec_wrap_i32_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i32 %in) {10979; SI-LABEL: global_atomic_udec_wrap_i32_ret_offset__amdgpu_no_remote_memory:10980; SI:       ; %bb.0:10981; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10982; SI-NEXT:    s_mov_b32 s6, 010983; SI-NEXT:    s_mov_b32 s7, 0xf00010984; SI-NEXT:    s_mov_b32 s4, s610985; SI-NEXT:    s_mov_b32 s5, s610986; SI-NEXT:    buffer_atomic_dec v2, v[0:1], s[4:7], 0 addr64 offset:16 glc10987; SI-NEXT:    s_waitcnt vmcnt(0)10988; SI-NEXT:    buffer_wbinvl110989; SI-NEXT:    v_mov_b32_e32 v0, v210990; SI-NEXT:    s_waitcnt expcnt(0)10991; SI-NEXT:    s_setpc_b64 s[30:31]10992;10993; VI-LABEL: global_atomic_udec_wrap_i32_ret_offset__amdgpu_no_remote_memory:10994; VI:       ; %bb.0:10995; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10996; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v010997; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc10998; VI-NEXT:    flat_atomic_dec v0, v[0:1], v2 glc10999; VI-NEXT:    s_waitcnt vmcnt(0)11000; VI-NEXT:    buffer_wbinvl1_vol11001; VI-NEXT:    s_setpc_b64 s[30:31]11002;11003; GFX9-LABEL: global_atomic_udec_wrap_i32_ret_offset__amdgpu_no_remote_memory:11004; GFX9:       ; %bb.0:11005; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11006; GFX9-NEXT:    global_atomic_dec v0, v[0:1], v2, off offset:16 glc11007; GFX9-NEXT:    s_waitcnt vmcnt(0)11008; GFX9-NEXT:    buffer_wbinvl1_vol11009; GFX9-NEXT:    s_setpc_b64 s[30:31]11010  %gep = getelementptr i32, ptr addrspace(1) %out, i64 411011  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !011012  ret i32 %result11013}11014 11015!0 = !{}11016