12297 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -enable-var-scope -check-prefixes=SI %s3; RUN: llc -mtriple=amdgcn -mcpu=tonga -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -enable-var-scope -check-prefixes=VI %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s5 6; ---------------------------------------------------------------------7; atomicrmw xchg8; ---------------------------------------------------------------------9 10define void @global_atomic_xchg_i64_noret(ptr addrspace(1) %ptr, i64 %in) {11; SI-LABEL: global_atomic_xchg_i64_noret:12; SI: ; %bb.0:13; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14; SI-NEXT: s_mov_b32 s6, 015; SI-NEXT: s_mov_b32 s7, 0xf00016; SI-NEXT: s_mov_b32 s4, s617; SI-NEXT: s_mov_b32 s5, s618; SI-NEXT: buffer_atomic_swap_x2 v[2:3], v[0:1], s[4:7], 0 addr6419; SI-NEXT: s_waitcnt vmcnt(0)20; SI-NEXT: buffer_wbinvl121; SI-NEXT: s_waitcnt expcnt(0)22; SI-NEXT: s_setpc_b64 s[30:31]23;24; VI-LABEL: global_atomic_xchg_i64_noret:25; VI: ; %bb.0:26; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)27; VI-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]28; VI-NEXT: s_waitcnt vmcnt(0)29; VI-NEXT: buffer_wbinvl1_vol30; VI-NEXT: s_setpc_b64 s[30:31]31;32; GFX9-LABEL: global_atomic_xchg_i64_noret:33; GFX9: ; %bb.0:34; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)35; GFX9-NEXT: global_atomic_swap_x2 v[0:1], v[2:3], off36; GFX9-NEXT: s_waitcnt vmcnt(0)37; GFX9-NEXT: buffer_wbinvl1_vol38; GFX9-NEXT: s_setpc_b64 s[30:31]39 %tmp0 = atomicrmw xchg ptr addrspace(1) %ptr, i64 %in seq_cst40 ret void41}42 43define void @global_atomic_xchg_i64_noret_offset(ptr addrspace(1) %out, i64 %in) {44; SI-LABEL: global_atomic_xchg_i64_noret_offset:45; SI: ; %bb.0:46; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)47; SI-NEXT: s_mov_b32 s6, 048; SI-NEXT: s_mov_b32 s7, 0xf00049; SI-NEXT: s_mov_b32 s4, s650; SI-NEXT: s_mov_b32 s5, s651; SI-NEXT: buffer_atomic_swap_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:3252; SI-NEXT: s_waitcnt vmcnt(0)53; SI-NEXT: buffer_wbinvl154; SI-NEXT: s_waitcnt expcnt(0)55; SI-NEXT: s_setpc_b64 s[30:31]56;57; VI-LABEL: global_atomic_xchg_i64_noret_offset:58; VI: ; %bb.0:59; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)60; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v061; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc62; VI-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]63; VI-NEXT: s_waitcnt vmcnt(0)64; VI-NEXT: buffer_wbinvl1_vol65; VI-NEXT: s_setpc_b64 s[30:31]66;67; GFX9-LABEL: global_atomic_xchg_i64_noret_offset:68; GFX9: ; %bb.0:69; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)70; GFX9-NEXT: global_atomic_swap_x2 v[0:1], v[2:3], off offset:3271; GFX9-NEXT: s_waitcnt vmcnt(0)72; GFX9-NEXT: buffer_wbinvl1_vol73; GFX9-NEXT: s_setpc_b64 s[30:31]74 %gep = getelementptr i64, ptr addrspace(1) %out, i64 475 %tmp0 = atomicrmw xchg ptr addrspace(1) %gep, i64 %in seq_cst76 ret void77}78 79define i64 @global_atomic_xchg_i64_ret(ptr addrspace(1) %ptr, i64 %in) {80; SI-LABEL: global_atomic_xchg_i64_ret:81; SI: ; %bb.0:82; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)83; SI-NEXT: s_mov_b32 s6, 084; SI-NEXT: s_mov_b32 s7, 0xf00085; SI-NEXT: s_mov_b32 s4, s686; SI-NEXT: s_mov_b32 s5, s687; SI-NEXT: buffer_atomic_swap_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc88; SI-NEXT: s_waitcnt vmcnt(0)89; SI-NEXT: buffer_wbinvl190; SI-NEXT: v_mov_b32_e32 v0, v291; SI-NEXT: v_mov_b32_e32 v1, v392; SI-NEXT: s_waitcnt expcnt(0)93; SI-NEXT: s_setpc_b64 s[30:31]94;95; VI-LABEL: global_atomic_xchg_i64_ret:96; VI: ; %bb.0:97; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)98; VI-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc99; VI-NEXT: s_waitcnt vmcnt(0)100; VI-NEXT: buffer_wbinvl1_vol101; VI-NEXT: s_setpc_b64 s[30:31]102;103; GFX9-LABEL: global_atomic_xchg_i64_ret:104; GFX9: ; %bb.0:105; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)106; GFX9-NEXT: global_atomic_swap_x2 v[0:1], v[0:1], v[2:3], off glc107; GFX9-NEXT: s_waitcnt vmcnt(0)108; GFX9-NEXT: buffer_wbinvl1_vol109; GFX9-NEXT: s_setpc_b64 s[30:31]110 %result = atomicrmw xchg ptr addrspace(1) %ptr, i64 %in seq_cst111 ret i64 %result112}113 114define i64 @global_atomic_xchg_i64_ret_offset(ptr addrspace(1) %out, i64 %in) {115; SI-LABEL: global_atomic_xchg_i64_ret_offset:116; SI: ; %bb.0:117; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)118; SI-NEXT: s_mov_b32 s6, 0119; SI-NEXT: s_mov_b32 s7, 0xf000120; SI-NEXT: s_mov_b32 s4, s6121; SI-NEXT: s_mov_b32 s5, s6122; SI-NEXT: buffer_atomic_swap_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc123; SI-NEXT: s_waitcnt vmcnt(0)124; SI-NEXT: buffer_wbinvl1125; SI-NEXT: v_mov_b32_e32 v0, v2126; SI-NEXT: v_mov_b32_e32 v1, v3127; SI-NEXT: s_waitcnt expcnt(0)128; SI-NEXT: s_setpc_b64 s[30:31]129;130; VI-LABEL: global_atomic_xchg_i64_ret_offset:131; VI: ; %bb.0:132; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)133; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v0134; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc135; VI-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc136; VI-NEXT: s_waitcnt vmcnt(0)137; VI-NEXT: buffer_wbinvl1_vol138; VI-NEXT: s_setpc_b64 s[30:31]139;140; GFX9-LABEL: global_atomic_xchg_i64_ret_offset:141; GFX9: ; %bb.0:142; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)143; GFX9-NEXT: global_atomic_swap_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc144; GFX9-NEXT: s_waitcnt vmcnt(0)145; GFX9-NEXT: buffer_wbinvl1_vol146; GFX9-NEXT: s_setpc_b64 s[30:31]147 %gep = getelementptr i64, ptr addrspace(1) %out, i64 4148 %result = atomicrmw xchg ptr addrspace(1) %gep, i64 %in seq_cst149 ret i64 %result150}151 152define amdgpu_gfx void @global_atomic_xchg_i64_noret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {153; SI-LABEL: global_atomic_xchg_i64_noret_scalar:154; SI: ; %bb.0:155; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)156; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1157; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill158; SI-NEXT: s_mov_b64 exec, s[34:35]159; SI-NEXT: s_waitcnt expcnt(0)160; SI-NEXT: v_writelane_b32 v2, s6, 0161; SI-NEXT: v_writelane_b32 v2, s7, 1162; SI-NEXT: s_mov_b32 s34, s7163; SI-NEXT: s_mov_b32 s35, s6164; SI-NEXT: s_mov_b32 s7, 0xf000165; SI-NEXT: s_mov_b32 s6, -1166; SI-NEXT: v_mov_b32_e32 v0, s35167; SI-NEXT: v_mov_b32_e32 v1, s34168; SI-NEXT: s_waitcnt vmcnt(0)169; SI-NEXT: buffer_atomic_swap_x2 v[0:1], off, s[4:7], 0170; SI-NEXT: s_waitcnt vmcnt(0)171; SI-NEXT: buffer_wbinvl1172; SI-NEXT: v_readlane_b32 s7, v2, 1173; SI-NEXT: v_readlane_b32 s6, v2, 0174; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1175; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload176; SI-NEXT: s_mov_b64 exec, s[34:35]177; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)178; SI-NEXT: s_setpc_b64 s[30:31]179;180; VI-LABEL: global_atomic_xchg_i64_noret_scalar:181; VI: ; %bb.0:182; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)183; VI-NEXT: v_mov_b32_e32 v0, s6184; VI-NEXT: v_mov_b32_e32 v1, s7185; VI-NEXT: v_mov_b32_e32 v2, s4186; VI-NEXT: v_mov_b32_e32 v3, s5187; VI-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]188; VI-NEXT: s_waitcnt vmcnt(0)189; VI-NEXT: buffer_wbinvl1_vol190; VI-NEXT: s_setpc_b64 s[30:31]191;192; GFX9-LABEL: global_atomic_xchg_i64_noret_scalar:193; GFX9: ; %bb.0:194; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)195; GFX9-NEXT: v_mov_b32_e32 v0, s6196; GFX9-NEXT: v_mov_b32_e32 v1, s7197; GFX9-NEXT: v_mov_b32_e32 v2, 0198; GFX9-NEXT: global_atomic_swap_x2 v2, v[0:1], s[4:5]199; GFX9-NEXT: s_waitcnt vmcnt(0)200; GFX9-NEXT: buffer_wbinvl1_vol201; GFX9-NEXT: s_setpc_b64 s[30:31]202 %tmp0 = atomicrmw xchg ptr addrspace(1) %ptr, i64 %in seq_cst203 ret void204}205 206define amdgpu_gfx void @global_atomic_xchg_i64_noret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {207; SI-LABEL: global_atomic_xchg_i64_noret_offset_scalar:208; SI: ; %bb.0:209; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)210; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1211; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill212; SI-NEXT: s_mov_b64 exec, s[34:35]213; SI-NEXT: s_waitcnt expcnt(0)214; SI-NEXT: v_writelane_b32 v2, s6, 0215; SI-NEXT: v_writelane_b32 v2, s7, 1216; SI-NEXT: v_mov_b32_e32 v0, s6217; SI-NEXT: v_mov_b32_e32 v1, s7218; SI-NEXT: s_mov_b32 s7, 0xf000219; SI-NEXT: s_mov_b32 s6, -1220; SI-NEXT: s_waitcnt vmcnt(0)221; SI-NEXT: buffer_atomic_swap_x2 v[0:1], off, s[4:7], 0 offset:32222; SI-NEXT: s_waitcnt vmcnt(0)223; SI-NEXT: buffer_wbinvl1224; SI-NEXT: v_readlane_b32 s7, v2, 1225; SI-NEXT: v_readlane_b32 s6, v2, 0226; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1227; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload228; SI-NEXT: s_mov_b64 exec, s[34:35]229; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)230; SI-NEXT: s_setpc_b64 s[30:31]231;232; VI-LABEL: global_atomic_xchg_i64_noret_offset_scalar:233; VI: ; %bb.0:234; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)235; VI-NEXT: s_add_u32 s34, s4, 32236; VI-NEXT: s_addc_u32 s35, s5, 0237; VI-NEXT: v_mov_b32_e32 v2, s34238; VI-NEXT: v_mov_b32_e32 v0, s6239; VI-NEXT: v_mov_b32_e32 v1, s7240; VI-NEXT: v_mov_b32_e32 v3, s35241; VI-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]242; VI-NEXT: s_waitcnt vmcnt(0)243; VI-NEXT: buffer_wbinvl1_vol244; VI-NEXT: s_setpc_b64 s[30:31]245;246; GFX9-LABEL: global_atomic_xchg_i64_noret_offset_scalar:247; GFX9: ; %bb.0:248; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)249; GFX9-NEXT: v_mov_b32_e32 v0, s6250; GFX9-NEXT: v_mov_b32_e32 v1, s7251; GFX9-NEXT: v_mov_b32_e32 v2, 0252; GFX9-NEXT: global_atomic_swap_x2 v2, v[0:1], s[4:5] offset:32253; GFX9-NEXT: s_waitcnt vmcnt(0)254; GFX9-NEXT: buffer_wbinvl1_vol255; GFX9-NEXT: s_setpc_b64 s[30:31]256 %gep = getelementptr i64, ptr addrspace(1) %out, i64 4257 %tmp0 = atomicrmw xchg ptr addrspace(1) %gep, i64 %in seq_cst258 ret void259}260 261define amdgpu_gfx i64 @global_atomic_xchg_i64_ret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {262; SI-LABEL: global_atomic_xchg_i64_ret_scalar:263; SI: ; %bb.0:264; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)265; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1266; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill267; SI-NEXT: s_mov_b64 exec, s[34:35]268; SI-NEXT: s_waitcnt expcnt(0)269; SI-NEXT: v_writelane_b32 v2, s6, 0270; SI-NEXT: v_writelane_b32 v2, s7, 1271; SI-NEXT: s_mov_b32 s34, s7272; SI-NEXT: s_mov_b32 s35, s6273; SI-NEXT: s_mov_b32 s7, 0xf000274; SI-NEXT: s_mov_b32 s6, -1275; SI-NEXT: v_mov_b32_e32 v0, s35276; SI-NEXT: v_mov_b32_e32 v1, s34277; SI-NEXT: s_waitcnt vmcnt(0)278; SI-NEXT: buffer_atomic_swap_x2 v[0:1], off, s[4:7], 0 glc279; SI-NEXT: s_waitcnt vmcnt(0)280; SI-NEXT: buffer_wbinvl1281; SI-NEXT: v_readlane_b32 s7, v2, 1282; SI-NEXT: v_readlane_b32 s6, v2, 0283; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1284; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload285; SI-NEXT: s_mov_b64 exec, s[34:35]286; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)287; SI-NEXT: s_setpc_b64 s[30:31]288;289; VI-LABEL: global_atomic_xchg_i64_ret_scalar:290; VI: ; %bb.0:291; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)292; VI-NEXT: v_mov_b32_e32 v0, s6293; VI-NEXT: v_mov_b32_e32 v1, s7294; VI-NEXT: v_mov_b32_e32 v2, s4295; VI-NEXT: v_mov_b32_e32 v3, s5296; VI-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc297; VI-NEXT: s_waitcnt vmcnt(0)298; VI-NEXT: buffer_wbinvl1_vol299; VI-NEXT: s_setpc_b64 s[30:31]300;301; GFX9-LABEL: global_atomic_xchg_i64_ret_scalar:302; GFX9: ; %bb.0:303; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)304; GFX9-NEXT: v_mov_b32_e32 v0, s6305; GFX9-NEXT: v_mov_b32_e32 v1, s7306; GFX9-NEXT: v_mov_b32_e32 v2, 0307; GFX9-NEXT: global_atomic_swap_x2 v[0:1], v2, v[0:1], s[4:5] glc308; GFX9-NEXT: s_waitcnt vmcnt(0)309; GFX9-NEXT: buffer_wbinvl1_vol310; GFX9-NEXT: s_setpc_b64 s[30:31]311 %result = atomicrmw xchg ptr addrspace(1) %ptr, i64 %in seq_cst312 ret i64 %result313}314 315define amdgpu_gfx i64 @global_atomic_xchg_i64_ret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {316; SI-LABEL: global_atomic_xchg_i64_ret_offset_scalar:317; SI: ; %bb.0:318; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)319; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1320; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill321; SI-NEXT: s_mov_b64 exec, s[34:35]322; SI-NEXT: s_waitcnt expcnt(0)323; SI-NEXT: v_writelane_b32 v2, s6, 0324; SI-NEXT: v_writelane_b32 v2, s7, 1325; SI-NEXT: v_mov_b32_e32 v0, s6326; SI-NEXT: v_mov_b32_e32 v1, s7327; SI-NEXT: s_mov_b32 s7, 0xf000328; SI-NEXT: s_mov_b32 s6, -1329; SI-NEXT: s_waitcnt vmcnt(0)330; SI-NEXT: buffer_atomic_swap_x2 v[0:1], off, s[4:7], 0 offset:32 glc331; SI-NEXT: s_waitcnt vmcnt(0)332; SI-NEXT: buffer_wbinvl1333; SI-NEXT: v_readlane_b32 s7, v2, 1334; SI-NEXT: v_readlane_b32 s6, v2, 0335; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1336; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload337; SI-NEXT: s_mov_b64 exec, s[34:35]338; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)339; SI-NEXT: s_setpc_b64 s[30:31]340;341; VI-LABEL: global_atomic_xchg_i64_ret_offset_scalar:342; VI: ; %bb.0:343; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)344; VI-NEXT: s_add_u32 s34, s4, 32345; VI-NEXT: s_addc_u32 s35, s5, 0346; VI-NEXT: v_mov_b32_e32 v2, s34347; VI-NEXT: v_mov_b32_e32 v0, s6348; VI-NEXT: v_mov_b32_e32 v1, s7349; VI-NEXT: v_mov_b32_e32 v3, s35350; VI-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc351; VI-NEXT: s_waitcnt vmcnt(0)352; VI-NEXT: buffer_wbinvl1_vol353; VI-NEXT: s_setpc_b64 s[30:31]354;355; GFX9-LABEL: global_atomic_xchg_i64_ret_offset_scalar:356; GFX9: ; %bb.0:357; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)358; GFX9-NEXT: v_mov_b32_e32 v0, s6359; GFX9-NEXT: v_mov_b32_e32 v1, s7360; GFX9-NEXT: v_mov_b32_e32 v2, 0361; GFX9-NEXT: global_atomic_swap_x2 v[0:1], v2, v[0:1], s[4:5] offset:32 glc362; GFX9-NEXT: s_waitcnt vmcnt(0)363; GFX9-NEXT: buffer_wbinvl1_vol364; GFX9-NEXT: s_setpc_b64 s[30:31]365 %gep = getelementptr i64, ptr addrspace(1) %out, i64 4366 %result = atomicrmw xchg ptr addrspace(1) %gep, i64 %in seq_cst367 ret i64 %result368}369 370define void @global_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {371; SI-LABEL: global_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory:372; SI: ; %bb.0:373; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)374; SI-NEXT: s_mov_b32 s6, 0375; SI-NEXT: s_mov_b32 s7, 0xf000376; SI-NEXT: s_mov_b32 s4, s6377; SI-NEXT: s_mov_b32 s5, s6378; SI-NEXT: buffer_atomic_swap_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32379; SI-NEXT: s_waitcnt vmcnt(0)380; SI-NEXT: buffer_wbinvl1381; SI-NEXT: s_waitcnt expcnt(0)382; SI-NEXT: s_setpc_b64 s[30:31]383;384; VI-LABEL: global_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory:385; VI: ; %bb.0:386; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)387; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v0388; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc389; VI-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]390; VI-NEXT: s_waitcnt vmcnt(0)391; VI-NEXT: buffer_wbinvl1_vol392; VI-NEXT: s_setpc_b64 s[30:31]393;394; GFX9-LABEL: global_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory:395; GFX9: ; %bb.0:396; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)397; GFX9-NEXT: global_atomic_swap_x2 v[0:1], v[2:3], off offset:32398; GFX9-NEXT: s_waitcnt vmcnt(0)399; GFX9-NEXT: buffer_wbinvl1_vol400; GFX9-NEXT: s_setpc_b64 s[30:31]401 %gep = getelementptr i64, ptr addrspace(1) %out, i64 4402 %tmp0 = atomicrmw xchg ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0403 ret void404}405 406define i64 @global_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {407; SI-LABEL: global_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory:408; SI: ; %bb.0:409; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)410; SI-NEXT: s_mov_b32 s6, 0411; SI-NEXT: s_mov_b32 s7, 0xf000412; SI-NEXT: s_mov_b32 s4, s6413; SI-NEXT: s_mov_b32 s5, s6414; SI-NEXT: buffer_atomic_swap_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc415; SI-NEXT: s_waitcnt vmcnt(0)416; SI-NEXT: buffer_wbinvl1417; SI-NEXT: v_mov_b32_e32 v0, v2418; SI-NEXT: v_mov_b32_e32 v1, v3419; SI-NEXT: s_waitcnt expcnt(0)420; SI-NEXT: s_setpc_b64 s[30:31]421;422; VI-LABEL: global_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory:423; VI: ; %bb.0:424; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)425; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v0426; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc427; VI-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc428; VI-NEXT: s_waitcnt vmcnt(0)429; VI-NEXT: buffer_wbinvl1_vol430; VI-NEXT: s_setpc_b64 s[30:31]431;432; GFX9-LABEL: global_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory:433; GFX9: ; %bb.0:434; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)435; GFX9-NEXT: global_atomic_swap_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc436; GFX9-NEXT: s_waitcnt vmcnt(0)437; GFX9-NEXT: buffer_wbinvl1_vol438; GFX9-NEXT: s_setpc_b64 s[30:31]439 %gep = getelementptr i64, ptr addrspace(1) %out, i64 4440 %result = atomicrmw xchg ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0441 ret i64 %result442}443 444; ---------------------------------------------------------------------445; atomicrmw xchg f64446; ---------------------------------------------------------------------447 448define void @global_atomic_xchg_f64_noret(ptr addrspace(1) %ptr, double %in) {449; SI-LABEL: global_atomic_xchg_f64_noret:450; SI: ; %bb.0:451; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)452; SI-NEXT: s_mov_b32 s6, 0453; SI-NEXT: s_mov_b32 s7, 0xf000454; SI-NEXT: s_mov_b32 s4, s6455; SI-NEXT: s_mov_b32 s5, s6456; SI-NEXT: buffer_atomic_swap_x2 v[2:3], v[0:1], s[4:7], 0 addr64457; SI-NEXT: s_waitcnt vmcnt(0)458; SI-NEXT: buffer_wbinvl1459; SI-NEXT: s_waitcnt expcnt(0)460; SI-NEXT: s_setpc_b64 s[30:31]461;462; VI-LABEL: global_atomic_xchg_f64_noret:463; VI: ; %bb.0:464; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)465; VI-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]466; VI-NEXT: s_waitcnt vmcnt(0)467; VI-NEXT: buffer_wbinvl1_vol468; VI-NEXT: s_setpc_b64 s[30:31]469;470; GFX9-LABEL: global_atomic_xchg_f64_noret:471; GFX9: ; %bb.0:472; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)473; GFX9-NEXT: global_atomic_swap_x2 v[0:1], v[2:3], off474; GFX9-NEXT: s_waitcnt vmcnt(0)475; GFX9-NEXT: buffer_wbinvl1_vol476; GFX9-NEXT: s_setpc_b64 s[30:31]477 %tmp0 = atomicrmw xchg ptr addrspace(1) %ptr, double %in seq_cst478 ret void479}480 481define void @global_atomic_xchg_f64_noret_offset(ptr addrspace(1) %out, double %in) {482; SI-LABEL: global_atomic_xchg_f64_noret_offset:483; SI: ; %bb.0:484; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)485; SI-NEXT: s_mov_b32 s6, 0486; SI-NEXT: s_mov_b32 s7, 0xf000487; SI-NEXT: s_mov_b32 s4, s6488; SI-NEXT: s_mov_b32 s5, s6489; SI-NEXT: buffer_atomic_swap_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32490; SI-NEXT: s_waitcnt vmcnt(0)491; SI-NEXT: buffer_wbinvl1492; SI-NEXT: s_waitcnt expcnt(0)493; SI-NEXT: s_setpc_b64 s[30:31]494;495; VI-LABEL: global_atomic_xchg_f64_noret_offset:496; VI: ; %bb.0:497; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)498; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v0499; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc500; VI-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]501; VI-NEXT: s_waitcnt vmcnt(0)502; VI-NEXT: buffer_wbinvl1_vol503; VI-NEXT: s_setpc_b64 s[30:31]504;505; GFX9-LABEL: global_atomic_xchg_f64_noret_offset:506; GFX9: ; %bb.0:507; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)508; GFX9-NEXT: global_atomic_swap_x2 v[0:1], v[2:3], off offset:32509; GFX9-NEXT: s_waitcnt vmcnt(0)510; GFX9-NEXT: buffer_wbinvl1_vol511; GFX9-NEXT: s_setpc_b64 s[30:31]512 %gep = getelementptr double, ptr addrspace(1) %out, i32 4513 %tmp0 = atomicrmw xchg ptr addrspace(1) %gep, double %in seq_cst514 ret void515}516 517define double @global_atomic_xchg_f64_ret(ptr addrspace(1) %ptr, double %in) {518; SI-LABEL: global_atomic_xchg_f64_ret:519; SI: ; %bb.0:520; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)521; SI-NEXT: s_mov_b32 s6, 0522; SI-NEXT: s_mov_b32 s7, 0xf000523; SI-NEXT: s_mov_b32 s4, s6524; SI-NEXT: s_mov_b32 s5, s6525; SI-NEXT: buffer_atomic_swap_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc526; SI-NEXT: s_waitcnt vmcnt(0)527; SI-NEXT: buffer_wbinvl1528; SI-NEXT: v_mov_b32_e32 v0, v2529; SI-NEXT: v_mov_b32_e32 v1, v3530; SI-NEXT: s_waitcnt expcnt(0)531; SI-NEXT: s_setpc_b64 s[30:31]532;533; VI-LABEL: global_atomic_xchg_f64_ret:534; VI: ; %bb.0:535; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)536; VI-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc537; VI-NEXT: s_waitcnt vmcnt(0)538; VI-NEXT: buffer_wbinvl1_vol539; VI-NEXT: s_setpc_b64 s[30:31]540;541; GFX9-LABEL: global_atomic_xchg_f64_ret:542; GFX9: ; %bb.0:543; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)544; GFX9-NEXT: global_atomic_swap_x2 v[0:1], v[0:1], v[2:3], off glc545; GFX9-NEXT: s_waitcnt vmcnt(0)546; GFX9-NEXT: buffer_wbinvl1_vol547; GFX9-NEXT: s_setpc_b64 s[30:31]548 %result = atomicrmw xchg ptr addrspace(1) %ptr, double %in seq_cst549 ret double %result550}551 552define double @global_atomic_xchg_f64_ret_offset(ptr addrspace(1) %out, double %in) {553; SI-LABEL: global_atomic_xchg_f64_ret_offset:554; SI: ; %bb.0:555; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)556; SI-NEXT: s_mov_b32 s6, 0557; SI-NEXT: s_mov_b32 s7, 0xf000558; SI-NEXT: s_mov_b32 s4, s6559; SI-NEXT: s_mov_b32 s5, s6560; SI-NEXT: buffer_atomic_swap_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc561; SI-NEXT: s_waitcnt vmcnt(0)562; SI-NEXT: buffer_wbinvl1563; SI-NEXT: v_mov_b32_e32 v0, v2564; SI-NEXT: v_mov_b32_e32 v1, v3565; SI-NEXT: s_waitcnt expcnt(0)566; SI-NEXT: s_setpc_b64 s[30:31]567;568; VI-LABEL: global_atomic_xchg_f64_ret_offset:569; VI: ; %bb.0:570; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)571; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v0572; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc573; VI-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc574; VI-NEXT: s_waitcnt vmcnt(0)575; VI-NEXT: buffer_wbinvl1_vol576; VI-NEXT: s_setpc_b64 s[30:31]577;578; GFX9-LABEL: global_atomic_xchg_f64_ret_offset:579; GFX9: ; %bb.0:580; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)581; GFX9-NEXT: global_atomic_swap_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc582; GFX9-NEXT: s_waitcnt vmcnt(0)583; GFX9-NEXT: buffer_wbinvl1_vol584; GFX9-NEXT: s_setpc_b64 s[30:31]585 %gep = getelementptr double, ptr addrspace(1) %out, i32 4586 %result = atomicrmw xchg ptr addrspace(1) %gep, double %in seq_cst587 ret double %result588}589 590define amdgpu_gfx void @global_atomic_xchg_f64_noret_scalar(ptr addrspace(1) inreg %ptr, double inreg %in) {591; SI-LABEL: global_atomic_xchg_f64_noret_scalar:592; SI: ; %bb.0:593; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)594; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1595; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill596; SI-NEXT: s_mov_b64 exec, s[34:35]597; SI-NEXT: s_waitcnt expcnt(0)598; SI-NEXT: v_writelane_b32 v2, s6, 0599; SI-NEXT: v_writelane_b32 v2, s7, 1600; SI-NEXT: s_mov_b32 s34, s7601; SI-NEXT: s_mov_b32 s35, s6602; SI-NEXT: s_mov_b32 s7, 0xf000603; SI-NEXT: s_mov_b32 s6, -1604; SI-NEXT: v_mov_b32_e32 v0, s35605; SI-NEXT: v_mov_b32_e32 v1, s34606; SI-NEXT: s_waitcnt vmcnt(0)607; SI-NEXT: buffer_atomic_swap_x2 v[0:1], off, s[4:7], 0608; SI-NEXT: s_waitcnt vmcnt(0)609; SI-NEXT: buffer_wbinvl1610; SI-NEXT: v_readlane_b32 s7, v2, 1611; SI-NEXT: v_readlane_b32 s6, v2, 0612; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1613; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload614; SI-NEXT: s_mov_b64 exec, s[34:35]615; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)616; SI-NEXT: s_setpc_b64 s[30:31]617;618; VI-LABEL: global_atomic_xchg_f64_noret_scalar:619; VI: ; %bb.0:620; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)621; VI-NEXT: v_mov_b32_e32 v0, s6622; VI-NEXT: v_mov_b32_e32 v1, s7623; VI-NEXT: v_mov_b32_e32 v2, s4624; VI-NEXT: v_mov_b32_e32 v3, s5625; VI-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]626; VI-NEXT: s_waitcnt vmcnt(0)627; VI-NEXT: buffer_wbinvl1_vol628; VI-NEXT: s_setpc_b64 s[30:31]629;630; GFX9-LABEL: global_atomic_xchg_f64_noret_scalar:631; GFX9: ; %bb.0:632; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)633; GFX9-NEXT: v_mov_b32_e32 v0, s6634; GFX9-NEXT: v_mov_b32_e32 v1, s7635; GFX9-NEXT: v_mov_b32_e32 v2, 0636; GFX9-NEXT: global_atomic_swap_x2 v2, v[0:1], s[4:5]637; GFX9-NEXT: s_waitcnt vmcnt(0)638; GFX9-NEXT: buffer_wbinvl1_vol639; GFX9-NEXT: s_setpc_b64 s[30:31]640 %tmp0 = atomicrmw xchg ptr addrspace(1) %ptr, double %in seq_cst641 ret void642}643 644define amdgpu_gfx void @global_atomic_xchg_f64_noret_offset_scalar(ptr addrspace(1) inreg %out, double inreg %in) {645; SI-LABEL: global_atomic_xchg_f64_noret_offset_scalar:646; SI: ; %bb.0:647; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)648; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1649; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill650; SI-NEXT: s_mov_b64 exec, s[34:35]651; SI-NEXT: s_waitcnt expcnt(0)652; SI-NEXT: v_writelane_b32 v2, s6, 0653; SI-NEXT: v_writelane_b32 v2, s7, 1654; SI-NEXT: v_mov_b32_e32 v0, s6655; SI-NEXT: v_mov_b32_e32 v1, s7656; SI-NEXT: s_mov_b32 s7, 0xf000657; SI-NEXT: s_mov_b32 s6, -1658; SI-NEXT: s_waitcnt vmcnt(0)659; SI-NEXT: buffer_atomic_swap_x2 v[0:1], off, s[4:7], 0 offset:32660; SI-NEXT: s_waitcnt vmcnt(0)661; SI-NEXT: buffer_wbinvl1662; SI-NEXT: v_readlane_b32 s7, v2, 1663; SI-NEXT: v_readlane_b32 s6, v2, 0664; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1665; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload666; SI-NEXT: s_mov_b64 exec, s[34:35]667; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)668; SI-NEXT: s_setpc_b64 s[30:31]669;670; VI-LABEL: global_atomic_xchg_f64_noret_offset_scalar:671; VI: ; %bb.0:672; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)673; VI-NEXT: s_add_u32 s34, s4, 32674; VI-NEXT: s_addc_u32 s35, s5, 0675; VI-NEXT: v_mov_b32_e32 v2, s34676; VI-NEXT: v_mov_b32_e32 v0, s6677; VI-NEXT: v_mov_b32_e32 v1, s7678; VI-NEXT: v_mov_b32_e32 v3, s35679; VI-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]680; VI-NEXT: s_waitcnt vmcnt(0)681; VI-NEXT: buffer_wbinvl1_vol682; VI-NEXT: s_setpc_b64 s[30:31]683;684; GFX9-LABEL: global_atomic_xchg_f64_noret_offset_scalar:685; GFX9: ; %bb.0:686; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)687; GFX9-NEXT: v_mov_b32_e32 v0, s6688; GFX9-NEXT: v_mov_b32_e32 v1, s7689; GFX9-NEXT: v_mov_b32_e32 v2, 0690; GFX9-NEXT: global_atomic_swap_x2 v2, v[0:1], s[4:5] offset:32691; GFX9-NEXT: s_waitcnt vmcnt(0)692; GFX9-NEXT: buffer_wbinvl1_vol693; GFX9-NEXT: s_setpc_b64 s[30:31]694 %gep = getelementptr double, ptr addrspace(1) %out, i32 4695 %tmp0 = atomicrmw xchg ptr addrspace(1) %gep, double %in seq_cst696 ret void697}698 699define amdgpu_gfx double @global_atomic_xchg_f64_ret_scalar(ptr addrspace(1) inreg %ptr, double inreg %in) {700; SI-LABEL: global_atomic_xchg_f64_ret_scalar:701; SI: ; %bb.0:702; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)703; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1704; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill705; SI-NEXT: s_mov_b64 exec, s[34:35]706; SI-NEXT: s_waitcnt expcnt(0)707; SI-NEXT: v_writelane_b32 v2, s6, 0708; SI-NEXT: v_writelane_b32 v2, s7, 1709; SI-NEXT: s_mov_b32 s34, s7710; SI-NEXT: s_mov_b32 s35, s6711; SI-NEXT: s_mov_b32 s7, 0xf000712; SI-NEXT: s_mov_b32 s6, -1713; SI-NEXT: v_mov_b32_e32 v0, s35714; SI-NEXT: v_mov_b32_e32 v1, s34715; SI-NEXT: s_waitcnt vmcnt(0)716; SI-NEXT: buffer_atomic_swap_x2 v[0:1], off, s[4:7], 0 glc717; SI-NEXT: s_waitcnt vmcnt(0)718; SI-NEXT: buffer_wbinvl1719; SI-NEXT: v_readlane_b32 s7, v2, 1720; SI-NEXT: v_readlane_b32 s6, v2, 0721; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1722; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload723; SI-NEXT: s_mov_b64 exec, s[34:35]724; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)725; SI-NEXT: s_setpc_b64 s[30:31]726;727; VI-LABEL: global_atomic_xchg_f64_ret_scalar:728; VI: ; %bb.0:729; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)730; VI-NEXT: v_mov_b32_e32 v0, s6731; VI-NEXT: v_mov_b32_e32 v1, s7732; VI-NEXT: v_mov_b32_e32 v2, s4733; VI-NEXT: v_mov_b32_e32 v3, s5734; VI-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc735; VI-NEXT: s_waitcnt vmcnt(0)736; VI-NEXT: buffer_wbinvl1_vol737; VI-NEXT: s_setpc_b64 s[30:31]738;739; GFX9-LABEL: global_atomic_xchg_f64_ret_scalar:740; GFX9: ; %bb.0:741; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)742; GFX9-NEXT: v_mov_b32_e32 v0, s6743; GFX9-NEXT: v_mov_b32_e32 v1, s7744; GFX9-NEXT: v_mov_b32_e32 v2, 0745; GFX9-NEXT: global_atomic_swap_x2 v[0:1], v2, v[0:1], s[4:5] glc746; GFX9-NEXT: s_waitcnt vmcnt(0)747; GFX9-NEXT: buffer_wbinvl1_vol748; GFX9-NEXT: s_setpc_b64 s[30:31]749 %result = atomicrmw xchg ptr addrspace(1) %ptr, double %in seq_cst750 ret double %result751}752 753define amdgpu_gfx double @global_atomic_xchg_f64_ret_offset_scalar(ptr addrspace(1) inreg %out, double inreg %in) {754; SI-LABEL: global_atomic_xchg_f64_ret_offset_scalar:755; SI: ; %bb.0:756; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)757; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1758; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill759; SI-NEXT: s_mov_b64 exec, s[34:35]760; SI-NEXT: s_waitcnt expcnt(0)761; SI-NEXT: v_writelane_b32 v2, s6, 0762; SI-NEXT: v_writelane_b32 v2, s7, 1763; SI-NEXT: v_mov_b32_e32 v0, s6764; SI-NEXT: v_mov_b32_e32 v1, s7765; SI-NEXT: s_mov_b32 s7, 0xf000766; SI-NEXT: s_mov_b32 s6, -1767; SI-NEXT: s_waitcnt vmcnt(0)768; SI-NEXT: buffer_atomic_swap_x2 v[0:1], off, s[4:7], 0 offset:32 glc769; SI-NEXT: s_waitcnt vmcnt(0)770; SI-NEXT: buffer_wbinvl1771; SI-NEXT: v_readlane_b32 s7, v2, 1772; SI-NEXT: v_readlane_b32 s6, v2, 0773; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1774; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload775; SI-NEXT: s_mov_b64 exec, s[34:35]776; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)777; SI-NEXT: s_setpc_b64 s[30:31]778;779; VI-LABEL: global_atomic_xchg_f64_ret_offset_scalar:780; VI: ; %bb.0:781; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)782; VI-NEXT: s_add_u32 s34, s4, 32783; VI-NEXT: s_addc_u32 s35, s5, 0784; VI-NEXT: v_mov_b32_e32 v2, s34785; VI-NEXT: v_mov_b32_e32 v0, s6786; VI-NEXT: v_mov_b32_e32 v1, s7787; VI-NEXT: v_mov_b32_e32 v3, s35788; VI-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc789; VI-NEXT: s_waitcnt vmcnt(0)790; VI-NEXT: buffer_wbinvl1_vol791; VI-NEXT: s_setpc_b64 s[30:31]792;793; GFX9-LABEL: global_atomic_xchg_f64_ret_offset_scalar:794; GFX9: ; %bb.0:795; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)796; GFX9-NEXT: v_mov_b32_e32 v0, s6797; GFX9-NEXT: v_mov_b32_e32 v1, s7798; GFX9-NEXT: v_mov_b32_e32 v2, 0799; GFX9-NEXT: global_atomic_swap_x2 v[0:1], v2, v[0:1], s[4:5] offset:32 glc800; GFX9-NEXT: s_waitcnt vmcnt(0)801; GFX9-NEXT: buffer_wbinvl1_vol802; GFX9-NEXT: s_setpc_b64 s[30:31]803 %gep = getelementptr double, ptr addrspace(1) %out, i32 4804 %result = atomicrmw xchg ptr addrspace(1) %gep, double %in seq_cst805 ret double %result806}807 808define void @global_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, double %in) {809; SI-LABEL: global_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory:810; SI: ; %bb.0:811; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)812; SI-NEXT: s_mov_b32 s6, 0813; SI-NEXT: s_mov_b32 s7, 0xf000814; SI-NEXT: s_mov_b32 s4, s6815; SI-NEXT: s_mov_b32 s5, s6816; SI-NEXT: buffer_atomic_swap_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:16817; SI-NEXT: s_waitcnt vmcnt(0)818; SI-NEXT: buffer_wbinvl1819; SI-NEXT: s_waitcnt expcnt(0)820; SI-NEXT: s_setpc_b64 s[30:31]821;822; VI-LABEL: global_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory:823; VI: ; %bb.0:824; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)825; VI-NEXT: v_add_u32_e32 v0, vcc, 16, v0826; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc827; VI-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]828; VI-NEXT: s_waitcnt vmcnt(0)829; VI-NEXT: buffer_wbinvl1_vol830; VI-NEXT: s_setpc_b64 s[30:31]831;832; GFX9-LABEL: global_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory:833; GFX9: ; %bb.0:834; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)835; GFX9-NEXT: global_atomic_swap_x2 v[0:1], v[2:3], off offset:16836; GFX9-NEXT: s_waitcnt vmcnt(0)837; GFX9-NEXT: buffer_wbinvl1_vol838; GFX9-NEXT: s_setpc_b64 s[30:31]839 %gep = getelementptr i32, ptr addrspace(1) %out, i64 4840 %tmp0 = atomicrmw xchg ptr addrspace(1) %gep, double %in seq_cst, !amdgpu.no.remote.memory !0841 ret void842}843 844define double @global_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, double %in) {845; SI-LABEL: global_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory:846; SI: ; %bb.0:847; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)848; SI-NEXT: s_mov_b32 s6, 0849; SI-NEXT: s_mov_b32 s7, 0xf000850; SI-NEXT: s_mov_b32 s4, s6851; SI-NEXT: s_mov_b32 s5, s6852; SI-NEXT: buffer_atomic_swap_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:16 glc853; SI-NEXT: s_waitcnt vmcnt(0)854; SI-NEXT: buffer_wbinvl1855; SI-NEXT: v_mov_b32_e32 v0, v2856; SI-NEXT: v_mov_b32_e32 v1, v3857; SI-NEXT: s_waitcnt expcnt(0)858; SI-NEXT: s_setpc_b64 s[30:31]859;860; VI-LABEL: global_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory:861; VI: ; %bb.0:862; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)863; VI-NEXT: v_add_u32_e32 v0, vcc, 16, v0864; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc865; VI-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc866; VI-NEXT: s_waitcnt vmcnt(0)867; VI-NEXT: buffer_wbinvl1_vol868; VI-NEXT: s_setpc_b64 s[30:31]869;870; GFX9-LABEL: global_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory:871; GFX9: ; %bb.0:872; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)873; GFX9-NEXT: global_atomic_swap_x2 v[0:1], v[0:1], v[2:3], off offset:16 glc874; GFX9-NEXT: s_waitcnt vmcnt(0)875; GFX9-NEXT: buffer_wbinvl1_vol876; GFX9-NEXT: s_setpc_b64 s[30:31]877 %gep = getelementptr i32, ptr addrspace(1) %out, i64 4878 %result = atomicrmw xchg ptr addrspace(1) %gep, double %in seq_cst, !amdgpu.no.remote.memory !0879 ret double %result880}881 882; ---------------------------------------------------------------------883; atomicrmw add884; ---------------------------------------------------------------------885 886define void @global_atomic_add_i64_noret(ptr addrspace(1) %ptr, i64 %in) {887; SI-LABEL: global_atomic_add_i64_noret:888; SI: ; %bb.0:889; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)890; SI-NEXT: s_mov_b32 s6, 0891; SI-NEXT: s_mov_b32 s7, 0xf000892; SI-NEXT: s_mov_b32 s4, s6893; SI-NEXT: s_mov_b32 s5, s6894; SI-NEXT: buffer_atomic_add_x2 v[2:3], v[0:1], s[4:7], 0 addr64895; SI-NEXT: s_waitcnt vmcnt(0)896; SI-NEXT: buffer_wbinvl1897; SI-NEXT: s_waitcnt expcnt(0)898; SI-NEXT: s_setpc_b64 s[30:31]899;900; VI-LABEL: global_atomic_add_i64_noret:901; VI: ; %bb.0:902; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)903; VI-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]904; VI-NEXT: s_waitcnt vmcnt(0)905; VI-NEXT: buffer_wbinvl1_vol906; VI-NEXT: s_setpc_b64 s[30:31]907;908; GFX9-LABEL: global_atomic_add_i64_noret:909; GFX9: ; %bb.0:910; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)911; GFX9-NEXT: global_atomic_add_x2 v[0:1], v[2:3], off912; GFX9-NEXT: s_waitcnt vmcnt(0)913; GFX9-NEXT: buffer_wbinvl1_vol914; GFX9-NEXT: s_setpc_b64 s[30:31]915 %tmp0 = atomicrmw add ptr addrspace(1) %ptr, i64 %in seq_cst916 ret void917}918 919define void @global_atomic_add_i64_noret_offset(ptr addrspace(1) %out, i64 %in) {920; SI-LABEL: global_atomic_add_i64_noret_offset:921; SI: ; %bb.0:922; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)923; SI-NEXT: s_mov_b32 s6, 0924; SI-NEXT: s_mov_b32 s7, 0xf000925; SI-NEXT: s_mov_b32 s4, s6926; SI-NEXT: s_mov_b32 s5, s6927; SI-NEXT: buffer_atomic_add_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32928; SI-NEXT: s_waitcnt vmcnt(0)929; SI-NEXT: buffer_wbinvl1930; SI-NEXT: s_waitcnt expcnt(0)931; SI-NEXT: s_setpc_b64 s[30:31]932;933; VI-LABEL: global_atomic_add_i64_noret_offset:934; VI: ; %bb.0:935; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)936; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v0937; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc938; VI-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]939; VI-NEXT: s_waitcnt vmcnt(0)940; VI-NEXT: buffer_wbinvl1_vol941; VI-NEXT: s_setpc_b64 s[30:31]942;943; GFX9-LABEL: global_atomic_add_i64_noret_offset:944; GFX9: ; %bb.0:945; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)946; GFX9-NEXT: global_atomic_add_x2 v[0:1], v[2:3], off offset:32947; GFX9-NEXT: s_waitcnt vmcnt(0)948; GFX9-NEXT: buffer_wbinvl1_vol949; GFX9-NEXT: s_setpc_b64 s[30:31]950 %gep = getelementptr i64, ptr addrspace(1) %out, i64 4951 %tmp0 = atomicrmw add ptr addrspace(1) %gep, i64 %in seq_cst952 ret void953}954 955define i64 @global_atomic_add_i64_ret(ptr addrspace(1) %ptr, i64 %in) {956; SI-LABEL: global_atomic_add_i64_ret:957; SI: ; %bb.0:958; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)959; SI-NEXT: s_mov_b32 s6, 0960; SI-NEXT: s_mov_b32 s7, 0xf000961; SI-NEXT: s_mov_b32 s4, s6962; SI-NEXT: s_mov_b32 s5, s6963; SI-NEXT: buffer_atomic_add_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc964; SI-NEXT: s_waitcnt vmcnt(0)965; SI-NEXT: buffer_wbinvl1966; SI-NEXT: v_mov_b32_e32 v0, v2967; SI-NEXT: v_mov_b32_e32 v1, v3968; SI-NEXT: s_waitcnt expcnt(0)969; SI-NEXT: s_setpc_b64 s[30:31]970;971; VI-LABEL: global_atomic_add_i64_ret:972; VI: ; %bb.0:973; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)974; VI-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc975; VI-NEXT: s_waitcnt vmcnt(0)976; VI-NEXT: buffer_wbinvl1_vol977; VI-NEXT: s_setpc_b64 s[30:31]978;979; GFX9-LABEL: global_atomic_add_i64_ret:980; GFX9: ; %bb.0:981; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)982; GFX9-NEXT: global_atomic_add_x2 v[0:1], v[0:1], v[2:3], off glc983; GFX9-NEXT: s_waitcnt vmcnt(0)984; GFX9-NEXT: buffer_wbinvl1_vol985; GFX9-NEXT: s_setpc_b64 s[30:31]986 %result = atomicrmw add ptr addrspace(1) %ptr, i64 %in seq_cst987 ret i64 %result988}989 990define i64 @global_atomic_add_i64_ret_offset(ptr addrspace(1) %out, i64 %in) {991; SI-LABEL: global_atomic_add_i64_ret_offset:992; SI: ; %bb.0:993; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)994; SI-NEXT: s_mov_b32 s6, 0995; SI-NEXT: s_mov_b32 s7, 0xf000996; SI-NEXT: s_mov_b32 s4, s6997; SI-NEXT: s_mov_b32 s5, s6998; SI-NEXT: buffer_atomic_add_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc999; SI-NEXT: s_waitcnt vmcnt(0)1000; SI-NEXT: buffer_wbinvl11001; SI-NEXT: v_mov_b32_e32 v0, v21002; SI-NEXT: v_mov_b32_e32 v1, v31003; SI-NEXT: s_waitcnt expcnt(0)1004; SI-NEXT: s_setpc_b64 s[30:31]1005;1006; VI-LABEL: global_atomic_add_i64_ret_offset:1007; VI: ; %bb.0:1008; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1009; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v01010; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1011; VI-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc1012; VI-NEXT: s_waitcnt vmcnt(0)1013; VI-NEXT: buffer_wbinvl1_vol1014; VI-NEXT: s_setpc_b64 s[30:31]1015;1016; GFX9-LABEL: global_atomic_add_i64_ret_offset:1017; GFX9: ; %bb.0:1018; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1019; GFX9-NEXT: global_atomic_add_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc1020; GFX9-NEXT: s_waitcnt vmcnt(0)1021; GFX9-NEXT: buffer_wbinvl1_vol1022; GFX9-NEXT: s_setpc_b64 s[30:31]1023 %gep = getelementptr i64, ptr addrspace(1) %out, i64 41024 %result = atomicrmw add ptr addrspace(1) %gep, i64 %in seq_cst1025 ret i64 %result1026}1027 1028define amdgpu_gfx void @global_atomic_add_i64_noret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {1029; SI-LABEL: global_atomic_add_i64_noret_scalar:1030; SI: ; %bb.0:1031; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1032; SI-NEXT: s_xor_saveexec_b64 s[34:35], -11033; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill1034; SI-NEXT: s_mov_b64 exec, s[34:35]1035; SI-NEXT: s_waitcnt expcnt(0)1036; SI-NEXT: v_writelane_b32 v2, s6, 01037; SI-NEXT: v_writelane_b32 v2, s7, 11038; SI-NEXT: s_mov_b32 s34, s71039; SI-NEXT: s_mov_b32 s35, s61040; SI-NEXT: s_mov_b32 s7, 0xf0001041; SI-NEXT: s_mov_b32 s6, -11042; SI-NEXT: v_mov_b32_e32 v0, s351043; SI-NEXT: v_mov_b32_e32 v1, s341044; SI-NEXT: s_waitcnt vmcnt(0)1045; SI-NEXT: buffer_atomic_add_x2 v[0:1], off, s[4:7], 01046; SI-NEXT: s_waitcnt vmcnt(0)1047; SI-NEXT: buffer_wbinvl11048; SI-NEXT: v_readlane_b32 s7, v2, 11049; SI-NEXT: v_readlane_b32 s6, v2, 01050; SI-NEXT: s_xor_saveexec_b64 s[34:35], -11051; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload1052; SI-NEXT: s_mov_b64 exec, s[34:35]1053; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)1054; SI-NEXT: s_setpc_b64 s[30:31]1055;1056; VI-LABEL: global_atomic_add_i64_noret_scalar:1057; VI: ; %bb.0:1058; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1059; VI-NEXT: v_mov_b32_e32 v0, s61060; VI-NEXT: v_mov_b32_e32 v1, s71061; VI-NEXT: v_mov_b32_e32 v2, s41062; VI-NEXT: v_mov_b32_e32 v3, s51063; VI-NEXT: flat_atomic_add_x2 v[2:3], v[0:1]1064; VI-NEXT: s_waitcnt vmcnt(0)1065; VI-NEXT: buffer_wbinvl1_vol1066; VI-NEXT: s_setpc_b64 s[30:31]1067;1068; GFX9-LABEL: global_atomic_add_i64_noret_scalar:1069; GFX9: ; %bb.0:1070; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1071; GFX9-NEXT: v_mov_b32_e32 v0, s61072; GFX9-NEXT: v_mov_b32_e32 v1, s71073; GFX9-NEXT: v_mov_b32_e32 v2, 01074; GFX9-NEXT: global_atomic_add_x2 v2, v[0:1], s[4:5]1075; GFX9-NEXT: s_waitcnt vmcnt(0)1076; GFX9-NEXT: buffer_wbinvl1_vol1077; GFX9-NEXT: s_setpc_b64 s[30:31]1078 %tmp0 = atomicrmw add ptr addrspace(1) %ptr, i64 %in seq_cst1079 ret void1080}1081 1082define amdgpu_gfx void @global_atomic_add_i64_noret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {1083; SI-LABEL: global_atomic_add_i64_noret_offset_scalar:1084; SI: ; %bb.0:1085; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1086; SI-NEXT: s_xor_saveexec_b64 s[34:35], -11087; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill1088; SI-NEXT: s_mov_b64 exec, s[34:35]1089; SI-NEXT: s_waitcnt expcnt(0)1090; SI-NEXT: v_writelane_b32 v2, s6, 01091; SI-NEXT: v_writelane_b32 v2, s7, 11092; SI-NEXT: v_mov_b32_e32 v0, s61093; SI-NEXT: v_mov_b32_e32 v1, s71094; SI-NEXT: s_mov_b32 s7, 0xf0001095; SI-NEXT: s_mov_b32 s6, -11096; SI-NEXT: s_waitcnt vmcnt(0)1097; SI-NEXT: buffer_atomic_add_x2 v[0:1], off, s[4:7], 0 offset:321098; SI-NEXT: s_waitcnt vmcnt(0)1099; SI-NEXT: buffer_wbinvl11100; SI-NEXT: v_readlane_b32 s7, v2, 11101; SI-NEXT: v_readlane_b32 s6, v2, 01102; SI-NEXT: s_xor_saveexec_b64 s[34:35], -11103; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload1104; SI-NEXT: s_mov_b64 exec, s[34:35]1105; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)1106; SI-NEXT: s_setpc_b64 s[30:31]1107;1108; VI-LABEL: global_atomic_add_i64_noret_offset_scalar:1109; VI: ; %bb.0:1110; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1111; VI-NEXT: s_add_u32 s34, s4, 321112; VI-NEXT: s_addc_u32 s35, s5, 01113; VI-NEXT: v_mov_b32_e32 v2, s341114; VI-NEXT: v_mov_b32_e32 v0, s61115; VI-NEXT: v_mov_b32_e32 v1, s71116; VI-NEXT: v_mov_b32_e32 v3, s351117; VI-NEXT: flat_atomic_add_x2 v[2:3], v[0:1]1118; VI-NEXT: s_waitcnt vmcnt(0)1119; VI-NEXT: buffer_wbinvl1_vol1120; VI-NEXT: s_setpc_b64 s[30:31]1121;1122; GFX9-LABEL: global_atomic_add_i64_noret_offset_scalar:1123; GFX9: ; %bb.0:1124; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1125; GFX9-NEXT: v_mov_b32_e32 v0, s61126; GFX9-NEXT: v_mov_b32_e32 v1, s71127; GFX9-NEXT: v_mov_b32_e32 v2, 01128; GFX9-NEXT: global_atomic_add_x2 v2, v[0:1], s[4:5] offset:321129; GFX9-NEXT: s_waitcnt vmcnt(0)1130; GFX9-NEXT: buffer_wbinvl1_vol1131; GFX9-NEXT: s_setpc_b64 s[30:31]1132 %gep = getelementptr i64, ptr addrspace(1) %out, i64 41133 %tmp0 = atomicrmw add ptr addrspace(1) %gep, i64 %in seq_cst1134 ret void1135}1136 1137define amdgpu_gfx i64 @global_atomic_add_i64_ret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {1138; SI-LABEL: global_atomic_add_i64_ret_scalar:1139; SI: ; %bb.0:1140; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1141; SI-NEXT: s_xor_saveexec_b64 s[34:35], -11142; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill1143; SI-NEXT: s_mov_b64 exec, s[34:35]1144; SI-NEXT: s_waitcnt expcnt(0)1145; SI-NEXT: v_writelane_b32 v2, s6, 01146; SI-NEXT: v_writelane_b32 v2, s7, 11147; SI-NEXT: s_mov_b32 s34, s71148; SI-NEXT: s_mov_b32 s35, s61149; SI-NEXT: s_mov_b32 s7, 0xf0001150; SI-NEXT: s_mov_b32 s6, -11151; SI-NEXT: v_mov_b32_e32 v0, s351152; SI-NEXT: v_mov_b32_e32 v1, s341153; SI-NEXT: s_waitcnt vmcnt(0)1154; SI-NEXT: buffer_atomic_add_x2 v[0:1], off, s[4:7], 0 glc1155; SI-NEXT: s_waitcnt vmcnt(0)1156; SI-NEXT: buffer_wbinvl11157; SI-NEXT: v_readlane_b32 s7, v2, 11158; SI-NEXT: v_readlane_b32 s6, v2, 01159; SI-NEXT: s_xor_saveexec_b64 s[34:35], -11160; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload1161; SI-NEXT: s_mov_b64 exec, s[34:35]1162; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)1163; SI-NEXT: s_setpc_b64 s[30:31]1164;1165; VI-LABEL: global_atomic_add_i64_ret_scalar:1166; VI: ; %bb.0:1167; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1168; VI-NEXT: v_mov_b32_e32 v0, s61169; VI-NEXT: v_mov_b32_e32 v1, s71170; VI-NEXT: v_mov_b32_e32 v2, s41171; VI-NEXT: v_mov_b32_e32 v3, s51172; VI-NEXT: flat_atomic_add_x2 v[0:1], v[2:3], v[0:1] glc1173; VI-NEXT: s_waitcnt vmcnt(0)1174; VI-NEXT: buffer_wbinvl1_vol1175; VI-NEXT: s_setpc_b64 s[30:31]1176;1177; GFX9-LABEL: global_atomic_add_i64_ret_scalar:1178; GFX9: ; %bb.0:1179; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1180; GFX9-NEXT: v_mov_b32_e32 v0, s61181; GFX9-NEXT: v_mov_b32_e32 v1, s71182; GFX9-NEXT: v_mov_b32_e32 v2, 01183; GFX9-NEXT: global_atomic_add_x2 v[0:1], v2, v[0:1], s[4:5] glc1184; GFX9-NEXT: s_waitcnt vmcnt(0)1185; GFX9-NEXT: buffer_wbinvl1_vol1186; GFX9-NEXT: s_setpc_b64 s[30:31]1187 %result = atomicrmw add ptr addrspace(1) %ptr, i64 %in seq_cst1188 ret i64 %result1189}1190 1191define amdgpu_gfx i64 @global_atomic_add_i64_ret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {1192; SI-LABEL: global_atomic_add_i64_ret_offset_scalar:1193; SI: ; %bb.0:1194; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1195; SI-NEXT: s_xor_saveexec_b64 s[34:35], -11196; SI-NEXT: buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill1197; SI-NEXT: s_mov_b64 exec, s[34:35]1198; SI-NEXT: s_waitcnt expcnt(0)1199; SI-NEXT: v_writelane_b32 v2, s6, 01200; SI-NEXT: v_writelane_b32 v2, s7, 11201; SI-NEXT: v_mov_b32_e32 v0, s61202; SI-NEXT: v_mov_b32_e32 v1, s71203; SI-NEXT: s_mov_b32 s7, 0xf0001204; SI-NEXT: s_mov_b32 s6, -11205; SI-NEXT: s_waitcnt vmcnt(0)1206; SI-NEXT: buffer_atomic_add_x2 v[0:1], off, s[4:7], 0 offset:32 glc1207; SI-NEXT: s_waitcnt vmcnt(0)1208; SI-NEXT: buffer_wbinvl11209; SI-NEXT: v_readlane_b32 s7, v2, 11210; SI-NEXT: v_readlane_b32 s6, v2, 01211; SI-NEXT: s_xor_saveexec_b64 s[34:35], -11212; SI-NEXT: buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload1213; SI-NEXT: s_mov_b64 exec, s[34:35]1214; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)1215; SI-NEXT: s_setpc_b64 s[30:31]1216;1217; VI-LABEL: global_atomic_add_i64_ret_offset_scalar:1218; VI: ; %bb.0:1219; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1220; VI-NEXT: s_add_u32 s34, s4, 321221; VI-NEXT: s_addc_u32 s35, s5, 01222; VI-NEXT: v_mov_b32_e32 v2, s341223; VI-NEXT: v_mov_b32_e32 v0, s61224; VI-NEXT: v_mov_b32_e32 v1, s71225; VI-NEXT: v_mov_b32_e32 v3, s351226; VI-NEXT: flat_atomic_add_x2 v[0:1], v[2:3], v[0:1] glc1227; VI-NEXT: s_waitcnt vmcnt(0)1228; VI-NEXT: buffer_wbinvl1_vol1229; VI-NEXT: s_setpc_b64 s[30:31]1230;1231; GFX9-LABEL: global_atomic_add_i64_ret_offset_scalar:1232; GFX9: ; %bb.0:1233; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1234; GFX9-NEXT: v_mov_b32_e32 v0, s61235; GFX9-NEXT: v_mov_b32_e32 v1, s71236; GFX9-NEXT: v_mov_b32_e32 v2, 01237; GFX9-NEXT: global_atomic_add_x2 v[0:1], v2, v[0:1], s[4:5] offset:32 glc1238; GFX9-NEXT: s_waitcnt vmcnt(0)1239; GFX9-NEXT: buffer_wbinvl1_vol1240; GFX9-NEXT: s_setpc_b64 s[30:31]1241 %gep = getelementptr i64, ptr addrspace(1) %out, i64 41242 %result = atomicrmw add ptr addrspace(1) %gep, i64 %in seq_cst1243 ret i64 %result1244}1245 1246define void @global_atomic_add_i64_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {1247; SI-LABEL: global_atomic_add_i64_noret_offset__amdgpu_no_remote_memory:1248; SI: ; %bb.0:1249; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1250; SI-NEXT: s_mov_b32 s6, 01251; SI-NEXT: s_mov_b32 s7, 0xf0001252; SI-NEXT: s_mov_b32 s4, s61253; SI-NEXT: s_mov_b32 s5, s61254; SI-NEXT: buffer_atomic_add_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:321255; SI-NEXT: s_waitcnt vmcnt(0)1256; SI-NEXT: buffer_wbinvl11257; SI-NEXT: s_waitcnt expcnt(0)1258; SI-NEXT: s_setpc_b64 s[30:31]1259;1260; VI-LABEL: global_atomic_add_i64_noret_offset__amdgpu_no_remote_memory:1261; VI: ; %bb.0:1262; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1263; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v01264; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1265; VI-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]1266; VI-NEXT: s_waitcnt vmcnt(0)1267; VI-NEXT: buffer_wbinvl1_vol1268; VI-NEXT: s_setpc_b64 s[30:31]1269;1270; GFX9-LABEL: global_atomic_add_i64_noret_offset__amdgpu_no_remote_memory:1271; GFX9: ; %bb.0:1272; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1273; GFX9-NEXT: global_atomic_add_x2 v[0:1], v[2:3], off offset:321274; GFX9-NEXT: s_waitcnt vmcnt(0)1275; GFX9-NEXT: buffer_wbinvl1_vol1276; GFX9-NEXT: s_setpc_b64 s[30:31]1277 %gep = getelementptr i64, ptr addrspace(1) %out, i64 41278 %tmp0 = atomicrmw add ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !01279 ret void1280}1281 1282define i64 @global_atomic_add_i64_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {1283; SI-LABEL: global_atomic_add_i64_ret_offset__amdgpu_no_remote_memory:1284; SI: ; %bb.0:1285; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1286; SI-NEXT: s_mov_b32 s6, 01287; SI-NEXT: s_mov_b32 s7, 0xf0001288; SI-NEXT: s_mov_b32 s4, s61289; SI-NEXT: s_mov_b32 s5, s61290; SI-NEXT: buffer_atomic_add_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc1291; SI-NEXT: s_waitcnt vmcnt(0)1292; SI-NEXT: buffer_wbinvl11293; SI-NEXT: v_mov_b32_e32 v0, v21294; SI-NEXT: v_mov_b32_e32 v1, v31295; SI-NEXT: s_waitcnt expcnt(0)1296; SI-NEXT: s_setpc_b64 s[30:31]1297;1298; VI-LABEL: global_atomic_add_i64_ret_offset__amdgpu_no_remote_memory:1299; VI: ; %bb.0:1300; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1301; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v01302; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1303; VI-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc1304; VI-NEXT: s_waitcnt vmcnt(0)1305; VI-NEXT: buffer_wbinvl1_vol1306; VI-NEXT: s_setpc_b64 s[30:31]1307;1308; GFX9-LABEL: global_atomic_add_i64_ret_offset__amdgpu_no_remote_memory:1309; GFX9: ; %bb.0:1310; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1311; GFX9-NEXT: global_atomic_add_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc1312; GFX9-NEXT: s_waitcnt vmcnt(0)1313; GFX9-NEXT: buffer_wbinvl1_vol1314; GFX9-NEXT: s_setpc_b64 s[30:31]1315 %gep = getelementptr i64, ptr addrspace(1) %out, i64 41316 %result = atomicrmw add ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !01317 ret i64 %result1318}1319 1320; ---------------------------------------------------------------------1321; atomicrmw sub1322; ---------------------------------------------------------------------1323 1324define void @global_atomic_sub_i64_noret(ptr addrspace(1) %ptr, i64 %in) {1325; SI-LABEL: global_atomic_sub_i64_noret:1326; SI: ; %bb.0:1327; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1328; SI-NEXT: s_mov_b32 s6, 01329; SI-NEXT: s_mov_b32 s7, 0xf0001330; SI-NEXT: s_mov_b32 s4, s61331; SI-NEXT: s_mov_b32 s5, s61332; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr641333; SI-NEXT: s_mov_b64 s[8:9], 01334; SI-NEXT: .LBB30_1: ; %atomicrmw.start1335; SI-NEXT: ; =>This Inner Loop Header: Depth=11336; SI-NEXT: s_waitcnt vmcnt(0)1337; SI-NEXT: v_sub_i32_e32 v4, vcc, v6, v21338; SI-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc1339; SI-NEXT: s_waitcnt expcnt(0)1340; SI-NEXT: v_mov_b32_e32 v11, v71341; SI-NEXT: v_mov_b32_e32 v10, v61342; SI-NEXT: v_mov_b32_e32 v9, v51343; SI-NEXT: v_mov_b32_e32 v8, v41344; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc1345; SI-NEXT: s_waitcnt vmcnt(0)1346; SI-NEXT: buffer_wbinvl11347; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]1348; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]1349; SI-NEXT: v_mov_b32_e32 v6, v81350; SI-NEXT: v_mov_b32_e32 v7, v91351; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]1352; SI-NEXT: s_cbranch_execnz .LBB30_11353; SI-NEXT: ; %bb.2: ; %atomicrmw.end1354; SI-NEXT: s_or_b64 exec, exec, s[8:9]1355; SI-NEXT: s_waitcnt expcnt(0)1356; SI-NEXT: s_setpc_b64 s[30:31]1357;1358; VI-LABEL: global_atomic_sub_i64_noret:1359; VI: ; %bb.0:1360; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1361; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]1362; VI-NEXT: s_mov_b64 s[4:5], 01363; VI-NEXT: .LBB30_1: ; %atomicrmw.start1364; VI-NEXT: ; =>This Inner Loop Header: Depth=11365; VI-NEXT: s_waitcnt vmcnt(0)1366; VI-NEXT: v_sub_u32_e32 v4, vcc, v6, v21367; VI-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc1368; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc1369; VI-NEXT: s_waitcnt vmcnt(0)1370; VI-NEXT: buffer_wbinvl1_vol1371; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]1372; VI-NEXT: v_mov_b32_e32 v7, v51373; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1374; VI-NEXT: v_mov_b32_e32 v6, v41375; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]1376; VI-NEXT: s_cbranch_execnz .LBB30_11377; VI-NEXT: ; %bb.2: ; %atomicrmw.end1378; VI-NEXT: s_or_b64 exec, exec, s[4:5]1379; VI-NEXT: s_setpc_b64 s[30:31]1380;1381; GFX9-LABEL: global_atomic_sub_i64_noret:1382; GFX9: ; %bb.0:1383; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1384; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off1385; GFX9-NEXT: s_mov_b64 s[4:5], 01386; GFX9-NEXT: .LBB30_1: ; %atomicrmw.start1387; GFX9-NEXT: ; =>This Inner Loop Header: Depth=11388; GFX9-NEXT: s_waitcnt vmcnt(0)1389; GFX9-NEXT: v_sub_co_u32_e32 v4, vcc, v6, v21390; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v7, v3, vcc1391; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc1392; GFX9-NEXT: s_waitcnt vmcnt(0)1393; GFX9-NEXT: buffer_wbinvl1_vol1394; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]1395; GFX9-NEXT: v_mov_b32_e32 v7, v51396; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1397; GFX9-NEXT: v_mov_b32_e32 v6, v41398; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]1399; GFX9-NEXT: s_cbranch_execnz .LBB30_11400; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end1401; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]1402; GFX9-NEXT: s_setpc_b64 s[30:31]1403 %tmp0 = atomicrmw sub ptr addrspace(1) %ptr, i64 %in seq_cst1404 ret void1405}1406 1407define void @global_atomic_sub_i64_noret_offset(ptr addrspace(1) %out, i64 %in) {1408; SI-LABEL: global_atomic_sub_i64_noret_offset:1409; SI: ; %bb.0:1410; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1411; SI-NEXT: s_mov_b32 s6, 01412; SI-NEXT: s_mov_b32 s7, 0xf0001413; SI-NEXT: s_mov_b32 s4, s61414; SI-NEXT: s_mov_b32 s5, s61415; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:321416; SI-NEXT: s_mov_b64 s[8:9], 01417; SI-NEXT: .LBB31_1: ; %atomicrmw.start1418; SI-NEXT: ; =>This Inner Loop Header: Depth=11419; SI-NEXT: s_waitcnt vmcnt(0)1420; SI-NEXT: v_sub_i32_e32 v4, vcc, v6, v21421; SI-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc1422; SI-NEXT: s_waitcnt expcnt(0)1423; SI-NEXT: v_mov_b32_e32 v11, v71424; SI-NEXT: v_mov_b32_e32 v10, v61425; SI-NEXT: v_mov_b32_e32 v9, v51426; SI-NEXT: v_mov_b32_e32 v8, v41427; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32 glc1428; SI-NEXT: s_waitcnt vmcnt(0)1429; SI-NEXT: buffer_wbinvl11430; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]1431; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]1432; SI-NEXT: v_mov_b32_e32 v6, v81433; SI-NEXT: v_mov_b32_e32 v7, v91434; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]1435; SI-NEXT: s_cbranch_execnz .LBB31_11436; SI-NEXT: ; %bb.2: ; %atomicrmw.end1437; SI-NEXT: s_or_b64 exec, exec, s[8:9]1438; SI-NEXT: s_waitcnt expcnt(0)1439; SI-NEXT: s_setpc_b64 s[30:31]1440;1441; VI-LABEL: global_atomic_sub_i64_noret_offset:1442; VI: ; %bb.0:1443; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1444; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v01445; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1446; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]1447; VI-NEXT: s_mov_b64 s[4:5], 01448; VI-NEXT: .LBB31_1: ; %atomicrmw.start1449; VI-NEXT: ; =>This Inner Loop Header: Depth=11450; VI-NEXT: s_waitcnt vmcnt(0)1451; VI-NEXT: v_sub_u32_e32 v4, vcc, v6, v21452; VI-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc1453; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc1454; VI-NEXT: s_waitcnt vmcnt(0)1455; VI-NEXT: buffer_wbinvl1_vol1456; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]1457; VI-NEXT: v_mov_b32_e32 v7, v51458; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1459; VI-NEXT: v_mov_b32_e32 v6, v41460; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]1461; VI-NEXT: s_cbranch_execnz .LBB31_11462; VI-NEXT: ; %bb.2: ; %atomicrmw.end1463; VI-NEXT: s_or_b64 exec, exec, s[4:5]1464; VI-NEXT: s_setpc_b64 s[30:31]1465;1466; GFX9-LABEL: global_atomic_sub_i64_noret_offset:1467; GFX9: ; %bb.0:1468; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1469; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:321470; GFX9-NEXT: s_mov_b64 s[4:5], 01471; GFX9-NEXT: .LBB31_1: ; %atomicrmw.start1472; GFX9-NEXT: ; =>This Inner Loop Header: Depth=11473; GFX9-NEXT: s_waitcnt vmcnt(0)1474; GFX9-NEXT: v_sub_co_u32_e32 v4, vcc, v6, v21475; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v7, v3, vcc1476; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc1477; GFX9-NEXT: s_waitcnt vmcnt(0)1478; GFX9-NEXT: buffer_wbinvl1_vol1479; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]1480; GFX9-NEXT: v_mov_b32_e32 v7, v51481; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1482; GFX9-NEXT: v_mov_b32_e32 v6, v41483; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]1484; GFX9-NEXT: s_cbranch_execnz .LBB31_11485; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end1486; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]1487; GFX9-NEXT: s_setpc_b64 s[30:31]1488 %gep = getelementptr i64, ptr addrspace(1) %out, i64 41489 %tmp0 = atomicrmw sub ptr addrspace(1) %gep, i64 %in seq_cst1490 ret void1491}1492 1493define i64 @global_atomic_sub_i64_ret(ptr addrspace(1) %ptr, i64 %in) {1494; SI-LABEL: global_atomic_sub_i64_ret:1495; SI: ; %bb.0:1496; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1497; SI-NEXT: v_mov_b32_e32 v6, v31498; SI-NEXT: v_mov_b32_e32 v7, v21499; SI-NEXT: v_mov_b32_e32 v5, v11500; SI-NEXT: v_mov_b32_e32 v4, v01501; SI-NEXT: s_mov_b32 s6, 01502; SI-NEXT: s_mov_b32 s7, 0xf0001503; SI-NEXT: s_mov_b32 s4, s61504; SI-NEXT: s_mov_b32 s5, s61505; SI-NEXT: buffer_load_dwordx2 v[10:11], v[4:5], s[4:7], 0 addr641506; SI-NEXT: s_mov_b64 s[8:9], 01507; SI-NEXT: .LBB32_1: ; %atomicrmw.start1508; SI-NEXT: ; =>This Inner Loop Header: Depth=11509; SI-NEXT: s_waitcnt vmcnt(0)1510; SI-NEXT: v_sub_i32_e32 v8, vcc, v10, v71511; SI-NEXT: v_subb_u32_e32 v9, vcc, v11, v6, vcc1512; SI-NEXT: s_waitcnt expcnt(0)1513; SI-NEXT: v_mov_b32_e32 v0, v81514; SI-NEXT: v_mov_b32_e32 v1, v91515; SI-NEXT: v_mov_b32_e32 v2, v101516; SI-NEXT: v_mov_b32_e32 v3, v111517; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 glc1518; SI-NEXT: s_waitcnt vmcnt(0)1519; SI-NEXT: buffer_wbinvl11520; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]1521; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]1522; SI-NEXT: v_mov_b32_e32 v11, v11523; SI-NEXT: v_mov_b32_e32 v10, v01524; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]1525; SI-NEXT: s_cbranch_execnz .LBB32_11526; SI-NEXT: ; %bb.2: ; %atomicrmw.end1527; SI-NEXT: s_or_b64 exec, exec, s[8:9]1528; SI-NEXT: s_waitcnt expcnt(0)1529; SI-NEXT: s_setpc_b64 s[30:31]1530;1531; VI-LABEL: global_atomic_sub_i64_ret:1532; VI: ; %bb.0:1533; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1534; VI-NEXT: flat_load_dwordx2 v[4:5], v[0:1]1535; VI-NEXT: s_mov_b64 s[4:5], 01536; VI-NEXT: .LBB32_1: ; %atomicrmw.start1537; VI-NEXT: ; =>This Inner Loop Header: Depth=11538; VI-NEXT: s_waitcnt vmcnt(0)1539; VI-NEXT: v_mov_b32_e32 v7, v51540; VI-NEXT: v_mov_b32_e32 v6, v41541; VI-NEXT: v_sub_u32_e32 v4, vcc, v6, v21542; VI-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc1543; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc1544; VI-NEXT: s_waitcnt vmcnt(0)1545; VI-NEXT: buffer_wbinvl1_vol1546; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]1547; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1548; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]1549; VI-NEXT: s_cbranch_execnz .LBB32_11550; VI-NEXT: ; %bb.2: ; %atomicrmw.end1551; VI-NEXT: s_or_b64 exec, exec, s[4:5]1552; VI-NEXT: v_mov_b32_e32 v0, v41553; VI-NEXT: v_mov_b32_e32 v1, v51554; VI-NEXT: s_setpc_b64 s[30:31]1555;1556; GFX9-LABEL: global_atomic_sub_i64_ret:1557; GFX9: ; %bb.0:1558; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1559; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off1560; GFX9-NEXT: s_mov_b64 s[4:5], 01561; GFX9-NEXT: .LBB32_1: ; %atomicrmw.start1562; GFX9-NEXT: ; =>This Inner Loop Header: Depth=11563; GFX9-NEXT: s_waitcnt vmcnt(0)1564; GFX9-NEXT: v_mov_b32_e32 v7, v51565; GFX9-NEXT: v_mov_b32_e32 v6, v41566; GFX9-NEXT: v_sub_co_u32_e32 v4, vcc, v6, v21567; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v7, v3, vcc1568; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc1569; GFX9-NEXT: s_waitcnt vmcnt(0)1570; GFX9-NEXT: buffer_wbinvl1_vol1571; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]1572; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1573; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]1574; GFX9-NEXT: s_cbranch_execnz .LBB32_11575; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end1576; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]1577; GFX9-NEXT: v_mov_b32_e32 v0, v41578; GFX9-NEXT: v_mov_b32_e32 v1, v51579; GFX9-NEXT: s_setpc_b64 s[30:31]1580 %result = atomicrmw sub ptr addrspace(1) %ptr, i64 %in seq_cst1581 ret i64 %result1582}1583 1584define i64 @global_atomic_sub_i64_ret_offset(ptr addrspace(1) %out, i64 %in) {1585; SI-LABEL: global_atomic_sub_i64_ret_offset:1586; SI: ; %bb.0:1587; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1588; SI-NEXT: v_mov_b32_e32 v6, v31589; SI-NEXT: v_mov_b32_e32 v7, v21590; SI-NEXT: v_mov_b32_e32 v5, v11591; SI-NEXT: v_mov_b32_e32 v4, v01592; SI-NEXT: s_mov_b32 s6, 01593; SI-NEXT: s_mov_b32 s7, 0xf0001594; SI-NEXT: s_mov_b32 s4, s61595; SI-NEXT: s_mov_b32 s5, s61596; SI-NEXT: buffer_load_dwordx2 v[10:11], v[4:5], s[4:7], 0 addr64 offset:321597; SI-NEXT: s_mov_b64 s[8:9], 01598; SI-NEXT: .LBB33_1: ; %atomicrmw.start1599; SI-NEXT: ; =>This Inner Loop Header: Depth=11600; SI-NEXT: s_waitcnt vmcnt(0)1601; SI-NEXT: v_sub_i32_e32 v8, vcc, v10, v71602; SI-NEXT: v_subb_u32_e32 v9, vcc, v11, v6, vcc1603; SI-NEXT: s_waitcnt expcnt(0)1604; SI-NEXT: v_mov_b32_e32 v0, v81605; SI-NEXT: v_mov_b32_e32 v1, v91606; SI-NEXT: v_mov_b32_e32 v2, v101607; SI-NEXT: v_mov_b32_e32 v3, v111608; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 offset:32 glc1609; SI-NEXT: s_waitcnt vmcnt(0)1610; SI-NEXT: buffer_wbinvl11611; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]1612; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]1613; SI-NEXT: v_mov_b32_e32 v11, v11614; SI-NEXT: v_mov_b32_e32 v10, v01615; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]1616; SI-NEXT: s_cbranch_execnz .LBB33_11617; SI-NEXT: ; %bb.2: ; %atomicrmw.end1618; SI-NEXT: s_or_b64 exec, exec, s[8:9]1619; SI-NEXT: s_waitcnt expcnt(0)1620; SI-NEXT: s_setpc_b64 s[30:31]1621;1622; VI-LABEL: global_atomic_sub_i64_ret_offset:1623; VI: ; %bb.0:1624; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1625; VI-NEXT: v_add_u32_e32 v4, vcc, 32, v01626; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc1627; VI-NEXT: flat_load_dwordx2 v[0:1], v[4:5]1628; VI-NEXT: s_mov_b64 s[4:5], 01629; VI-NEXT: .LBB33_1: ; %atomicrmw.start1630; VI-NEXT: ; =>This Inner Loop Header: Depth=11631; VI-NEXT: s_waitcnt vmcnt(0)1632; VI-NEXT: v_mov_b32_e32 v9, v11633; VI-NEXT: v_mov_b32_e32 v8, v01634; VI-NEXT: v_sub_u32_e32 v6, vcc, v8, v21635; VI-NEXT: v_subb_u32_e32 v7, vcc, v9, v3, vcc1636; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc1637; VI-NEXT: s_waitcnt vmcnt(0)1638; VI-NEXT: buffer_wbinvl1_vol1639; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]1640; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1641; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]1642; VI-NEXT: s_cbranch_execnz .LBB33_11643; VI-NEXT: ; %bb.2: ; %atomicrmw.end1644; VI-NEXT: s_or_b64 exec, exec, s[4:5]1645; VI-NEXT: s_setpc_b64 s[30:31]1646;1647; GFX9-LABEL: global_atomic_sub_i64_ret_offset:1648; GFX9: ; %bb.0:1649; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1650; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:321651; GFX9-NEXT: s_mov_b64 s[4:5], 01652; GFX9-NEXT: .LBB33_1: ; %atomicrmw.start1653; GFX9-NEXT: ; =>This Inner Loop Header: Depth=11654; GFX9-NEXT: s_waitcnt vmcnt(0)1655; GFX9-NEXT: v_mov_b32_e32 v7, v51656; GFX9-NEXT: v_mov_b32_e32 v6, v41657; GFX9-NEXT: v_sub_co_u32_e32 v4, vcc, v6, v21658; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v7, v3, vcc1659; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc1660; GFX9-NEXT: s_waitcnt vmcnt(0)1661; GFX9-NEXT: buffer_wbinvl1_vol1662; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]1663; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1664; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]1665; GFX9-NEXT: s_cbranch_execnz .LBB33_11666; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end1667; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]1668; GFX9-NEXT: v_mov_b32_e32 v0, v41669; GFX9-NEXT: v_mov_b32_e32 v1, v51670; GFX9-NEXT: s_setpc_b64 s[30:31]1671 %gep = getelementptr i64, ptr addrspace(1) %out, i64 41672 %result = atomicrmw sub ptr addrspace(1) %gep, i64 %in seq_cst1673 ret i64 %result1674}1675 1676define amdgpu_gfx void @global_atomic_sub_i64_noret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {1677; SI-LABEL: global_atomic_sub_i64_noret_scalar:1678; SI: ; %bb.0:1679; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1680; SI-NEXT: s_xor_saveexec_b64 s[34:35], -11681; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 ; 4-byte Folded Spill1682; SI-NEXT: s_mov_b64 exec, s[34:35]1683; SI-NEXT: s_waitcnt expcnt(0)1684; SI-NEXT: v_writelane_b32 v9, s6, 01685; SI-NEXT: v_writelane_b32 v9, s7, 11686; SI-NEXT: s_mov_b32 s35, s71687; SI-NEXT: s_mov_b32 s34, s61688; SI-NEXT: s_mov_b32 s7, 0xf0001689; SI-NEXT: s_mov_b32 s6, -11690; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 01691; SI-NEXT: s_mov_b64 s[36:37], 01692; SI-NEXT: v_mov_b32_e32 v4, s351693; SI-NEXT: .LBB34_1: ; %atomicrmw.start1694; SI-NEXT: ; =>This Inner Loop Header: Depth=11695; SI-NEXT: s_waitcnt vmcnt(0)1696; SI-NEXT: v_subrev_i32_e32 v0, vcc, s34, v21697; SI-NEXT: v_subb_u32_e32 v1, vcc, v3, v4, vcc1698; SI-NEXT: s_waitcnt expcnt(0)1699; SI-NEXT: v_mov_b32_e32 v8, v31700; SI-NEXT: v_mov_b32_e32 v7, v21701; SI-NEXT: v_mov_b32_e32 v6, v11702; SI-NEXT: v_mov_b32_e32 v5, v01703; SI-NEXT: buffer_atomic_cmpswap_x2 v[5:8], off, s[4:7], 0 glc1704; SI-NEXT: s_waitcnt vmcnt(0)1705; SI-NEXT: buffer_wbinvl11706; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[5:6], v[2:3]1707; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]1708; SI-NEXT: v_mov_b32_e32 v2, v51709; SI-NEXT: v_mov_b32_e32 v3, v61710; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]1711; SI-NEXT: s_cbranch_execnz .LBB34_11712; SI-NEXT: ; %bb.2: ; %atomicrmw.end1713; SI-NEXT: s_or_b64 exec, exec, s[36:37]1714; SI-NEXT: v_readlane_b32 s7, v9, 11715; SI-NEXT: v_readlane_b32 s6, v9, 01716; SI-NEXT: s_xor_saveexec_b64 s[34:35], -11717; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 ; 4-byte Folded Reload1718; SI-NEXT: s_mov_b64 exec, s[34:35]1719; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)1720; SI-NEXT: s_setpc_b64 s[30:31]1721;1722; VI-LABEL: global_atomic_sub_i64_noret_scalar:1723; VI: ; %bb.0:1724; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1725; VI-NEXT: v_mov_b32_e32 v0, s41726; VI-NEXT: v_mov_b32_e32 v1, s51727; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]1728; VI-NEXT: v_mov_b32_e32 v4, s41729; VI-NEXT: s_mov_b64 s[34:35], 01730; VI-NEXT: v_mov_b32_e32 v6, s71731; VI-NEXT: v_mov_b32_e32 v5, s51732; VI-NEXT: .LBB34_1: ; %atomicrmw.start1733; VI-NEXT: ; =>This Inner Loop Header: Depth=11734; VI-NEXT: s_waitcnt vmcnt(0)1735; VI-NEXT: v_subrev_u32_e32 v0, vcc, s6, v21736; VI-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc1737; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc1738; VI-NEXT: s_waitcnt vmcnt(0)1739; VI-NEXT: buffer_wbinvl1_vol1740; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]1741; VI-NEXT: v_mov_b32_e32 v3, v11742; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]1743; VI-NEXT: v_mov_b32_e32 v2, v01744; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]1745; VI-NEXT: s_cbranch_execnz .LBB34_11746; VI-NEXT: ; %bb.2: ; %atomicrmw.end1747; VI-NEXT: s_or_b64 exec, exec, s[34:35]1748; VI-NEXT: s_setpc_b64 s[30:31]1749;1750; GFX9-LABEL: global_atomic_sub_i64_noret_scalar:1751; GFX9: ; %bb.0:1752; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1753; GFX9-NEXT: v_mov_b32_e32 v4, 01754; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5]1755; GFX9-NEXT: s_mov_b64 s[34:35], 01756; GFX9-NEXT: v_mov_b32_e32 v5, s71757; GFX9-NEXT: .LBB34_1: ; %atomicrmw.start1758; GFX9-NEXT: ; =>This Inner Loop Header: Depth=11759; GFX9-NEXT: s_waitcnt vmcnt(0)1760; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, s6, v21761; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc1762; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc1763; GFX9-NEXT: s_waitcnt vmcnt(0)1764; GFX9-NEXT: buffer_wbinvl1_vol1765; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]1766; GFX9-NEXT: v_mov_b32_e32 v3, v11767; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]1768; GFX9-NEXT: v_mov_b32_e32 v2, v01769; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]1770; GFX9-NEXT: s_cbranch_execnz .LBB34_11771; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end1772; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]1773; GFX9-NEXT: s_setpc_b64 s[30:31]1774 %tmp0 = atomicrmw sub ptr addrspace(1) %ptr, i64 %in seq_cst1775 ret void1776}1777 1778define amdgpu_gfx void @global_atomic_sub_i64_noret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {1779; SI-LABEL: global_atomic_sub_i64_noret_offset_scalar:1780; SI: ; %bb.0:1781; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1782; SI-NEXT: s_xor_saveexec_b64 s[34:35], -11783; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 ; 4-byte Folded Spill1784; SI-NEXT: s_mov_b64 exec, s[34:35]1785; SI-NEXT: s_waitcnt expcnt(0)1786; SI-NEXT: v_writelane_b32 v9, s6, 01787; SI-NEXT: v_writelane_b32 v9, s7, 11788; SI-NEXT: s_mov_b32 s35, s71789; SI-NEXT: s_mov_b32 s34, s61790; SI-NEXT: s_mov_b32 s7, 0xf0001791; SI-NEXT: s_mov_b32 s6, -11792; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:321793; SI-NEXT: s_mov_b64 s[36:37], 01794; SI-NEXT: v_mov_b32_e32 v4, s351795; SI-NEXT: .LBB35_1: ; %atomicrmw.start1796; SI-NEXT: ; =>This Inner Loop Header: Depth=11797; SI-NEXT: s_waitcnt vmcnt(0)1798; SI-NEXT: v_subrev_i32_e32 v0, vcc, s34, v21799; SI-NEXT: v_subb_u32_e32 v1, vcc, v3, v4, vcc1800; SI-NEXT: s_waitcnt expcnt(0)1801; SI-NEXT: v_mov_b32_e32 v8, v31802; SI-NEXT: v_mov_b32_e32 v7, v21803; SI-NEXT: v_mov_b32_e32 v6, v11804; SI-NEXT: v_mov_b32_e32 v5, v01805; SI-NEXT: buffer_atomic_cmpswap_x2 v[5:8], off, s[4:7], 0 offset:32 glc1806; SI-NEXT: s_waitcnt vmcnt(0)1807; SI-NEXT: buffer_wbinvl11808; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[5:6], v[2:3]1809; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]1810; SI-NEXT: v_mov_b32_e32 v2, v51811; SI-NEXT: v_mov_b32_e32 v3, v61812; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]1813; SI-NEXT: s_cbranch_execnz .LBB35_11814; SI-NEXT: ; %bb.2: ; %atomicrmw.end1815; SI-NEXT: s_or_b64 exec, exec, s[36:37]1816; SI-NEXT: v_readlane_b32 s7, v9, 11817; SI-NEXT: v_readlane_b32 s6, v9, 01818; SI-NEXT: s_xor_saveexec_b64 s[34:35], -11819; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 ; 4-byte Folded Reload1820; SI-NEXT: s_mov_b64 exec, s[34:35]1821; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)1822; SI-NEXT: s_setpc_b64 s[30:31]1823;1824; VI-LABEL: global_atomic_sub_i64_noret_offset_scalar:1825; VI: ; %bb.0:1826; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1827; VI-NEXT: s_add_u32 s34, s4, 321828; VI-NEXT: s_addc_u32 s35, s5, 01829; VI-NEXT: v_mov_b32_e32 v4, s341830; VI-NEXT: v_mov_b32_e32 v5, s351831; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]1832; VI-NEXT: s_mov_b64 s[34:35], 01833; VI-NEXT: v_mov_b32_e32 v6, s71834; VI-NEXT: .LBB35_1: ; %atomicrmw.start1835; VI-NEXT: ; =>This Inner Loop Header: Depth=11836; VI-NEXT: s_waitcnt vmcnt(0)1837; VI-NEXT: v_subrev_u32_e32 v0, vcc, s6, v21838; VI-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc1839; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc1840; VI-NEXT: s_waitcnt vmcnt(0)1841; VI-NEXT: buffer_wbinvl1_vol1842; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]1843; VI-NEXT: v_mov_b32_e32 v3, v11844; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]1845; VI-NEXT: v_mov_b32_e32 v2, v01846; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]1847; VI-NEXT: s_cbranch_execnz .LBB35_11848; VI-NEXT: ; %bb.2: ; %atomicrmw.end1849; VI-NEXT: s_or_b64 exec, exec, s[34:35]1850; VI-NEXT: s_setpc_b64 s[30:31]1851;1852; GFX9-LABEL: global_atomic_sub_i64_noret_offset_scalar:1853; GFX9: ; %bb.0:1854; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1855; GFX9-NEXT: v_mov_b32_e32 v4, 01856; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5] offset:321857; GFX9-NEXT: s_mov_b64 s[34:35], 01858; GFX9-NEXT: v_mov_b32_e32 v5, s71859; GFX9-NEXT: .LBB35_1: ; %atomicrmw.start1860; GFX9-NEXT: ; =>This Inner Loop Header: Depth=11861; GFX9-NEXT: s_waitcnt vmcnt(0)1862; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, s6, v21863; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc1864; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc1865; GFX9-NEXT: s_waitcnt vmcnt(0)1866; GFX9-NEXT: buffer_wbinvl1_vol1867; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]1868; GFX9-NEXT: v_mov_b32_e32 v3, v11869; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]1870; GFX9-NEXT: v_mov_b32_e32 v2, v01871; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]1872; GFX9-NEXT: s_cbranch_execnz .LBB35_11873; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end1874; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]1875; GFX9-NEXT: s_setpc_b64 s[30:31]1876 %gep = getelementptr i64, ptr addrspace(1) %out, i64 41877 %tmp0 = atomicrmw sub ptr addrspace(1) %gep, i64 %in seq_cst1878 ret void1879}1880 1881define amdgpu_gfx i64 @global_atomic_sub_i64_ret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {1882; SI-LABEL: global_atomic_sub_i64_ret_scalar:1883; SI: ; %bb.0:1884; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1885; SI-NEXT: s_xor_saveexec_b64 s[34:35], -11886; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 ; 4-byte Folded Spill1887; SI-NEXT: s_mov_b64 exec, s[34:35]1888; SI-NEXT: s_waitcnt expcnt(0)1889; SI-NEXT: v_writelane_b32 v7, s6, 01890; SI-NEXT: v_writelane_b32 v7, s7, 11891; SI-NEXT: s_mov_b32 s35, s71892; SI-NEXT: s_mov_b32 s34, s61893; SI-NEXT: s_mov_b32 s7, 0xf0001894; SI-NEXT: s_mov_b32 s6, -11895; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 01896; SI-NEXT: s_mov_b64 s[36:37], 01897; SI-NEXT: v_mov_b32_e32 v6, s351898; SI-NEXT: .LBB36_1: ; %atomicrmw.start1899; SI-NEXT: ; =>This Inner Loop Header: Depth=11900; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)1901; SI-NEXT: v_subrev_i32_e32 v2, vcc, s34, v41902; SI-NEXT: v_subb_u32_e32 v3, vcc, v5, v6, vcc1903; SI-NEXT: v_mov_b32_e32 v0, v21904; SI-NEXT: v_mov_b32_e32 v1, v31905; SI-NEXT: v_mov_b32_e32 v2, v41906; SI-NEXT: v_mov_b32_e32 v3, v51907; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc1908; SI-NEXT: s_waitcnt vmcnt(0)1909; SI-NEXT: buffer_wbinvl11910; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]1911; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]1912; SI-NEXT: v_mov_b32_e32 v5, v11913; SI-NEXT: v_mov_b32_e32 v4, v01914; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]1915; SI-NEXT: s_cbranch_execnz .LBB36_11916; SI-NEXT: ; %bb.2: ; %atomicrmw.end1917; SI-NEXT: s_or_b64 exec, exec, s[36:37]1918; SI-NEXT: v_readlane_b32 s7, v7, 11919; SI-NEXT: v_readlane_b32 s6, v7, 01920; SI-NEXT: s_xor_saveexec_b64 s[34:35], -11921; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload1922; SI-NEXT: s_mov_b64 exec, s[34:35]1923; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)1924; SI-NEXT: s_setpc_b64 s[30:31]1925;1926; VI-LABEL: global_atomic_sub_i64_ret_scalar:1927; VI: ; %bb.0:1928; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1929; VI-NEXT: v_mov_b32_e32 v0, s41930; VI-NEXT: v_mov_b32_e32 v1, s51931; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]1932; VI-NEXT: v_mov_b32_e32 v2, s41933; VI-NEXT: s_mov_b64 s[34:35], 01934; VI-NEXT: v_mov_b32_e32 v4, s71935; VI-NEXT: v_mov_b32_e32 v3, s51936; VI-NEXT: .LBB36_1: ; %atomicrmw.start1937; VI-NEXT: ; =>This Inner Loop Header: Depth=11938; VI-NEXT: s_waitcnt vmcnt(0)1939; VI-NEXT: v_mov_b32_e32 v8, v11940; VI-NEXT: v_mov_b32_e32 v7, v01941; VI-NEXT: v_subrev_u32_e32 v5, vcc, s6, v71942; VI-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc1943; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc1944; VI-NEXT: s_waitcnt vmcnt(0)1945; VI-NEXT: buffer_wbinvl1_vol1946; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]1947; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]1948; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]1949; VI-NEXT: s_cbranch_execnz .LBB36_11950; VI-NEXT: ; %bb.2: ; %atomicrmw.end1951; VI-NEXT: s_or_b64 exec, exec, s[34:35]1952; VI-NEXT: s_setpc_b64 s[30:31]1953;1954; GFX9-LABEL: global_atomic_sub_i64_ret_scalar:1955; GFX9: ; %bb.0:1956; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1957; GFX9-NEXT: v_mov_b32_e32 v2, 01958; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]1959; GFX9-NEXT: s_mov_b64 s[34:35], 01960; GFX9-NEXT: v_mov_b32_e32 v3, s71961; GFX9-NEXT: .LBB36_1: ; %atomicrmw.start1962; GFX9-NEXT: ; =>This Inner Loop Header: Depth=11963; GFX9-NEXT: s_waitcnt vmcnt(0)1964; GFX9-NEXT: v_mov_b32_e32 v7, v11965; GFX9-NEXT: v_mov_b32_e32 v6, v01966; GFX9-NEXT: v_subrev_co_u32_e32 v4, vcc, s6, v61967; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v7, v3, vcc1968; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[4:7], s[4:5] glc1969; GFX9-NEXT: s_waitcnt vmcnt(0)1970; GFX9-NEXT: buffer_wbinvl1_vol1971; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]1972; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]1973; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]1974; GFX9-NEXT: s_cbranch_execnz .LBB36_11975; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end1976; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]1977; GFX9-NEXT: s_setpc_b64 s[30:31]1978 %result = atomicrmw sub ptr addrspace(1) %ptr, i64 %in seq_cst1979 ret i64 %result1980}1981 1982define amdgpu_gfx i64 @global_atomic_sub_i64_ret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {1983; SI-LABEL: global_atomic_sub_i64_ret_offset_scalar:1984; SI: ; %bb.0:1985; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1986; SI-NEXT: s_xor_saveexec_b64 s[34:35], -11987; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 ; 4-byte Folded Spill1988; SI-NEXT: s_mov_b64 exec, s[34:35]1989; SI-NEXT: s_waitcnt expcnt(0)1990; SI-NEXT: v_writelane_b32 v7, s6, 01991; SI-NEXT: v_writelane_b32 v7, s7, 11992; SI-NEXT: s_mov_b32 s35, s71993; SI-NEXT: s_mov_b32 s34, s61994; SI-NEXT: s_mov_b32 s7, 0xf0001995; SI-NEXT: s_mov_b32 s6, -11996; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0 offset:321997; SI-NEXT: s_mov_b64 s[36:37], 01998; SI-NEXT: v_mov_b32_e32 v6, s351999; SI-NEXT: .LBB37_1: ; %atomicrmw.start2000; SI-NEXT: ; =>This Inner Loop Header: Depth=12001; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)2002; SI-NEXT: v_subrev_i32_e32 v2, vcc, s34, v42003; SI-NEXT: v_subb_u32_e32 v3, vcc, v5, v6, vcc2004; SI-NEXT: v_mov_b32_e32 v0, v22005; SI-NEXT: v_mov_b32_e32 v1, v32006; SI-NEXT: v_mov_b32_e32 v2, v42007; SI-NEXT: v_mov_b32_e32 v3, v52008; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 offset:32 glc2009; SI-NEXT: s_waitcnt vmcnt(0)2010; SI-NEXT: buffer_wbinvl12011; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]2012; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]2013; SI-NEXT: v_mov_b32_e32 v5, v12014; SI-NEXT: v_mov_b32_e32 v4, v02015; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]2016; SI-NEXT: s_cbranch_execnz .LBB37_12017; SI-NEXT: ; %bb.2: ; %atomicrmw.end2018; SI-NEXT: s_or_b64 exec, exec, s[36:37]2019; SI-NEXT: v_readlane_b32 s7, v7, 12020; SI-NEXT: v_readlane_b32 s6, v7, 02021; SI-NEXT: s_xor_saveexec_b64 s[34:35], -12022; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload2023; SI-NEXT: s_mov_b64 exec, s[34:35]2024; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)2025; SI-NEXT: s_setpc_b64 s[30:31]2026;2027; VI-LABEL: global_atomic_sub_i64_ret_offset_scalar:2028; VI: ; %bb.0:2029; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2030; VI-NEXT: s_add_u32 s34, s4, 322031; VI-NEXT: s_addc_u32 s35, s5, 02032; VI-NEXT: v_mov_b32_e32 v2, s342033; VI-NEXT: v_mov_b32_e32 v3, s352034; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]2035; VI-NEXT: s_mov_b64 s[34:35], 02036; VI-NEXT: v_mov_b32_e32 v4, s72037; VI-NEXT: .LBB37_1: ; %atomicrmw.start2038; VI-NEXT: ; =>This Inner Loop Header: Depth=12039; VI-NEXT: s_waitcnt vmcnt(0)2040; VI-NEXT: v_mov_b32_e32 v8, v12041; VI-NEXT: v_mov_b32_e32 v7, v02042; VI-NEXT: v_subrev_u32_e32 v5, vcc, s6, v72043; VI-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc2044; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc2045; VI-NEXT: s_waitcnt vmcnt(0)2046; VI-NEXT: buffer_wbinvl1_vol2047; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]2048; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]2049; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]2050; VI-NEXT: s_cbranch_execnz .LBB37_12051; VI-NEXT: ; %bb.2: ; %atomicrmw.end2052; VI-NEXT: s_or_b64 exec, exec, s[34:35]2053; VI-NEXT: s_setpc_b64 s[30:31]2054;2055; GFX9-LABEL: global_atomic_sub_i64_ret_offset_scalar:2056; GFX9: ; %bb.0:2057; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2058; GFX9-NEXT: v_mov_b32_e32 v2, 02059; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5] offset:322060; GFX9-NEXT: s_mov_b64 s[34:35], 02061; GFX9-NEXT: v_mov_b32_e32 v3, s72062; GFX9-NEXT: .LBB37_1: ; %atomicrmw.start2063; GFX9-NEXT: ; =>This Inner Loop Header: Depth=12064; GFX9-NEXT: s_waitcnt vmcnt(0)2065; GFX9-NEXT: v_mov_b32_e32 v7, v12066; GFX9-NEXT: v_mov_b32_e32 v6, v02067; GFX9-NEXT: v_subrev_co_u32_e32 v4, vcc, s6, v62068; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v7, v3, vcc2069; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[4:7], s[4:5] offset:32 glc2070; GFX9-NEXT: s_waitcnt vmcnt(0)2071; GFX9-NEXT: buffer_wbinvl1_vol2072; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]2073; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]2074; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]2075; GFX9-NEXT: s_cbranch_execnz .LBB37_12076; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end2077; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]2078; GFX9-NEXT: s_setpc_b64 s[30:31]2079 %gep = getelementptr i64, ptr addrspace(1) %out, i64 42080 %result = atomicrmw sub ptr addrspace(1) %gep, i64 %in seq_cst2081 ret i64 %result2082}2083 2084define void @global_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {2085; SI-LABEL: global_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory:2086; SI: ; %bb.0:2087; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2088; SI-NEXT: s_mov_b32 s6, 02089; SI-NEXT: s_mov_b32 s7, 0xf0002090; SI-NEXT: s_mov_b32 s4, s62091; SI-NEXT: s_mov_b32 s5, s62092; SI-NEXT: buffer_atomic_sub_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:322093; SI-NEXT: s_waitcnt vmcnt(0)2094; SI-NEXT: buffer_wbinvl12095; SI-NEXT: s_waitcnt expcnt(0)2096; SI-NEXT: s_setpc_b64 s[30:31]2097;2098; VI-LABEL: global_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory:2099; VI: ; %bb.0:2100; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2101; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v02102; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2103; VI-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]2104; VI-NEXT: s_waitcnt vmcnt(0)2105; VI-NEXT: buffer_wbinvl1_vol2106; VI-NEXT: s_setpc_b64 s[30:31]2107;2108; GFX9-LABEL: global_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory:2109; GFX9: ; %bb.0:2110; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2111; GFX9-NEXT: global_atomic_sub_x2 v[0:1], v[2:3], off offset:322112; GFX9-NEXT: s_waitcnt vmcnt(0)2113; GFX9-NEXT: buffer_wbinvl1_vol2114; GFX9-NEXT: s_setpc_b64 s[30:31]2115 %gep = getelementptr i64, ptr addrspace(1) %out, i64 42116 %tmp0 = atomicrmw sub ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !02117 ret void2118}2119 2120define i64 @global_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {2121; SI-LABEL: global_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory:2122; SI: ; %bb.0:2123; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2124; SI-NEXT: s_mov_b32 s6, 02125; SI-NEXT: s_mov_b32 s7, 0xf0002126; SI-NEXT: s_mov_b32 s4, s62127; SI-NEXT: s_mov_b32 s5, s62128; SI-NEXT: buffer_atomic_sub_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc2129; SI-NEXT: s_waitcnt vmcnt(0)2130; SI-NEXT: buffer_wbinvl12131; SI-NEXT: v_mov_b32_e32 v0, v22132; SI-NEXT: v_mov_b32_e32 v1, v32133; SI-NEXT: s_waitcnt expcnt(0)2134; SI-NEXT: s_setpc_b64 s[30:31]2135;2136; VI-LABEL: global_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory:2137; VI: ; %bb.0:2138; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2139; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v02140; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2141; VI-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc2142; VI-NEXT: s_waitcnt vmcnt(0)2143; VI-NEXT: buffer_wbinvl1_vol2144; VI-NEXT: s_setpc_b64 s[30:31]2145;2146; GFX9-LABEL: global_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory:2147; GFX9: ; %bb.0:2148; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2149; GFX9-NEXT: global_atomic_sub_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc2150; GFX9-NEXT: s_waitcnt vmcnt(0)2151; GFX9-NEXT: buffer_wbinvl1_vol2152; GFX9-NEXT: s_setpc_b64 s[30:31]2153 %gep = getelementptr i64, ptr addrspace(1) %out, i64 42154 %result = atomicrmw sub ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !02155 ret i64 %result2156}2157 2158; ---------------------------------------------------------------------2159; atomicrmw and2160; ---------------------------------------------------------------------2161 2162define void @global_atomic_and_i64_noret(ptr addrspace(1) %ptr, i64 %in) {2163; SI-LABEL: global_atomic_and_i64_noret:2164; SI: ; %bb.0:2165; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2166; SI-NEXT: s_mov_b32 s6, 02167; SI-NEXT: s_mov_b32 s7, 0xf0002168; SI-NEXT: s_mov_b32 s4, s62169; SI-NEXT: s_mov_b32 s5, s62170; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr642171; SI-NEXT: s_mov_b64 s[8:9], 02172; SI-NEXT: .LBB40_1: ; %atomicrmw.start2173; SI-NEXT: ; =>This Inner Loop Header: Depth=12174; SI-NEXT: s_waitcnt vmcnt(0)2175; SI-NEXT: v_and_b32_e32 v5, v7, v32176; SI-NEXT: v_and_b32_e32 v4, v6, v22177; SI-NEXT: s_waitcnt expcnt(0)2178; SI-NEXT: v_mov_b32_e32 v11, v72179; SI-NEXT: v_mov_b32_e32 v10, v62180; SI-NEXT: v_mov_b32_e32 v9, v52181; SI-NEXT: v_mov_b32_e32 v8, v42182; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc2183; SI-NEXT: s_waitcnt vmcnt(0)2184; SI-NEXT: buffer_wbinvl12185; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]2186; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]2187; SI-NEXT: v_mov_b32_e32 v6, v82188; SI-NEXT: v_mov_b32_e32 v7, v92189; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]2190; SI-NEXT: s_cbranch_execnz .LBB40_12191; SI-NEXT: ; %bb.2: ; %atomicrmw.end2192; SI-NEXT: s_or_b64 exec, exec, s[8:9]2193; SI-NEXT: s_waitcnt expcnt(0)2194; SI-NEXT: s_setpc_b64 s[30:31]2195;2196; VI-LABEL: global_atomic_and_i64_noret:2197; VI: ; %bb.0:2198; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2199; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]2200; VI-NEXT: s_mov_b64 s[4:5], 02201; VI-NEXT: .LBB40_1: ; %atomicrmw.start2202; VI-NEXT: ; =>This Inner Loop Header: Depth=12203; VI-NEXT: s_waitcnt vmcnt(0)2204; VI-NEXT: v_and_b32_e32 v5, v7, v32205; VI-NEXT: v_and_b32_e32 v4, v6, v22206; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc2207; VI-NEXT: s_waitcnt vmcnt(0)2208; VI-NEXT: buffer_wbinvl1_vol2209; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]2210; VI-NEXT: v_mov_b32_e32 v7, v52211; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2212; VI-NEXT: v_mov_b32_e32 v6, v42213; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]2214; VI-NEXT: s_cbranch_execnz .LBB40_12215; VI-NEXT: ; %bb.2: ; %atomicrmw.end2216; VI-NEXT: s_or_b64 exec, exec, s[4:5]2217; VI-NEXT: s_setpc_b64 s[30:31]2218;2219; GFX9-LABEL: global_atomic_and_i64_noret:2220; GFX9: ; %bb.0:2221; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2222; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off2223; GFX9-NEXT: s_mov_b64 s[4:5], 02224; GFX9-NEXT: .LBB40_1: ; %atomicrmw.start2225; GFX9-NEXT: ; =>This Inner Loop Header: Depth=12226; GFX9-NEXT: s_waitcnt vmcnt(0)2227; GFX9-NEXT: v_and_b32_e32 v5, v7, v32228; GFX9-NEXT: v_and_b32_e32 v4, v6, v22229; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc2230; GFX9-NEXT: s_waitcnt vmcnt(0)2231; GFX9-NEXT: buffer_wbinvl1_vol2232; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]2233; GFX9-NEXT: v_mov_b32_e32 v7, v52234; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2235; GFX9-NEXT: v_mov_b32_e32 v6, v42236; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]2237; GFX9-NEXT: s_cbranch_execnz .LBB40_12238; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end2239; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]2240; GFX9-NEXT: s_setpc_b64 s[30:31]2241 %tmp0 = atomicrmw and ptr addrspace(1) %ptr, i64 %in seq_cst2242 ret void2243}2244 2245define void @global_atomic_and_i64_noret_offset(ptr addrspace(1) %out, i64 %in) {2246; SI-LABEL: global_atomic_and_i64_noret_offset:2247; SI: ; %bb.0:2248; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2249; SI-NEXT: s_mov_b32 s6, 02250; SI-NEXT: s_mov_b32 s7, 0xf0002251; SI-NEXT: s_mov_b32 s4, s62252; SI-NEXT: s_mov_b32 s5, s62253; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:322254; SI-NEXT: s_mov_b64 s[8:9], 02255; SI-NEXT: .LBB41_1: ; %atomicrmw.start2256; SI-NEXT: ; =>This Inner Loop Header: Depth=12257; SI-NEXT: s_waitcnt vmcnt(0)2258; SI-NEXT: v_and_b32_e32 v5, v7, v32259; SI-NEXT: v_and_b32_e32 v4, v6, v22260; SI-NEXT: s_waitcnt expcnt(0)2261; SI-NEXT: v_mov_b32_e32 v11, v72262; SI-NEXT: v_mov_b32_e32 v10, v62263; SI-NEXT: v_mov_b32_e32 v9, v52264; SI-NEXT: v_mov_b32_e32 v8, v42265; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32 glc2266; SI-NEXT: s_waitcnt vmcnt(0)2267; SI-NEXT: buffer_wbinvl12268; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]2269; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]2270; SI-NEXT: v_mov_b32_e32 v6, v82271; SI-NEXT: v_mov_b32_e32 v7, v92272; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]2273; SI-NEXT: s_cbranch_execnz .LBB41_12274; SI-NEXT: ; %bb.2: ; %atomicrmw.end2275; SI-NEXT: s_or_b64 exec, exec, s[8:9]2276; SI-NEXT: s_waitcnt expcnt(0)2277; SI-NEXT: s_setpc_b64 s[30:31]2278;2279; VI-LABEL: global_atomic_and_i64_noret_offset:2280; VI: ; %bb.0:2281; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2282; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v02283; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2284; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]2285; VI-NEXT: s_mov_b64 s[4:5], 02286; VI-NEXT: .LBB41_1: ; %atomicrmw.start2287; VI-NEXT: ; =>This Inner Loop Header: Depth=12288; VI-NEXT: s_waitcnt vmcnt(0)2289; VI-NEXT: v_and_b32_e32 v5, v7, v32290; VI-NEXT: v_and_b32_e32 v4, v6, v22291; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc2292; VI-NEXT: s_waitcnt vmcnt(0)2293; VI-NEXT: buffer_wbinvl1_vol2294; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]2295; VI-NEXT: v_mov_b32_e32 v7, v52296; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2297; VI-NEXT: v_mov_b32_e32 v6, v42298; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]2299; VI-NEXT: s_cbranch_execnz .LBB41_12300; VI-NEXT: ; %bb.2: ; %atomicrmw.end2301; VI-NEXT: s_or_b64 exec, exec, s[4:5]2302; VI-NEXT: s_setpc_b64 s[30:31]2303;2304; GFX9-LABEL: global_atomic_and_i64_noret_offset:2305; GFX9: ; %bb.0:2306; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2307; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:322308; GFX9-NEXT: s_mov_b64 s[4:5], 02309; GFX9-NEXT: .LBB41_1: ; %atomicrmw.start2310; GFX9-NEXT: ; =>This Inner Loop Header: Depth=12311; GFX9-NEXT: s_waitcnt vmcnt(0)2312; GFX9-NEXT: v_and_b32_e32 v5, v7, v32313; GFX9-NEXT: v_and_b32_e32 v4, v6, v22314; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc2315; GFX9-NEXT: s_waitcnt vmcnt(0)2316; GFX9-NEXT: buffer_wbinvl1_vol2317; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]2318; GFX9-NEXT: v_mov_b32_e32 v7, v52319; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2320; GFX9-NEXT: v_mov_b32_e32 v6, v42321; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]2322; GFX9-NEXT: s_cbranch_execnz .LBB41_12323; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end2324; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]2325; GFX9-NEXT: s_setpc_b64 s[30:31]2326 %gep = getelementptr i64, ptr addrspace(1) %out, i64 42327 %tmp0 = atomicrmw and ptr addrspace(1) %gep, i64 %in seq_cst2328 ret void2329}2330 2331define i64 @global_atomic_and_i64_ret(ptr addrspace(1) %ptr, i64 %in) {2332; SI-LABEL: global_atomic_and_i64_ret:2333; SI: ; %bb.0:2334; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2335; SI-NEXT: v_mov_b32_e32 v6, v32336; SI-NEXT: v_mov_b32_e32 v7, v22337; SI-NEXT: v_mov_b32_e32 v5, v12338; SI-NEXT: v_mov_b32_e32 v4, v02339; SI-NEXT: s_mov_b32 s6, 02340; SI-NEXT: s_mov_b32 s7, 0xf0002341; SI-NEXT: s_mov_b32 s4, s62342; SI-NEXT: s_mov_b32 s5, s62343; SI-NEXT: buffer_load_dwordx2 v[10:11], v[4:5], s[4:7], 0 addr642344; SI-NEXT: s_mov_b64 s[8:9], 02345; SI-NEXT: .LBB42_1: ; %atomicrmw.start2346; SI-NEXT: ; =>This Inner Loop Header: Depth=12347; SI-NEXT: s_waitcnt vmcnt(0)2348; SI-NEXT: v_and_b32_e32 v9, v11, v62349; SI-NEXT: v_and_b32_e32 v8, v10, v72350; SI-NEXT: s_waitcnt expcnt(0)2351; SI-NEXT: v_mov_b32_e32 v0, v82352; SI-NEXT: v_mov_b32_e32 v1, v92353; SI-NEXT: v_mov_b32_e32 v2, v102354; SI-NEXT: v_mov_b32_e32 v3, v112355; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 glc2356; SI-NEXT: s_waitcnt vmcnt(0)2357; SI-NEXT: buffer_wbinvl12358; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]2359; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]2360; SI-NEXT: v_mov_b32_e32 v11, v12361; SI-NEXT: v_mov_b32_e32 v10, v02362; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]2363; SI-NEXT: s_cbranch_execnz .LBB42_12364; SI-NEXT: ; %bb.2: ; %atomicrmw.end2365; SI-NEXT: s_or_b64 exec, exec, s[8:9]2366; SI-NEXT: s_waitcnt expcnt(0)2367; SI-NEXT: s_setpc_b64 s[30:31]2368;2369; VI-LABEL: global_atomic_and_i64_ret:2370; VI: ; %bb.0:2371; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2372; VI-NEXT: flat_load_dwordx2 v[4:5], v[0:1]2373; VI-NEXT: s_mov_b64 s[4:5], 02374; VI-NEXT: .LBB42_1: ; %atomicrmw.start2375; VI-NEXT: ; =>This Inner Loop Header: Depth=12376; VI-NEXT: s_waitcnt vmcnt(0)2377; VI-NEXT: v_mov_b32_e32 v7, v52378; VI-NEXT: v_mov_b32_e32 v6, v42379; VI-NEXT: v_and_b32_e32 v5, v7, v32380; VI-NEXT: v_and_b32_e32 v4, v6, v22381; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc2382; VI-NEXT: s_waitcnt vmcnt(0)2383; VI-NEXT: buffer_wbinvl1_vol2384; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]2385; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2386; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]2387; VI-NEXT: s_cbranch_execnz .LBB42_12388; VI-NEXT: ; %bb.2: ; %atomicrmw.end2389; VI-NEXT: s_or_b64 exec, exec, s[4:5]2390; VI-NEXT: v_mov_b32_e32 v0, v42391; VI-NEXT: v_mov_b32_e32 v1, v52392; VI-NEXT: s_setpc_b64 s[30:31]2393;2394; GFX9-LABEL: global_atomic_and_i64_ret:2395; GFX9: ; %bb.0:2396; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2397; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off2398; GFX9-NEXT: s_mov_b64 s[4:5], 02399; GFX9-NEXT: .LBB42_1: ; %atomicrmw.start2400; GFX9-NEXT: ; =>This Inner Loop Header: Depth=12401; GFX9-NEXT: s_waitcnt vmcnt(0)2402; GFX9-NEXT: v_mov_b32_e32 v7, v52403; GFX9-NEXT: v_mov_b32_e32 v6, v42404; GFX9-NEXT: v_and_b32_e32 v5, v7, v32405; GFX9-NEXT: v_and_b32_e32 v4, v6, v22406; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc2407; GFX9-NEXT: s_waitcnt vmcnt(0)2408; GFX9-NEXT: buffer_wbinvl1_vol2409; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]2410; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2411; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]2412; GFX9-NEXT: s_cbranch_execnz .LBB42_12413; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end2414; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]2415; GFX9-NEXT: v_mov_b32_e32 v0, v42416; GFX9-NEXT: v_mov_b32_e32 v1, v52417; GFX9-NEXT: s_setpc_b64 s[30:31]2418 %result = atomicrmw and ptr addrspace(1) %ptr, i64 %in seq_cst2419 ret i64 %result2420}2421 2422define i64 @global_atomic_and_i64_ret_offset(ptr addrspace(1) %out, i64 %in) {2423; SI-LABEL: global_atomic_and_i64_ret_offset:2424; SI: ; %bb.0:2425; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2426; SI-NEXT: v_mov_b32_e32 v6, v32427; SI-NEXT: v_mov_b32_e32 v7, v22428; SI-NEXT: v_mov_b32_e32 v5, v12429; SI-NEXT: v_mov_b32_e32 v4, v02430; SI-NEXT: s_mov_b32 s6, 02431; SI-NEXT: s_mov_b32 s7, 0xf0002432; SI-NEXT: s_mov_b32 s4, s62433; SI-NEXT: s_mov_b32 s5, s62434; SI-NEXT: buffer_load_dwordx2 v[10:11], v[4:5], s[4:7], 0 addr64 offset:322435; SI-NEXT: s_mov_b64 s[8:9], 02436; SI-NEXT: .LBB43_1: ; %atomicrmw.start2437; SI-NEXT: ; =>This Inner Loop Header: Depth=12438; SI-NEXT: s_waitcnt vmcnt(0)2439; SI-NEXT: v_and_b32_e32 v9, v11, v62440; SI-NEXT: v_and_b32_e32 v8, v10, v72441; SI-NEXT: s_waitcnt expcnt(0)2442; SI-NEXT: v_mov_b32_e32 v0, v82443; SI-NEXT: v_mov_b32_e32 v1, v92444; SI-NEXT: v_mov_b32_e32 v2, v102445; SI-NEXT: v_mov_b32_e32 v3, v112446; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 offset:32 glc2447; SI-NEXT: s_waitcnt vmcnt(0)2448; SI-NEXT: buffer_wbinvl12449; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]2450; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]2451; SI-NEXT: v_mov_b32_e32 v11, v12452; SI-NEXT: v_mov_b32_e32 v10, v02453; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]2454; SI-NEXT: s_cbranch_execnz .LBB43_12455; SI-NEXT: ; %bb.2: ; %atomicrmw.end2456; SI-NEXT: s_or_b64 exec, exec, s[8:9]2457; SI-NEXT: s_waitcnt expcnt(0)2458; SI-NEXT: s_setpc_b64 s[30:31]2459;2460; VI-LABEL: global_atomic_and_i64_ret_offset:2461; VI: ; %bb.0:2462; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2463; VI-NEXT: v_add_u32_e32 v4, vcc, 32, v02464; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc2465; VI-NEXT: flat_load_dwordx2 v[0:1], v[4:5]2466; VI-NEXT: s_mov_b64 s[4:5], 02467; VI-NEXT: .LBB43_1: ; %atomicrmw.start2468; VI-NEXT: ; =>This Inner Loop Header: Depth=12469; VI-NEXT: s_waitcnt vmcnt(0)2470; VI-NEXT: v_mov_b32_e32 v9, v12471; VI-NEXT: v_mov_b32_e32 v8, v02472; VI-NEXT: v_and_b32_e32 v7, v9, v32473; VI-NEXT: v_and_b32_e32 v6, v8, v22474; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc2475; VI-NEXT: s_waitcnt vmcnt(0)2476; VI-NEXT: buffer_wbinvl1_vol2477; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]2478; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2479; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]2480; VI-NEXT: s_cbranch_execnz .LBB43_12481; VI-NEXT: ; %bb.2: ; %atomicrmw.end2482; VI-NEXT: s_or_b64 exec, exec, s[4:5]2483; VI-NEXT: s_setpc_b64 s[30:31]2484;2485; GFX9-LABEL: global_atomic_and_i64_ret_offset:2486; GFX9: ; %bb.0:2487; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2488; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:322489; GFX9-NEXT: s_mov_b64 s[4:5], 02490; GFX9-NEXT: .LBB43_1: ; %atomicrmw.start2491; GFX9-NEXT: ; =>This Inner Loop Header: Depth=12492; GFX9-NEXT: s_waitcnt vmcnt(0)2493; GFX9-NEXT: v_mov_b32_e32 v7, v52494; GFX9-NEXT: v_mov_b32_e32 v6, v42495; GFX9-NEXT: v_and_b32_e32 v5, v7, v32496; GFX9-NEXT: v_and_b32_e32 v4, v6, v22497; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc2498; GFX9-NEXT: s_waitcnt vmcnt(0)2499; GFX9-NEXT: buffer_wbinvl1_vol2500; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]2501; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2502; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]2503; GFX9-NEXT: s_cbranch_execnz .LBB43_12504; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end2505; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]2506; GFX9-NEXT: v_mov_b32_e32 v0, v42507; GFX9-NEXT: v_mov_b32_e32 v1, v52508; GFX9-NEXT: s_setpc_b64 s[30:31]2509 %gep = getelementptr i64, ptr addrspace(1) %out, i64 42510 %result = atomicrmw and ptr addrspace(1) %gep, i64 %in seq_cst2511 ret i64 %result2512}2513 2514define amdgpu_gfx void @global_atomic_and_i64_noret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {2515; SI-LABEL: global_atomic_and_i64_noret_scalar:2516; SI: ; %bb.0:2517; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2518; SI-NEXT: s_xor_saveexec_b64 s[34:35], -12519; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill2520; SI-NEXT: s_mov_b64 exec, s[34:35]2521; SI-NEXT: s_waitcnt expcnt(0)2522; SI-NEXT: v_writelane_b32 v8, s6, 02523; SI-NEXT: v_writelane_b32 v8, s7, 12524; SI-NEXT: s_mov_b32 s34, s72525; SI-NEXT: s_mov_b32 s35, s62526; SI-NEXT: s_mov_b32 s7, 0xf0002527; SI-NEXT: s_mov_b32 s6, -12528; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 02529; SI-NEXT: s_mov_b64 s[36:37], 02530; SI-NEXT: .LBB44_1: ; %atomicrmw.start2531; SI-NEXT: ; =>This Inner Loop Header: Depth=12532; SI-NEXT: s_waitcnt vmcnt(0)2533; SI-NEXT: v_and_b32_e32 v1, s34, v32534; SI-NEXT: v_and_b32_e32 v0, s35, v22535; SI-NEXT: s_waitcnt expcnt(0)2536; SI-NEXT: v_mov_b32_e32 v7, v32537; SI-NEXT: v_mov_b32_e32 v6, v22538; SI-NEXT: v_mov_b32_e32 v5, v12539; SI-NEXT: v_mov_b32_e32 v4, v02540; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 glc2541; SI-NEXT: s_waitcnt vmcnt(0)2542; SI-NEXT: buffer_wbinvl12543; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]2544; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]2545; SI-NEXT: v_mov_b32_e32 v2, v42546; SI-NEXT: v_mov_b32_e32 v3, v52547; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]2548; SI-NEXT: s_cbranch_execnz .LBB44_12549; SI-NEXT: ; %bb.2: ; %atomicrmw.end2550; SI-NEXT: s_or_b64 exec, exec, s[36:37]2551; SI-NEXT: v_readlane_b32 s7, v8, 12552; SI-NEXT: v_readlane_b32 s6, v8, 02553; SI-NEXT: s_xor_saveexec_b64 s[34:35], -12554; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload2555; SI-NEXT: s_mov_b64 exec, s[34:35]2556; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)2557; SI-NEXT: s_setpc_b64 s[30:31]2558;2559; VI-LABEL: global_atomic_and_i64_noret_scalar:2560; VI: ; %bb.0:2561; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2562; VI-NEXT: v_mov_b32_e32 v0, s42563; VI-NEXT: v_mov_b32_e32 v1, s52564; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]2565; VI-NEXT: v_mov_b32_e32 v4, s42566; VI-NEXT: s_mov_b64 s[34:35], 02567; VI-NEXT: v_mov_b32_e32 v5, s52568; VI-NEXT: .LBB44_1: ; %atomicrmw.start2569; VI-NEXT: ; =>This Inner Loop Header: Depth=12570; VI-NEXT: s_waitcnt vmcnt(0)2571; VI-NEXT: v_and_b32_e32 v1, s7, v32572; VI-NEXT: v_and_b32_e32 v0, s6, v22573; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc2574; VI-NEXT: s_waitcnt vmcnt(0)2575; VI-NEXT: buffer_wbinvl1_vol2576; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]2577; VI-NEXT: v_mov_b32_e32 v3, v12578; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]2579; VI-NEXT: v_mov_b32_e32 v2, v02580; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]2581; VI-NEXT: s_cbranch_execnz .LBB44_12582; VI-NEXT: ; %bb.2: ; %atomicrmw.end2583; VI-NEXT: s_or_b64 exec, exec, s[34:35]2584; VI-NEXT: s_setpc_b64 s[30:31]2585;2586; GFX9-LABEL: global_atomic_and_i64_noret_scalar:2587; GFX9: ; %bb.0:2588; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2589; GFX9-NEXT: v_mov_b32_e32 v4, 02590; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5]2591; GFX9-NEXT: s_mov_b64 s[34:35], 02592; GFX9-NEXT: .LBB44_1: ; %atomicrmw.start2593; GFX9-NEXT: ; =>This Inner Loop Header: Depth=12594; GFX9-NEXT: s_waitcnt vmcnt(0)2595; GFX9-NEXT: v_and_b32_e32 v1, s7, v32596; GFX9-NEXT: v_and_b32_e32 v0, s6, v22597; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc2598; GFX9-NEXT: s_waitcnt vmcnt(0)2599; GFX9-NEXT: buffer_wbinvl1_vol2600; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]2601; GFX9-NEXT: v_mov_b32_e32 v3, v12602; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]2603; GFX9-NEXT: v_mov_b32_e32 v2, v02604; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]2605; GFX9-NEXT: s_cbranch_execnz .LBB44_12606; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end2607; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]2608; GFX9-NEXT: s_setpc_b64 s[30:31]2609 %tmp0 = atomicrmw and ptr addrspace(1) %ptr, i64 %in seq_cst2610 ret void2611}2612 2613define amdgpu_gfx void @global_atomic_and_i64_noret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {2614; SI-LABEL: global_atomic_and_i64_noret_offset_scalar:2615; SI: ; %bb.0:2616; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2617; SI-NEXT: s_xor_saveexec_b64 s[34:35], -12618; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill2619; SI-NEXT: s_mov_b64 exec, s[34:35]2620; SI-NEXT: s_waitcnt expcnt(0)2621; SI-NEXT: v_writelane_b32 v8, s6, 02622; SI-NEXT: v_writelane_b32 v8, s7, 12623; SI-NEXT: s_mov_b32 s34, s72624; SI-NEXT: s_mov_b32 s35, s62625; SI-NEXT: s_mov_b32 s7, 0xf0002626; SI-NEXT: s_mov_b32 s6, -12627; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:322628; SI-NEXT: s_mov_b64 s[36:37], 02629; SI-NEXT: .LBB45_1: ; %atomicrmw.start2630; SI-NEXT: ; =>This Inner Loop Header: Depth=12631; SI-NEXT: s_waitcnt vmcnt(0)2632; SI-NEXT: v_and_b32_e32 v1, s34, v32633; SI-NEXT: v_and_b32_e32 v0, s35, v22634; SI-NEXT: s_waitcnt expcnt(0)2635; SI-NEXT: v_mov_b32_e32 v7, v32636; SI-NEXT: v_mov_b32_e32 v6, v22637; SI-NEXT: v_mov_b32_e32 v5, v12638; SI-NEXT: v_mov_b32_e32 v4, v02639; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 offset:32 glc2640; SI-NEXT: s_waitcnt vmcnt(0)2641; SI-NEXT: buffer_wbinvl12642; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]2643; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]2644; SI-NEXT: v_mov_b32_e32 v2, v42645; SI-NEXT: v_mov_b32_e32 v3, v52646; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]2647; SI-NEXT: s_cbranch_execnz .LBB45_12648; SI-NEXT: ; %bb.2: ; %atomicrmw.end2649; SI-NEXT: s_or_b64 exec, exec, s[36:37]2650; SI-NEXT: v_readlane_b32 s7, v8, 12651; SI-NEXT: v_readlane_b32 s6, v8, 02652; SI-NEXT: s_xor_saveexec_b64 s[34:35], -12653; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload2654; SI-NEXT: s_mov_b64 exec, s[34:35]2655; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)2656; SI-NEXT: s_setpc_b64 s[30:31]2657;2658; VI-LABEL: global_atomic_and_i64_noret_offset_scalar:2659; VI: ; %bb.0:2660; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2661; VI-NEXT: s_add_u32 s34, s4, 322662; VI-NEXT: s_addc_u32 s35, s5, 02663; VI-NEXT: v_mov_b32_e32 v4, s342664; VI-NEXT: v_mov_b32_e32 v5, s352665; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]2666; VI-NEXT: s_mov_b64 s[34:35], 02667; VI-NEXT: .LBB45_1: ; %atomicrmw.start2668; VI-NEXT: ; =>This Inner Loop Header: Depth=12669; VI-NEXT: s_waitcnt vmcnt(0)2670; VI-NEXT: v_and_b32_e32 v1, s7, v32671; VI-NEXT: v_and_b32_e32 v0, s6, v22672; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc2673; VI-NEXT: s_waitcnt vmcnt(0)2674; VI-NEXT: buffer_wbinvl1_vol2675; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]2676; VI-NEXT: v_mov_b32_e32 v3, v12677; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]2678; VI-NEXT: v_mov_b32_e32 v2, v02679; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]2680; VI-NEXT: s_cbranch_execnz .LBB45_12681; VI-NEXT: ; %bb.2: ; %atomicrmw.end2682; VI-NEXT: s_or_b64 exec, exec, s[34:35]2683; VI-NEXT: s_setpc_b64 s[30:31]2684;2685; GFX9-LABEL: global_atomic_and_i64_noret_offset_scalar:2686; GFX9: ; %bb.0:2687; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2688; GFX9-NEXT: v_mov_b32_e32 v4, 02689; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5] offset:322690; GFX9-NEXT: s_mov_b64 s[34:35], 02691; GFX9-NEXT: .LBB45_1: ; %atomicrmw.start2692; GFX9-NEXT: ; =>This Inner Loop Header: Depth=12693; GFX9-NEXT: s_waitcnt vmcnt(0)2694; GFX9-NEXT: v_and_b32_e32 v1, s7, v32695; GFX9-NEXT: v_and_b32_e32 v0, s6, v22696; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc2697; GFX9-NEXT: s_waitcnt vmcnt(0)2698; GFX9-NEXT: buffer_wbinvl1_vol2699; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]2700; GFX9-NEXT: v_mov_b32_e32 v3, v12701; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]2702; GFX9-NEXT: v_mov_b32_e32 v2, v02703; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]2704; GFX9-NEXT: s_cbranch_execnz .LBB45_12705; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end2706; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]2707; GFX9-NEXT: s_setpc_b64 s[30:31]2708 %gep = getelementptr i64, ptr addrspace(1) %out, i64 42709 %tmp0 = atomicrmw and ptr addrspace(1) %gep, i64 %in seq_cst2710 ret void2711}2712 2713define amdgpu_gfx i64 @global_atomic_and_i64_ret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {2714; SI-LABEL: global_atomic_and_i64_ret_scalar:2715; SI: ; %bb.0:2716; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2717; SI-NEXT: s_xor_saveexec_b64 s[34:35], -12718; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill2719; SI-NEXT: s_mov_b64 exec, s[34:35]2720; SI-NEXT: s_waitcnt expcnt(0)2721; SI-NEXT: v_writelane_b32 v6, s6, 02722; SI-NEXT: v_writelane_b32 v6, s7, 12723; SI-NEXT: s_mov_b32 s34, s72724; SI-NEXT: s_mov_b32 s35, s62725; SI-NEXT: s_mov_b32 s7, 0xf0002726; SI-NEXT: s_mov_b32 s6, -12727; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 02728; SI-NEXT: s_mov_b64 s[36:37], 02729; SI-NEXT: .LBB46_1: ; %atomicrmw.start2730; SI-NEXT: ; =>This Inner Loop Header: Depth=12731; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)2732; SI-NEXT: v_and_b32_e32 v3, s34, v52733; SI-NEXT: v_and_b32_e32 v2, s35, v42734; SI-NEXT: v_mov_b32_e32 v0, v22735; SI-NEXT: v_mov_b32_e32 v1, v32736; SI-NEXT: v_mov_b32_e32 v2, v42737; SI-NEXT: v_mov_b32_e32 v3, v52738; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc2739; SI-NEXT: s_waitcnt vmcnt(0)2740; SI-NEXT: buffer_wbinvl12741; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]2742; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]2743; SI-NEXT: v_mov_b32_e32 v5, v12744; SI-NEXT: v_mov_b32_e32 v4, v02745; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]2746; SI-NEXT: s_cbranch_execnz .LBB46_12747; SI-NEXT: ; %bb.2: ; %atomicrmw.end2748; SI-NEXT: s_or_b64 exec, exec, s[36:37]2749; SI-NEXT: v_readlane_b32 s7, v6, 12750; SI-NEXT: v_readlane_b32 s6, v6, 02751; SI-NEXT: s_xor_saveexec_b64 s[34:35], -12752; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload2753; SI-NEXT: s_mov_b64 exec, s[34:35]2754; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)2755; SI-NEXT: s_setpc_b64 s[30:31]2756;2757; VI-LABEL: global_atomic_and_i64_ret_scalar:2758; VI: ; %bb.0:2759; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2760; VI-NEXT: v_mov_b32_e32 v0, s42761; VI-NEXT: v_mov_b32_e32 v1, s52762; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]2763; VI-NEXT: v_mov_b32_e32 v2, s42764; VI-NEXT: s_mov_b64 s[34:35], 02765; VI-NEXT: v_mov_b32_e32 v3, s52766; VI-NEXT: .LBB46_1: ; %atomicrmw.start2767; VI-NEXT: ; =>This Inner Loop Header: Depth=12768; VI-NEXT: s_waitcnt vmcnt(0)2769; VI-NEXT: v_mov_b32_e32 v7, v12770; VI-NEXT: v_mov_b32_e32 v6, v02771; VI-NEXT: v_and_b32_e32 v5, s7, v72772; VI-NEXT: v_and_b32_e32 v4, s6, v62773; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc2774; VI-NEXT: s_waitcnt vmcnt(0)2775; VI-NEXT: buffer_wbinvl1_vol2776; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]2777; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]2778; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]2779; VI-NEXT: s_cbranch_execnz .LBB46_12780; VI-NEXT: ; %bb.2: ; %atomicrmw.end2781; VI-NEXT: s_or_b64 exec, exec, s[34:35]2782; VI-NEXT: s_setpc_b64 s[30:31]2783;2784; GFX9-LABEL: global_atomic_and_i64_ret_scalar:2785; GFX9: ; %bb.0:2786; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2787; GFX9-NEXT: v_mov_b32_e32 v2, 02788; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]2789; GFX9-NEXT: s_mov_b64 s[34:35], 02790; GFX9-NEXT: .LBB46_1: ; %atomicrmw.start2791; GFX9-NEXT: ; =>This Inner Loop Header: Depth=12792; GFX9-NEXT: s_waitcnt vmcnt(0)2793; GFX9-NEXT: v_mov_b32_e32 v6, v12794; GFX9-NEXT: v_mov_b32_e32 v5, v02795; GFX9-NEXT: v_and_b32_e32 v4, s7, v62796; GFX9-NEXT: v_and_b32_e32 v3, s6, v52797; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] glc2798; GFX9-NEXT: s_waitcnt vmcnt(0)2799; GFX9-NEXT: buffer_wbinvl1_vol2800; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]2801; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]2802; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]2803; GFX9-NEXT: s_cbranch_execnz .LBB46_12804; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end2805; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]2806; GFX9-NEXT: s_setpc_b64 s[30:31]2807 %result = atomicrmw and ptr addrspace(1) %ptr, i64 %in seq_cst2808 ret i64 %result2809}2810 2811define amdgpu_gfx i64 @global_atomic_and_i64_ret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {2812; SI-LABEL: global_atomic_and_i64_ret_offset_scalar:2813; SI: ; %bb.0:2814; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2815; SI-NEXT: s_xor_saveexec_b64 s[34:35], -12816; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill2817; SI-NEXT: s_mov_b64 exec, s[34:35]2818; SI-NEXT: s_waitcnt expcnt(0)2819; SI-NEXT: v_writelane_b32 v6, s6, 02820; SI-NEXT: v_writelane_b32 v6, s7, 12821; SI-NEXT: s_mov_b32 s34, s72822; SI-NEXT: s_mov_b32 s35, s62823; SI-NEXT: s_mov_b32 s7, 0xf0002824; SI-NEXT: s_mov_b32 s6, -12825; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0 offset:322826; SI-NEXT: s_mov_b64 s[36:37], 02827; SI-NEXT: .LBB47_1: ; %atomicrmw.start2828; SI-NEXT: ; =>This Inner Loop Header: Depth=12829; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)2830; SI-NEXT: v_and_b32_e32 v3, s34, v52831; SI-NEXT: v_and_b32_e32 v2, s35, v42832; SI-NEXT: v_mov_b32_e32 v0, v22833; SI-NEXT: v_mov_b32_e32 v1, v32834; SI-NEXT: v_mov_b32_e32 v2, v42835; SI-NEXT: v_mov_b32_e32 v3, v52836; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 offset:32 glc2837; SI-NEXT: s_waitcnt vmcnt(0)2838; SI-NEXT: buffer_wbinvl12839; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]2840; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]2841; SI-NEXT: v_mov_b32_e32 v5, v12842; SI-NEXT: v_mov_b32_e32 v4, v02843; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]2844; SI-NEXT: s_cbranch_execnz .LBB47_12845; SI-NEXT: ; %bb.2: ; %atomicrmw.end2846; SI-NEXT: s_or_b64 exec, exec, s[36:37]2847; SI-NEXT: v_readlane_b32 s7, v6, 12848; SI-NEXT: v_readlane_b32 s6, v6, 02849; SI-NEXT: s_xor_saveexec_b64 s[34:35], -12850; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload2851; SI-NEXT: s_mov_b64 exec, s[34:35]2852; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)2853; SI-NEXT: s_setpc_b64 s[30:31]2854;2855; VI-LABEL: global_atomic_and_i64_ret_offset_scalar:2856; VI: ; %bb.0:2857; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2858; VI-NEXT: s_add_u32 s34, s4, 322859; VI-NEXT: s_addc_u32 s35, s5, 02860; VI-NEXT: v_mov_b32_e32 v2, s342861; VI-NEXT: v_mov_b32_e32 v3, s352862; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]2863; VI-NEXT: s_mov_b64 s[34:35], 02864; VI-NEXT: .LBB47_1: ; %atomicrmw.start2865; VI-NEXT: ; =>This Inner Loop Header: Depth=12866; VI-NEXT: s_waitcnt vmcnt(0)2867; VI-NEXT: v_mov_b32_e32 v7, v12868; VI-NEXT: v_mov_b32_e32 v6, v02869; VI-NEXT: v_and_b32_e32 v5, s7, v72870; VI-NEXT: v_and_b32_e32 v4, s6, v62871; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc2872; VI-NEXT: s_waitcnt vmcnt(0)2873; VI-NEXT: buffer_wbinvl1_vol2874; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]2875; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]2876; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]2877; VI-NEXT: s_cbranch_execnz .LBB47_12878; VI-NEXT: ; %bb.2: ; %atomicrmw.end2879; VI-NEXT: s_or_b64 exec, exec, s[34:35]2880; VI-NEXT: s_setpc_b64 s[30:31]2881;2882; GFX9-LABEL: global_atomic_and_i64_ret_offset_scalar:2883; GFX9: ; %bb.0:2884; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2885; GFX9-NEXT: v_mov_b32_e32 v2, 02886; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5] offset:322887; GFX9-NEXT: s_mov_b64 s[34:35], 02888; GFX9-NEXT: .LBB47_1: ; %atomicrmw.start2889; GFX9-NEXT: ; =>This Inner Loop Header: Depth=12890; GFX9-NEXT: s_waitcnt vmcnt(0)2891; GFX9-NEXT: v_mov_b32_e32 v6, v12892; GFX9-NEXT: v_mov_b32_e32 v5, v02893; GFX9-NEXT: v_and_b32_e32 v4, s7, v62894; GFX9-NEXT: v_and_b32_e32 v3, s6, v52895; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] offset:32 glc2896; GFX9-NEXT: s_waitcnt vmcnt(0)2897; GFX9-NEXT: buffer_wbinvl1_vol2898; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]2899; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]2900; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]2901; GFX9-NEXT: s_cbranch_execnz .LBB47_12902; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end2903; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]2904; GFX9-NEXT: s_setpc_b64 s[30:31]2905 %gep = getelementptr i64, ptr addrspace(1) %out, i64 42906 %result = atomicrmw and ptr addrspace(1) %gep, i64 %in seq_cst2907 ret i64 %result2908}2909 2910define void @global_atomic_and_i64_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {2911; SI-LABEL: global_atomic_and_i64_noret_offset__amdgpu_no_remote_memory:2912; SI: ; %bb.0:2913; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2914; SI-NEXT: s_mov_b32 s6, 02915; SI-NEXT: s_mov_b32 s7, 0xf0002916; SI-NEXT: s_mov_b32 s4, s62917; SI-NEXT: s_mov_b32 s5, s62918; SI-NEXT: buffer_atomic_and_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:322919; SI-NEXT: s_waitcnt vmcnt(0)2920; SI-NEXT: buffer_wbinvl12921; SI-NEXT: s_waitcnt expcnt(0)2922; SI-NEXT: s_setpc_b64 s[30:31]2923;2924; VI-LABEL: global_atomic_and_i64_noret_offset__amdgpu_no_remote_memory:2925; VI: ; %bb.0:2926; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2927; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v02928; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2929; VI-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]2930; VI-NEXT: s_waitcnt vmcnt(0)2931; VI-NEXT: buffer_wbinvl1_vol2932; VI-NEXT: s_setpc_b64 s[30:31]2933;2934; GFX9-LABEL: global_atomic_and_i64_noret_offset__amdgpu_no_remote_memory:2935; GFX9: ; %bb.0:2936; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2937; GFX9-NEXT: global_atomic_and_x2 v[0:1], v[2:3], off offset:322938; GFX9-NEXT: s_waitcnt vmcnt(0)2939; GFX9-NEXT: buffer_wbinvl1_vol2940; GFX9-NEXT: s_setpc_b64 s[30:31]2941 %gep = getelementptr i64, ptr addrspace(1) %out, i64 42942 %tmp0 = atomicrmw and ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !02943 ret void2944}2945 2946define i64 @global_atomic_and_i64_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {2947; SI-LABEL: global_atomic_and_i64_ret_offset__amdgpu_no_remote_memory:2948; SI: ; %bb.0:2949; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2950; SI-NEXT: s_mov_b32 s6, 02951; SI-NEXT: s_mov_b32 s7, 0xf0002952; SI-NEXT: s_mov_b32 s4, s62953; SI-NEXT: s_mov_b32 s5, s62954; SI-NEXT: buffer_atomic_and_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc2955; SI-NEXT: s_waitcnt vmcnt(0)2956; SI-NEXT: buffer_wbinvl12957; SI-NEXT: v_mov_b32_e32 v0, v22958; SI-NEXT: v_mov_b32_e32 v1, v32959; SI-NEXT: s_waitcnt expcnt(0)2960; SI-NEXT: s_setpc_b64 s[30:31]2961;2962; VI-LABEL: global_atomic_and_i64_ret_offset__amdgpu_no_remote_memory:2963; VI: ; %bb.0:2964; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2965; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v02966; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2967; VI-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc2968; VI-NEXT: s_waitcnt vmcnt(0)2969; VI-NEXT: buffer_wbinvl1_vol2970; VI-NEXT: s_setpc_b64 s[30:31]2971;2972; GFX9-LABEL: global_atomic_and_i64_ret_offset__amdgpu_no_remote_memory:2973; GFX9: ; %bb.0:2974; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2975; GFX9-NEXT: global_atomic_and_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc2976; GFX9-NEXT: s_waitcnt vmcnt(0)2977; GFX9-NEXT: buffer_wbinvl1_vol2978; GFX9-NEXT: s_setpc_b64 s[30:31]2979 %gep = getelementptr i64, ptr addrspace(1) %out, i64 42980 %result = atomicrmw and ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !02981 ret i64 %result2982}2983 2984; ---------------------------------------------------------------------2985; atomicrmw nand2986; ---------------------------------------------------------------------2987 2988define void @global_atomic_nand_i64_noret(ptr addrspace(1) %ptr, i64 %in) {2989; SI-LABEL: global_atomic_nand_i64_noret:2990; SI: ; %bb.0:2991; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2992; SI-NEXT: s_mov_b32 s6, 02993; SI-NEXT: s_mov_b32 s7, 0xf0002994; SI-NEXT: s_mov_b32 s4, s62995; SI-NEXT: s_mov_b32 s5, s62996; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr642997; SI-NEXT: s_mov_b64 s[8:9], 02998; SI-NEXT: .LBB50_1: ; %atomicrmw.start2999; SI-NEXT: ; =>This Inner Loop Header: Depth=13000; SI-NEXT: s_waitcnt vmcnt(0)3001; SI-NEXT: v_and_b32_e32 v4, v7, v33002; SI-NEXT: s_waitcnt expcnt(0)3003; SI-NEXT: v_and_b32_e32 v8, v6, v23004; SI-NEXT: v_not_b32_e32 v5, v43005; SI-NEXT: v_not_b32_e32 v4, v83006; SI-NEXT: v_mov_b32_e32 v11, v73007; SI-NEXT: v_mov_b32_e32 v10, v63008; SI-NEXT: v_mov_b32_e32 v9, v53009; SI-NEXT: v_mov_b32_e32 v8, v43010; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc3011; SI-NEXT: s_waitcnt vmcnt(0)3012; SI-NEXT: buffer_wbinvl13013; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]3014; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]3015; SI-NEXT: v_mov_b32_e32 v6, v83016; SI-NEXT: v_mov_b32_e32 v7, v93017; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]3018; SI-NEXT: s_cbranch_execnz .LBB50_13019; SI-NEXT: ; %bb.2: ; %atomicrmw.end3020; SI-NEXT: s_or_b64 exec, exec, s[8:9]3021; SI-NEXT: s_waitcnt expcnt(0)3022; SI-NEXT: s_setpc_b64 s[30:31]3023;3024; VI-LABEL: global_atomic_nand_i64_noret:3025; VI: ; %bb.0:3026; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3027; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]3028; VI-NEXT: s_mov_b64 s[4:5], 03029; VI-NEXT: .LBB50_1: ; %atomicrmw.start3030; VI-NEXT: ; =>This Inner Loop Header: Depth=13031; VI-NEXT: s_waitcnt vmcnt(0)3032; VI-NEXT: v_and_b32_e32 v4, v7, v33033; VI-NEXT: v_and_b32_e32 v8, v6, v23034; VI-NEXT: v_not_b32_e32 v5, v43035; VI-NEXT: v_not_b32_e32 v4, v83036; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc3037; VI-NEXT: s_waitcnt vmcnt(0)3038; VI-NEXT: buffer_wbinvl1_vol3039; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]3040; VI-NEXT: v_mov_b32_e32 v7, v53041; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3042; VI-NEXT: v_mov_b32_e32 v6, v43043; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]3044; VI-NEXT: s_cbranch_execnz .LBB50_13045; VI-NEXT: ; %bb.2: ; %atomicrmw.end3046; VI-NEXT: s_or_b64 exec, exec, s[4:5]3047; VI-NEXT: s_setpc_b64 s[30:31]3048;3049; GFX9-LABEL: global_atomic_nand_i64_noret:3050; GFX9: ; %bb.0:3051; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3052; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off3053; GFX9-NEXT: s_mov_b64 s[4:5], 03054; GFX9-NEXT: .LBB50_1: ; %atomicrmw.start3055; GFX9-NEXT: ; =>This Inner Loop Header: Depth=13056; GFX9-NEXT: s_waitcnt vmcnt(0)3057; GFX9-NEXT: v_and_b32_e32 v4, v7, v33058; GFX9-NEXT: v_and_b32_e32 v8, v6, v23059; GFX9-NEXT: v_not_b32_e32 v5, v43060; GFX9-NEXT: v_not_b32_e32 v4, v83061; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc3062; GFX9-NEXT: s_waitcnt vmcnt(0)3063; GFX9-NEXT: buffer_wbinvl1_vol3064; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]3065; GFX9-NEXT: v_mov_b32_e32 v7, v53066; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3067; GFX9-NEXT: v_mov_b32_e32 v6, v43068; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]3069; GFX9-NEXT: s_cbranch_execnz .LBB50_13070; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end3071; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]3072; GFX9-NEXT: s_setpc_b64 s[30:31]3073 %tmp0 = atomicrmw nand ptr addrspace(1) %ptr, i64 %in seq_cst3074 ret void3075}3076 3077define void @global_atomic_nand_i64_noret_offset(ptr addrspace(1) %out, i64 %in) {3078; SI-LABEL: global_atomic_nand_i64_noret_offset:3079; SI: ; %bb.0:3080; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3081; SI-NEXT: s_mov_b32 s6, 03082; SI-NEXT: s_mov_b32 s7, 0xf0003083; SI-NEXT: s_mov_b32 s4, s63084; SI-NEXT: s_mov_b32 s5, s63085; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:323086; SI-NEXT: s_mov_b64 s[8:9], 03087; SI-NEXT: .LBB51_1: ; %atomicrmw.start3088; SI-NEXT: ; =>This Inner Loop Header: Depth=13089; SI-NEXT: s_waitcnt vmcnt(0)3090; SI-NEXT: v_and_b32_e32 v4, v7, v33091; SI-NEXT: s_waitcnt expcnt(0)3092; SI-NEXT: v_and_b32_e32 v8, v6, v23093; SI-NEXT: v_not_b32_e32 v5, v43094; SI-NEXT: v_not_b32_e32 v4, v83095; SI-NEXT: v_mov_b32_e32 v11, v73096; SI-NEXT: v_mov_b32_e32 v10, v63097; SI-NEXT: v_mov_b32_e32 v9, v53098; SI-NEXT: v_mov_b32_e32 v8, v43099; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32 glc3100; SI-NEXT: s_waitcnt vmcnt(0)3101; SI-NEXT: buffer_wbinvl13102; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]3103; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]3104; SI-NEXT: v_mov_b32_e32 v6, v83105; SI-NEXT: v_mov_b32_e32 v7, v93106; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]3107; SI-NEXT: s_cbranch_execnz .LBB51_13108; SI-NEXT: ; %bb.2: ; %atomicrmw.end3109; SI-NEXT: s_or_b64 exec, exec, s[8:9]3110; SI-NEXT: s_waitcnt expcnt(0)3111; SI-NEXT: s_setpc_b64 s[30:31]3112;3113; VI-LABEL: global_atomic_nand_i64_noret_offset:3114; VI: ; %bb.0:3115; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3116; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v03117; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3118; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]3119; VI-NEXT: s_mov_b64 s[4:5], 03120; VI-NEXT: .LBB51_1: ; %atomicrmw.start3121; VI-NEXT: ; =>This Inner Loop Header: Depth=13122; VI-NEXT: s_waitcnt vmcnt(0)3123; VI-NEXT: v_and_b32_e32 v4, v7, v33124; VI-NEXT: v_and_b32_e32 v8, v6, v23125; VI-NEXT: v_not_b32_e32 v5, v43126; VI-NEXT: v_not_b32_e32 v4, v83127; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc3128; VI-NEXT: s_waitcnt vmcnt(0)3129; VI-NEXT: buffer_wbinvl1_vol3130; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]3131; VI-NEXT: v_mov_b32_e32 v7, v53132; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3133; VI-NEXT: v_mov_b32_e32 v6, v43134; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]3135; VI-NEXT: s_cbranch_execnz .LBB51_13136; VI-NEXT: ; %bb.2: ; %atomicrmw.end3137; VI-NEXT: s_or_b64 exec, exec, s[4:5]3138; VI-NEXT: s_setpc_b64 s[30:31]3139;3140; GFX9-LABEL: global_atomic_nand_i64_noret_offset:3141; GFX9: ; %bb.0:3142; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3143; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:323144; GFX9-NEXT: s_mov_b64 s[4:5], 03145; GFX9-NEXT: .LBB51_1: ; %atomicrmw.start3146; GFX9-NEXT: ; =>This Inner Loop Header: Depth=13147; GFX9-NEXT: s_waitcnt vmcnt(0)3148; GFX9-NEXT: v_and_b32_e32 v4, v7, v33149; GFX9-NEXT: v_and_b32_e32 v8, v6, v23150; GFX9-NEXT: v_not_b32_e32 v5, v43151; GFX9-NEXT: v_not_b32_e32 v4, v83152; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc3153; GFX9-NEXT: s_waitcnt vmcnt(0)3154; GFX9-NEXT: buffer_wbinvl1_vol3155; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]3156; GFX9-NEXT: v_mov_b32_e32 v7, v53157; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3158; GFX9-NEXT: v_mov_b32_e32 v6, v43159; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]3160; GFX9-NEXT: s_cbranch_execnz .LBB51_13161; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end3162; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]3163; GFX9-NEXT: s_setpc_b64 s[30:31]3164 %gep = getelementptr i64, ptr addrspace(1) %out, i64 43165 %tmp0 = atomicrmw nand ptr addrspace(1) %gep, i64 %in seq_cst3166 ret void3167}3168 3169define i64 @global_atomic_nand_i64_ret(ptr addrspace(1) %ptr, i64 %in) {3170; SI-LABEL: global_atomic_nand_i64_ret:3171; SI: ; %bb.0:3172; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3173; SI-NEXT: v_mov_b32_e32 v6, v33174; SI-NEXT: v_mov_b32_e32 v7, v23175; SI-NEXT: v_mov_b32_e32 v5, v13176; SI-NEXT: v_mov_b32_e32 v4, v03177; SI-NEXT: s_mov_b32 s6, 03178; SI-NEXT: s_mov_b32 s7, 0xf0003179; SI-NEXT: s_mov_b32 s4, s63180; SI-NEXT: s_mov_b32 s5, s63181; SI-NEXT: buffer_load_dwordx2 v[10:11], v[4:5], s[4:7], 0 addr643182; SI-NEXT: s_mov_b64 s[8:9], 03183; SI-NEXT: .LBB52_1: ; %atomicrmw.start3184; SI-NEXT: ; =>This Inner Loop Header: Depth=13185; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)3186; SI-NEXT: v_and_b32_e32 v0, v11, v63187; SI-NEXT: v_and_b32_e32 v1, v10, v73188; SI-NEXT: v_not_b32_e32 v9, v03189; SI-NEXT: v_not_b32_e32 v8, v13190; SI-NEXT: v_mov_b32_e32 v0, v83191; SI-NEXT: v_mov_b32_e32 v1, v93192; SI-NEXT: v_mov_b32_e32 v2, v103193; SI-NEXT: v_mov_b32_e32 v3, v113194; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 glc3195; SI-NEXT: s_waitcnt vmcnt(0)3196; SI-NEXT: buffer_wbinvl13197; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]3198; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]3199; SI-NEXT: v_mov_b32_e32 v11, v13200; SI-NEXT: v_mov_b32_e32 v10, v03201; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]3202; SI-NEXT: s_cbranch_execnz .LBB52_13203; SI-NEXT: ; %bb.2: ; %atomicrmw.end3204; SI-NEXT: s_or_b64 exec, exec, s[8:9]3205; SI-NEXT: s_waitcnt expcnt(0)3206; SI-NEXT: s_setpc_b64 s[30:31]3207;3208; VI-LABEL: global_atomic_nand_i64_ret:3209; VI: ; %bb.0:3210; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3211; VI-NEXT: flat_load_dwordx2 v[4:5], v[0:1]3212; VI-NEXT: s_mov_b64 s[4:5], 03213; VI-NEXT: .LBB52_1: ; %atomicrmw.start3214; VI-NEXT: ; =>This Inner Loop Header: Depth=13215; VI-NEXT: s_waitcnt vmcnt(0)3216; VI-NEXT: v_mov_b32_e32 v7, v53217; VI-NEXT: v_mov_b32_e32 v6, v43218; VI-NEXT: v_and_b32_e32 v4, v7, v33219; VI-NEXT: v_and_b32_e32 v8, v6, v23220; VI-NEXT: v_not_b32_e32 v5, v43221; VI-NEXT: v_not_b32_e32 v4, v83222; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc3223; VI-NEXT: s_waitcnt vmcnt(0)3224; VI-NEXT: buffer_wbinvl1_vol3225; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]3226; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3227; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]3228; VI-NEXT: s_cbranch_execnz .LBB52_13229; VI-NEXT: ; %bb.2: ; %atomicrmw.end3230; VI-NEXT: s_or_b64 exec, exec, s[4:5]3231; VI-NEXT: v_mov_b32_e32 v0, v43232; VI-NEXT: v_mov_b32_e32 v1, v53233; VI-NEXT: s_setpc_b64 s[30:31]3234;3235; GFX9-LABEL: global_atomic_nand_i64_ret:3236; GFX9: ; %bb.0:3237; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3238; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off3239; GFX9-NEXT: s_mov_b64 s[4:5], 03240; GFX9-NEXT: .LBB52_1: ; %atomicrmw.start3241; GFX9-NEXT: ; =>This Inner Loop Header: Depth=13242; GFX9-NEXT: s_waitcnt vmcnt(0)3243; GFX9-NEXT: v_mov_b32_e32 v7, v53244; GFX9-NEXT: v_mov_b32_e32 v6, v43245; GFX9-NEXT: v_and_b32_e32 v4, v7, v33246; GFX9-NEXT: v_and_b32_e32 v8, v6, v23247; GFX9-NEXT: v_not_b32_e32 v5, v43248; GFX9-NEXT: v_not_b32_e32 v4, v83249; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc3250; GFX9-NEXT: s_waitcnt vmcnt(0)3251; GFX9-NEXT: buffer_wbinvl1_vol3252; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]3253; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3254; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]3255; GFX9-NEXT: s_cbranch_execnz .LBB52_13256; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end3257; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]3258; GFX9-NEXT: v_mov_b32_e32 v0, v43259; GFX9-NEXT: v_mov_b32_e32 v1, v53260; GFX9-NEXT: s_setpc_b64 s[30:31]3261 %result = atomicrmw nand ptr addrspace(1) %ptr, i64 %in seq_cst3262 ret i64 %result3263}3264 3265define i64 @global_atomic_nand_i64_ret_offset(ptr addrspace(1) %out, i64 %in) {3266; SI-LABEL: global_atomic_nand_i64_ret_offset:3267; SI: ; %bb.0:3268; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3269; SI-NEXT: v_mov_b32_e32 v6, v33270; SI-NEXT: v_mov_b32_e32 v7, v23271; SI-NEXT: v_mov_b32_e32 v5, v13272; SI-NEXT: v_mov_b32_e32 v4, v03273; SI-NEXT: s_mov_b32 s6, 03274; SI-NEXT: s_mov_b32 s7, 0xf0003275; SI-NEXT: s_mov_b32 s4, s63276; SI-NEXT: s_mov_b32 s5, s63277; SI-NEXT: buffer_load_dwordx2 v[10:11], v[4:5], s[4:7], 0 addr64 offset:323278; SI-NEXT: s_mov_b64 s[8:9], 03279; SI-NEXT: .LBB53_1: ; %atomicrmw.start3280; SI-NEXT: ; =>This Inner Loop Header: Depth=13281; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)3282; SI-NEXT: v_and_b32_e32 v0, v11, v63283; SI-NEXT: v_and_b32_e32 v1, v10, v73284; SI-NEXT: v_not_b32_e32 v9, v03285; SI-NEXT: v_not_b32_e32 v8, v13286; SI-NEXT: v_mov_b32_e32 v0, v83287; SI-NEXT: v_mov_b32_e32 v1, v93288; SI-NEXT: v_mov_b32_e32 v2, v103289; SI-NEXT: v_mov_b32_e32 v3, v113290; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 offset:32 glc3291; SI-NEXT: s_waitcnt vmcnt(0)3292; SI-NEXT: buffer_wbinvl13293; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]3294; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]3295; SI-NEXT: v_mov_b32_e32 v11, v13296; SI-NEXT: v_mov_b32_e32 v10, v03297; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]3298; SI-NEXT: s_cbranch_execnz .LBB53_13299; SI-NEXT: ; %bb.2: ; %atomicrmw.end3300; SI-NEXT: s_or_b64 exec, exec, s[8:9]3301; SI-NEXT: s_waitcnt expcnt(0)3302; SI-NEXT: s_setpc_b64 s[30:31]3303;3304; VI-LABEL: global_atomic_nand_i64_ret_offset:3305; VI: ; %bb.0:3306; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3307; VI-NEXT: v_add_u32_e32 v4, vcc, 32, v03308; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc3309; VI-NEXT: flat_load_dwordx2 v[0:1], v[4:5]3310; VI-NEXT: s_mov_b64 s[4:5], 03311; VI-NEXT: .LBB53_1: ; %atomicrmw.start3312; VI-NEXT: ; =>This Inner Loop Header: Depth=13313; VI-NEXT: s_waitcnt vmcnt(0)3314; VI-NEXT: v_mov_b32_e32 v9, v13315; VI-NEXT: v_mov_b32_e32 v8, v03316; VI-NEXT: v_and_b32_e32 v0, v9, v33317; VI-NEXT: v_and_b32_e32 v1, v8, v23318; VI-NEXT: v_not_b32_e32 v7, v03319; VI-NEXT: v_not_b32_e32 v6, v13320; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc3321; VI-NEXT: s_waitcnt vmcnt(0)3322; VI-NEXT: buffer_wbinvl1_vol3323; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]3324; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3325; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]3326; VI-NEXT: s_cbranch_execnz .LBB53_13327; VI-NEXT: ; %bb.2: ; %atomicrmw.end3328; VI-NEXT: s_or_b64 exec, exec, s[4:5]3329; VI-NEXT: s_setpc_b64 s[30:31]3330;3331; GFX9-LABEL: global_atomic_nand_i64_ret_offset:3332; GFX9: ; %bb.0:3333; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3334; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:323335; GFX9-NEXT: s_mov_b64 s[4:5], 03336; GFX9-NEXT: .LBB53_1: ; %atomicrmw.start3337; GFX9-NEXT: ; =>This Inner Loop Header: Depth=13338; GFX9-NEXT: s_waitcnt vmcnt(0)3339; GFX9-NEXT: v_mov_b32_e32 v7, v53340; GFX9-NEXT: v_mov_b32_e32 v6, v43341; GFX9-NEXT: v_and_b32_e32 v4, v7, v33342; GFX9-NEXT: v_and_b32_e32 v8, v6, v23343; GFX9-NEXT: v_not_b32_e32 v5, v43344; GFX9-NEXT: v_not_b32_e32 v4, v83345; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc3346; GFX9-NEXT: s_waitcnt vmcnt(0)3347; GFX9-NEXT: buffer_wbinvl1_vol3348; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]3349; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3350; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]3351; GFX9-NEXT: s_cbranch_execnz .LBB53_13352; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end3353; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]3354; GFX9-NEXT: v_mov_b32_e32 v0, v43355; GFX9-NEXT: v_mov_b32_e32 v1, v53356; GFX9-NEXT: s_setpc_b64 s[30:31]3357 %gep = getelementptr i64, ptr addrspace(1) %out, i64 43358 %result = atomicrmw nand ptr addrspace(1) %gep, i64 %in seq_cst3359 ret i64 %result3360}3361 3362define amdgpu_gfx void @global_atomic_nand_i64_noret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {3363; SI-LABEL: global_atomic_nand_i64_noret_scalar:3364; SI: ; %bb.0:3365; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3366; SI-NEXT: s_xor_saveexec_b64 s[34:35], -13367; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill3368; SI-NEXT: s_mov_b64 exec, s[34:35]3369; SI-NEXT: s_waitcnt expcnt(0)3370; SI-NEXT: v_writelane_b32 v8, s6, 03371; SI-NEXT: v_writelane_b32 v8, s7, 13372; SI-NEXT: s_mov_b32 s34, s73373; SI-NEXT: s_mov_b32 s35, s63374; SI-NEXT: s_mov_b32 s7, 0xf0003375; SI-NEXT: s_mov_b32 s6, -13376; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 03377; SI-NEXT: s_mov_b64 s[36:37], 03378; SI-NEXT: .LBB54_1: ; %atomicrmw.start3379; SI-NEXT: ; =>This Inner Loop Header: Depth=13380; SI-NEXT: s_waitcnt vmcnt(0)3381; SI-NEXT: v_and_b32_e32 v0, s34, v33382; SI-NEXT: s_waitcnt expcnt(0)3383; SI-NEXT: v_and_b32_e32 v4, s35, v23384; SI-NEXT: v_not_b32_e32 v1, v03385; SI-NEXT: v_not_b32_e32 v0, v43386; SI-NEXT: v_mov_b32_e32 v7, v33387; SI-NEXT: v_mov_b32_e32 v6, v23388; SI-NEXT: v_mov_b32_e32 v5, v13389; SI-NEXT: v_mov_b32_e32 v4, v03390; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 glc3391; SI-NEXT: s_waitcnt vmcnt(0)3392; SI-NEXT: buffer_wbinvl13393; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]3394; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]3395; SI-NEXT: v_mov_b32_e32 v2, v43396; SI-NEXT: v_mov_b32_e32 v3, v53397; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]3398; SI-NEXT: s_cbranch_execnz .LBB54_13399; SI-NEXT: ; %bb.2: ; %atomicrmw.end3400; SI-NEXT: s_or_b64 exec, exec, s[36:37]3401; SI-NEXT: v_readlane_b32 s7, v8, 13402; SI-NEXT: v_readlane_b32 s6, v8, 03403; SI-NEXT: s_xor_saveexec_b64 s[34:35], -13404; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload3405; SI-NEXT: s_mov_b64 exec, s[34:35]3406; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)3407; SI-NEXT: s_setpc_b64 s[30:31]3408;3409; VI-LABEL: global_atomic_nand_i64_noret_scalar:3410; VI: ; %bb.0:3411; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3412; VI-NEXT: v_mov_b32_e32 v0, s43413; VI-NEXT: v_mov_b32_e32 v1, s53414; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]3415; VI-NEXT: v_mov_b32_e32 v4, s43416; VI-NEXT: s_mov_b64 s[34:35], 03417; VI-NEXT: v_mov_b32_e32 v5, s53418; VI-NEXT: .LBB54_1: ; %atomicrmw.start3419; VI-NEXT: ; =>This Inner Loop Header: Depth=13420; VI-NEXT: s_waitcnt vmcnt(0)3421; VI-NEXT: v_and_b32_e32 v0, s7, v33422; VI-NEXT: v_and_b32_e32 v6, s6, v23423; VI-NEXT: v_not_b32_e32 v1, v03424; VI-NEXT: v_not_b32_e32 v0, v63425; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc3426; VI-NEXT: s_waitcnt vmcnt(0)3427; VI-NEXT: buffer_wbinvl1_vol3428; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3429; VI-NEXT: v_mov_b32_e32 v3, v13430; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]3431; VI-NEXT: v_mov_b32_e32 v2, v03432; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]3433; VI-NEXT: s_cbranch_execnz .LBB54_13434; VI-NEXT: ; %bb.2: ; %atomicrmw.end3435; VI-NEXT: s_or_b64 exec, exec, s[34:35]3436; VI-NEXT: s_setpc_b64 s[30:31]3437;3438; GFX9-LABEL: global_atomic_nand_i64_noret_scalar:3439; GFX9: ; %bb.0:3440; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3441; GFX9-NEXT: v_mov_b32_e32 v4, 03442; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5]3443; GFX9-NEXT: s_mov_b64 s[34:35], 03444; GFX9-NEXT: .LBB54_1: ; %atomicrmw.start3445; GFX9-NEXT: ; =>This Inner Loop Header: Depth=13446; GFX9-NEXT: s_waitcnt vmcnt(0)3447; GFX9-NEXT: v_and_b32_e32 v0, s7, v33448; GFX9-NEXT: v_and_b32_e32 v5, s6, v23449; GFX9-NEXT: v_not_b32_e32 v1, v03450; GFX9-NEXT: v_not_b32_e32 v0, v53451; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc3452; GFX9-NEXT: s_waitcnt vmcnt(0)3453; GFX9-NEXT: buffer_wbinvl1_vol3454; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3455; GFX9-NEXT: v_mov_b32_e32 v3, v13456; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]3457; GFX9-NEXT: v_mov_b32_e32 v2, v03458; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]3459; GFX9-NEXT: s_cbranch_execnz .LBB54_13460; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end3461; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]3462; GFX9-NEXT: s_setpc_b64 s[30:31]3463 %tmp0 = atomicrmw nand ptr addrspace(1) %ptr, i64 %in seq_cst3464 ret void3465}3466 3467define amdgpu_gfx void @global_atomic_nand_i64_noret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {3468; SI-LABEL: global_atomic_nand_i64_noret_offset_scalar:3469; SI: ; %bb.0:3470; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3471; SI-NEXT: s_xor_saveexec_b64 s[34:35], -13472; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill3473; SI-NEXT: s_mov_b64 exec, s[34:35]3474; SI-NEXT: s_waitcnt expcnt(0)3475; SI-NEXT: v_writelane_b32 v8, s6, 03476; SI-NEXT: v_writelane_b32 v8, s7, 13477; SI-NEXT: s_mov_b32 s34, s73478; SI-NEXT: s_mov_b32 s35, s63479; SI-NEXT: s_mov_b32 s7, 0xf0003480; SI-NEXT: s_mov_b32 s6, -13481; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:323482; SI-NEXT: s_mov_b64 s[36:37], 03483; SI-NEXT: .LBB55_1: ; %atomicrmw.start3484; SI-NEXT: ; =>This Inner Loop Header: Depth=13485; SI-NEXT: s_waitcnt vmcnt(0)3486; SI-NEXT: v_and_b32_e32 v0, s34, v33487; SI-NEXT: s_waitcnt expcnt(0)3488; SI-NEXT: v_and_b32_e32 v4, s35, v23489; SI-NEXT: v_not_b32_e32 v1, v03490; SI-NEXT: v_not_b32_e32 v0, v43491; SI-NEXT: v_mov_b32_e32 v7, v33492; SI-NEXT: v_mov_b32_e32 v6, v23493; SI-NEXT: v_mov_b32_e32 v5, v13494; SI-NEXT: v_mov_b32_e32 v4, v03495; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 offset:32 glc3496; SI-NEXT: s_waitcnt vmcnt(0)3497; SI-NEXT: buffer_wbinvl13498; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]3499; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]3500; SI-NEXT: v_mov_b32_e32 v2, v43501; SI-NEXT: v_mov_b32_e32 v3, v53502; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]3503; SI-NEXT: s_cbranch_execnz .LBB55_13504; SI-NEXT: ; %bb.2: ; %atomicrmw.end3505; SI-NEXT: s_or_b64 exec, exec, s[36:37]3506; SI-NEXT: v_readlane_b32 s7, v8, 13507; SI-NEXT: v_readlane_b32 s6, v8, 03508; SI-NEXT: s_xor_saveexec_b64 s[34:35], -13509; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload3510; SI-NEXT: s_mov_b64 exec, s[34:35]3511; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)3512; SI-NEXT: s_setpc_b64 s[30:31]3513;3514; VI-LABEL: global_atomic_nand_i64_noret_offset_scalar:3515; VI: ; %bb.0:3516; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3517; VI-NEXT: s_add_u32 s34, s4, 323518; VI-NEXT: s_addc_u32 s35, s5, 03519; VI-NEXT: v_mov_b32_e32 v4, s343520; VI-NEXT: v_mov_b32_e32 v5, s353521; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]3522; VI-NEXT: s_mov_b64 s[34:35], 03523; VI-NEXT: .LBB55_1: ; %atomicrmw.start3524; VI-NEXT: ; =>This Inner Loop Header: Depth=13525; VI-NEXT: s_waitcnt vmcnt(0)3526; VI-NEXT: v_and_b32_e32 v0, s7, v33527; VI-NEXT: v_and_b32_e32 v6, s6, v23528; VI-NEXT: v_not_b32_e32 v1, v03529; VI-NEXT: v_not_b32_e32 v0, v63530; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc3531; VI-NEXT: s_waitcnt vmcnt(0)3532; VI-NEXT: buffer_wbinvl1_vol3533; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3534; VI-NEXT: v_mov_b32_e32 v3, v13535; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]3536; VI-NEXT: v_mov_b32_e32 v2, v03537; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]3538; VI-NEXT: s_cbranch_execnz .LBB55_13539; VI-NEXT: ; %bb.2: ; %atomicrmw.end3540; VI-NEXT: s_or_b64 exec, exec, s[34:35]3541; VI-NEXT: s_setpc_b64 s[30:31]3542;3543; GFX9-LABEL: global_atomic_nand_i64_noret_offset_scalar:3544; GFX9: ; %bb.0:3545; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3546; GFX9-NEXT: v_mov_b32_e32 v4, 03547; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5] offset:323548; GFX9-NEXT: s_mov_b64 s[34:35], 03549; GFX9-NEXT: .LBB55_1: ; %atomicrmw.start3550; GFX9-NEXT: ; =>This Inner Loop Header: Depth=13551; GFX9-NEXT: s_waitcnt vmcnt(0)3552; GFX9-NEXT: v_and_b32_e32 v0, s7, v33553; GFX9-NEXT: v_and_b32_e32 v5, s6, v23554; GFX9-NEXT: v_not_b32_e32 v1, v03555; GFX9-NEXT: v_not_b32_e32 v0, v53556; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc3557; GFX9-NEXT: s_waitcnt vmcnt(0)3558; GFX9-NEXT: buffer_wbinvl1_vol3559; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3560; GFX9-NEXT: v_mov_b32_e32 v3, v13561; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]3562; GFX9-NEXT: v_mov_b32_e32 v2, v03563; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]3564; GFX9-NEXT: s_cbranch_execnz .LBB55_13565; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end3566; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]3567; GFX9-NEXT: s_setpc_b64 s[30:31]3568 %gep = getelementptr i64, ptr addrspace(1) %out, i64 43569 %tmp0 = atomicrmw nand ptr addrspace(1) %gep, i64 %in seq_cst3570 ret void3571}3572 3573define amdgpu_gfx i64 @global_atomic_nand_i64_ret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {3574; SI-LABEL: global_atomic_nand_i64_ret_scalar:3575; SI: ; %bb.0:3576; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3577; SI-NEXT: s_xor_saveexec_b64 s[34:35], -13578; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill3579; SI-NEXT: s_mov_b64 exec, s[34:35]3580; SI-NEXT: s_waitcnt expcnt(0)3581; SI-NEXT: v_writelane_b32 v6, s6, 03582; SI-NEXT: v_writelane_b32 v6, s7, 13583; SI-NEXT: s_mov_b32 s34, s73584; SI-NEXT: s_mov_b32 s35, s63585; SI-NEXT: s_mov_b32 s7, 0xf0003586; SI-NEXT: s_mov_b32 s6, -13587; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 03588; SI-NEXT: s_mov_b64 s[36:37], 03589; SI-NEXT: .LBB56_1: ; %atomicrmw.start3590; SI-NEXT: ; =>This Inner Loop Header: Depth=13591; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)3592; SI-NEXT: v_and_b32_e32 v0, s34, v53593; SI-NEXT: v_and_b32_e32 v1, s35, v43594; SI-NEXT: v_not_b32_e32 v3, v03595; SI-NEXT: v_not_b32_e32 v2, v13596; SI-NEXT: v_mov_b32_e32 v0, v23597; SI-NEXT: v_mov_b32_e32 v1, v33598; SI-NEXT: v_mov_b32_e32 v2, v43599; SI-NEXT: v_mov_b32_e32 v3, v53600; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc3601; SI-NEXT: s_waitcnt vmcnt(0)3602; SI-NEXT: buffer_wbinvl13603; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]3604; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]3605; SI-NEXT: v_mov_b32_e32 v5, v13606; SI-NEXT: v_mov_b32_e32 v4, v03607; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]3608; SI-NEXT: s_cbranch_execnz .LBB56_13609; SI-NEXT: ; %bb.2: ; %atomicrmw.end3610; SI-NEXT: s_or_b64 exec, exec, s[36:37]3611; SI-NEXT: v_readlane_b32 s7, v6, 13612; SI-NEXT: v_readlane_b32 s6, v6, 03613; SI-NEXT: s_xor_saveexec_b64 s[34:35], -13614; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload3615; SI-NEXT: s_mov_b64 exec, s[34:35]3616; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)3617; SI-NEXT: s_setpc_b64 s[30:31]3618;3619; VI-LABEL: global_atomic_nand_i64_ret_scalar:3620; VI: ; %bb.0:3621; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3622; VI-NEXT: v_mov_b32_e32 v0, s43623; VI-NEXT: v_mov_b32_e32 v1, s53624; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]3625; VI-NEXT: v_mov_b32_e32 v2, s43626; VI-NEXT: s_mov_b64 s[34:35], 03627; VI-NEXT: v_mov_b32_e32 v3, s53628; VI-NEXT: .LBB56_1: ; %atomicrmw.start3629; VI-NEXT: ; =>This Inner Loop Header: Depth=13630; VI-NEXT: s_waitcnt vmcnt(0)3631; VI-NEXT: v_mov_b32_e32 v7, v13632; VI-NEXT: v_mov_b32_e32 v6, v03633; VI-NEXT: v_and_b32_e32 v0, s7, v73634; VI-NEXT: v_and_b32_e32 v1, s6, v63635; VI-NEXT: v_not_b32_e32 v5, v03636; VI-NEXT: v_not_b32_e32 v4, v13637; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc3638; VI-NEXT: s_waitcnt vmcnt(0)3639; VI-NEXT: buffer_wbinvl1_vol3640; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]3641; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]3642; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]3643; VI-NEXT: s_cbranch_execnz .LBB56_13644; VI-NEXT: ; %bb.2: ; %atomicrmw.end3645; VI-NEXT: s_or_b64 exec, exec, s[34:35]3646; VI-NEXT: s_setpc_b64 s[30:31]3647;3648; GFX9-LABEL: global_atomic_nand_i64_ret_scalar:3649; GFX9: ; %bb.0:3650; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3651; GFX9-NEXT: v_mov_b32_e32 v2, 03652; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]3653; GFX9-NEXT: s_mov_b64 s[34:35], 03654; GFX9-NEXT: .LBB56_1: ; %atomicrmw.start3655; GFX9-NEXT: ; =>This Inner Loop Header: Depth=13656; GFX9-NEXT: s_waitcnt vmcnt(0)3657; GFX9-NEXT: v_mov_b32_e32 v6, v13658; GFX9-NEXT: v_mov_b32_e32 v5, v03659; GFX9-NEXT: v_and_b32_e32 v0, s7, v63660; GFX9-NEXT: v_and_b32_e32 v1, s6, v53661; GFX9-NEXT: v_not_b32_e32 v4, v03662; GFX9-NEXT: v_not_b32_e32 v3, v13663; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] glc3664; GFX9-NEXT: s_waitcnt vmcnt(0)3665; GFX9-NEXT: buffer_wbinvl1_vol3666; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]3667; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]3668; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]3669; GFX9-NEXT: s_cbranch_execnz .LBB56_13670; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end3671; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]3672; GFX9-NEXT: s_setpc_b64 s[30:31]3673 %result = atomicrmw nand ptr addrspace(1) %ptr, i64 %in seq_cst3674 ret i64 %result3675}3676 3677define amdgpu_gfx i64 @global_atomic_nand_i64_ret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {3678; SI-LABEL: global_atomic_nand_i64_ret_offset_scalar:3679; SI: ; %bb.0:3680; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3681; SI-NEXT: s_xor_saveexec_b64 s[34:35], -13682; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill3683; SI-NEXT: s_mov_b64 exec, s[34:35]3684; SI-NEXT: s_waitcnt expcnt(0)3685; SI-NEXT: v_writelane_b32 v6, s6, 03686; SI-NEXT: v_writelane_b32 v6, s7, 13687; SI-NEXT: s_mov_b32 s34, s73688; SI-NEXT: s_mov_b32 s35, s63689; SI-NEXT: s_mov_b32 s7, 0xf0003690; SI-NEXT: s_mov_b32 s6, -13691; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0 offset:323692; SI-NEXT: s_mov_b64 s[36:37], 03693; SI-NEXT: .LBB57_1: ; %atomicrmw.start3694; SI-NEXT: ; =>This Inner Loop Header: Depth=13695; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)3696; SI-NEXT: v_and_b32_e32 v0, s34, v53697; SI-NEXT: v_and_b32_e32 v1, s35, v43698; SI-NEXT: v_not_b32_e32 v3, v03699; SI-NEXT: v_not_b32_e32 v2, v13700; SI-NEXT: v_mov_b32_e32 v0, v23701; SI-NEXT: v_mov_b32_e32 v1, v33702; SI-NEXT: v_mov_b32_e32 v2, v43703; SI-NEXT: v_mov_b32_e32 v3, v53704; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 offset:32 glc3705; SI-NEXT: s_waitcnt vmcnt(0)3706; SI-NEXT: buffer_wbinvl13707; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]3708; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]3709; SI-NEXT: v_mov_b32_e32 v5, v13710; SI-NEXT: v_mov_b32_e32 v4, v03711; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]3712; SI-NEXT: s_cbranch_execnz .LBB57_13713; SI-NEXT: ; %bb.2: ; %atomicrmw.end3714; SI-NEXT: s_or_b64 exec, exec, s[36:37]3715; SI-NEXT: v_readlane_b32 s7, v6, 13716; SI-NEXT: v_readlane_b32 s6, v6, 03717; SI-NEXT: s_xor_saveexec_b64 s[34:35], -13718; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload3719; SI-NEXT: s_mov_b64 exec, s[34:35]3720; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)3721; SI-NEXT: s_setpc_b64 s[30:31]3722;3723; VI-LABEL: global_atomic_nand_i64_ret_offset_scalar:3724; VI: ; %bb.0:3725; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3726; VI-NEXT: s_add_u32 s34, s4, 323727; VI-NEXT: s_addc_u32 s35, s5, 03728; VI-NEXT: v_mov_b32_e32 v2, s343729; VI-NEXT: v_mov_b32_e32 v3, s353730; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]3731; VI-NEXT: s_mov_b64 s[34:35], 03732; VI-NEXT: .LBB57_1: ; %atomicrmw.start3733; VI-NEXT: ; =>This Inner Loop Header: Depth=13734; VI-NEXT: s_waitcnt vmcnt(0)3735; VI-NEXT: v_mov_b32_e32 v7, v13736; VI-NEXT: v_mov_b32_e32 v6, v03737; VI-NEXT: v_and_b32_e32 v0, s7, v73738; VI-NEXT: v_and_b32_e32 v1, s6, v63739; VI-NEXT: v_not_b32_e32 v5, v03740; VI-NEXT: v_not_b32_e32 v4, v13741; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc3742; VI-NEXT: s_waitcnt vmcnt(0)3743; VI-NEXT: buffer_wbinvl1_vol3744; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]3745; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]3746; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]3747; VI-NEXT: s_cbranch_execnz .LBB57_13748; VI-NEXT: ; %bb.2: ; %atomicrmw.end3749; VI-NEXT: s_or_b64 exec, exec, s[34:35]3750; VI-NEXT: s_setpc_b64 s[30:31]3751;3752; GFX9-LABEL: global_atomic_nand_i64_ret_offset_scalar:3753; GFX9: ; %bb.0:3754; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3755; GFX9-NEXT: v_mov_b32_e32 v2, 03756; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5] offset:323757; GFX9-NEXT: s_mov_b64 s[34:35], 03758; GFX9-NEXT: .LBB57_1: ; %atomicrmw.start3759; GFX9-NEXT: ; =>This Inner Loop Header: Depth=13760; GFX9-NEXT: s_waitcnt vmcnt(0)3761; GFX9-NEXT: v_mov_b32_e32 v6, v13762; GFX9-NEXT: v_mov_b32_e32 v5, v03763; GFX9-NEXT: v_and_b32_e32 v0, s7, v63764; GFX9-NEXT: v_and_b32_e32 v1, s6, v53765; GFX9-NEXT: v_not_b32_e32 v4, v03766; GFX9-NEXT: v_not_b32_e32 v3, v13767; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] offset:32 glc3768; GFX9-NEXT: s_waitcnt vmcnt(0)3769; GFX9-NEXT: buffer_wbinvl1_vol3770; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]3771; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]3772; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]3773; GFX9-NEXT: s_cbranch_execnz .LBB57_13774; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end3775; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]3776; GFX9-NEXT: s_setpc_b64 s[30:31]3777 %gep = getelementptr i64, ptr addrspace(1) %out, i64 43778 %result = atomicrmw nand ptr addrspace(1) %gep, i64 %in seq_cst3779 ret i64 %result3780}3781 3782define void @global_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {3783; SI-LABEL: global_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory:3784; SI: ; %bb.0:3785; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3786; SI-NEXT: s_mov_b32 s6, 03787; SI-NEXT: s_mov_b32 s7, 0xf0003788; SI-NEXT: s_mov_b32 s4, s63789; SI-NEXT: s_mov_b32 s5, s63790; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:323791; SI-NEXT: s_mov_b64 s[8:9], 03792; SI-NEXT: .LBB58_1: ; %atomicrmw.start3793; SI-NEXT: ; =>This Inner Loop Header: Depth=13794; SI-NEXT: s_waitcnt vmcnt(0)3795; SI-NEXT: v_and_b32_e32 v4, v7, v33796; SI-NEXT: s_waitcnt expcnt(0)3797; SI-NEXT: v_and_b32_e32 v8, v6, v23798; SI-NEXT: v_not_b32_e32 v5, v43799; SI-NEXT: v_not_b32_e32 v4, v83800; SI-NEXT: v_mov_b32_e32 v11, v73801; SI-NEXT: v_mov_b32_e32 v10, v63802; SI-NEXT: v_mov_b32_e32 v9, v53803; SI-NEXT: v_mov_b32_e32 v8, v43804; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32 glc3805; SI-NEXT: s_waitcnt vmcnt(0)3806; SI-NEXT: buffer_wbinvl13807; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]3808; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]3809; SI-NEXT: v_mov_b32_e32 v6, v83810; SI-NEXT: v_mov_b32_e32 v7, v93811; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]3812; SI-NEXT: s_cbranch_execnz .LBB58_13813; SI-NEXT: ; %bb.2: ; %atomicrmw.end3814; SI-NEXT: s_or_b64 exec, exec, s[8:9]3815; SI-NEXT: s_waitcnt expcnt(0)3816; SI-NEXT: s_setpc_b64 s[30:31]3817;3818; VI-LABEL: global_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory:3819; VI: ; %bb.0:3820; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3821; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v03822; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3823; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]3824; VI-NEXT: s_mov_b64 s[4:5], 03825; VI-NEXT: .LBB58_1: ; %atomicrmw.start3826; VI-NEXT: ; =>This Inner Loop Header: Depth=13827; VI-NEXT: s_waitcnt vmcnt(0)3828; VI-NEXT: v_and_b32_e32 v4, v7, v33829; VI-NEXT: v_and_b32_e32 v8, v6, v23830; VI-NEXT: v_not_b32_e32 v5, v43831; VI-NEXT: v_not_b32_e32 v4, v83832; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc3833; VI-NEXT: s_waitcnt vmcnt(0)3834; VI-NEXT: buffer_wbinvl1_vol3835; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]3836; VI-NEXT: v_mov_b32_e32 v7, v53837; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3838; VI-NEXT: v_mov_b32_e32 v6, v43839; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]3840; VI-NEXT: s_cbranch_execnz .LBB58_13841; VI-NEXT: ; %bb.2: ; %atomicrmw.end3842; VI-NEXT: s_or_b64 exec, exec, s[4:5]3843; VI-NEXT: s_setpc_b64 s[30:31]3844;3845; GFX9-LABEL: global_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory:3846; GFX9: ; %bb.0:3847; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3848; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:323849; GFX9-NEXT: s_mov_b64 s[4:5], 03850; GFX9-NEXT: .LBB58_1: ; %atomicrmw.start3851; GFX9-NEXT: ; =>This Inner Loop Header: Depth=13852; GFX9-NEXT: s_waitcnt vmcnt(0)3853; GFX9-NEXT: v_and_b32_e32 v4, v7, v33854; GFX9-NEXT: v_and_b32_e32 v8, v6, v23855; GFX9-NEXT: v_not_b32_e32 v5, v43856; GFX9-NEXT: v_not_b32_e32 v4, v83857; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc3858; GFX9-NEXT: s_waitcnt vmcnt(0)3859; GFX9-NEXT: buffer_wbinvl1_vol3860; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]3861; GFX9-NEXT: v_mov_b32_e32 v7, v53862; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3863; GFX9-NEXT: v_mov_b32_e32 v6, v43864; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]3865; GFX9-NEXT: s_cbranch_execnz .LBB58_13866; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end3867; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]3868; GFX9-NEXT: s_setpc_b64 s[30:31]3869 %gep = getelementptr i64, ptr addrspace(1) %out, i64 43870 %tmp0 = atomicrmw nand ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !03871 ret void3872}3873 3874define i64 @global_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {3875; SI-LABEL: global_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory:3876; SI: ; %bb.0:3877; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3878; SI-NEXT: v_mov_b32_e32 v6, v33879; SI-NEXT: v_mov_b32_e32 v7, v23880; SI-NEXT: v_mov_b32_e32 v5, v13881; SI-NEXT: v_mov_b32_e32 v4, v03882; SI-NEXT: s_mov_b32 s6, 03883; SI-NEXT: s_mov_b32 s7, 0xf0003884; SI-NEXT: s_mov_b32 s4, s63885; SI-NEXT: s_mov_b32 s5, s63886; SI-NEXT: buffer_load_dwordx2 v[10:11], v[4:5], s[4:7], 0 addr64 offset:323887; SI-NEXT: s_mov_b64 s[8:9], 03888; SI-NEXT: .LBB59_1: ; %atomicrmw.start3889; SI-NEXT: ; =>This Inner Loop Header: Depth=13890; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)3891; SI-NEXT: v_and_b32_e32 v0, v11, v63892; SI-NEXT: v_and_b32_e32 v1, v10, v73893; SI-NEXT: v_not_b32_e32 v9, v03894; SI-NEXT: v_not_b32_e32 v8, v13895; SI-NEXT: v_mov_b32_e32 v0, v83896; SI-NEXT: v_mov_b32_e32 v1, v93897; SI-NEXT: v_mov_b32_e32 v2, v103898; SI-NEXT: v_mov_b32_e32 v3, v113899; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 offset:32 glc3900; SI-NEXT: s_waitcnt vmcnt(0)3901; SI-NEXT: buffer_wbinvl13902; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]3903; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]3904; SI-NEXT: v_mov_b32_e32 v11, v13905; SI-NEXT: v_mov_b32_e32 v10, v03906; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]3907; SI-NEXT: s_cbranch_execnz .LBB59_13908; SI-NEXT: ; %bb.2: ; %atomicrmw.end3909; SI-NEXT: s_or_b64 exec, exec, s[8:9]3910; SI-NEXT: s_waitcnt expcnt(0)3911; SI-NEXT: s_setpc_b64 s[30:31]3912;3913; VI-LABEL: global_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory:3914; VI: ; %bb.0:3915; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3916; VI-NEXT: v_add_u32_e32 v4, vcc, 32, v03917; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc3918; VI-NEXT: flat_load_dwordx2 v[0:1], v[4:5]3919; VI-NEXT: s_mov_b64 s[4:5], 03920; VI-NEXT: .LBB59_1: ; %atomicrmw.start3921; VI-NEXT: ; =>This Inner Loop Header: Depth=13922; VI-NEXT: s_waitcnt vmcnt(0)3923; VI-NEXT: v_mov_b32_e32 v9, v13924; VI-NEXT: v_mov_b32_e32 v8, v03925; VI-NEXT: v_and_b32_e32 v0, v9, v33926; VI-NEXT: v_and_b32_e32 v1, v8, v23927; VI-NEXT: v_not_b32_e32 v7, v03928; VI-NEXT: v_not_b32_e32 v6, v13929; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc3930; VI-NEXT: s_waitcnt vmcnt(0)3931; VI-NEXT: buffer_wbinvl1_vol3932; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]3933; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3934; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]3935; VI-NEXT: s_cbranch_execnz .LBB59_13936; VI-NEXT: ; %bb.2: ; %atomicrmw.end3937; VI-NEXT: s_or_b64 exec, exec, s[4:5]3938; VI-NEXT: s_setpc_b64 s[30:31]3939;3940; GFX9-LABEL: global_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory:3941; GFX9: ; %bb.0:3942; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3943; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:323944; GFX9-NEXT: s_mov_b64 s[4:5], 03945; GFX9-NEXT: .LBB59_1: ; %atomicrmw.start3946; GFX9-NEXT: ; =>This Inner Loop Header: Depth=13947; GFX9-NEXT: s_waitcnt vmcnt(0)3948; GFX9-NEXT: v_mov_b32_e32 v7, v53949; GFX9-NEXT: v_mov_b32_e32 v6, v43950; GFX9-NEXT: v_and_b32_e32 v4, v7, v33951; GFX9-NEXT: v_and_b32_e32 v8, v6, v23952; GFX9-NEXT: v_not_b32_e32 v5, v43953; GFX9-NEXT: v_not_b32_e32 v4, v83954; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc3955; GFX9-NEXT: s_waitcnt vmcnt(0)3956; GFX9-NEXT: buffer_wbinvl1_vol3957; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]3958; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3959; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]3960; GFX9-NEXT: s_cbranch_execnz .LBB59_13961; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end3962; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]3963; GFX9-NEXT: v_mov_b32_e32 v0, v43964; GFX9-NEXT: v_mov_b32_e32 v1, v53965; GFX9-NEXT: s_setpc_b64 s[30:31]3966 %gep = getelementptr i64, ptr addrspace(1) %out, i64 43967 %result = atomicrmw nand ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !03968 ret i64 %result3969}3970 3971; ---------------------------------------------------------------------3972; atomicrmw or3973; ---------------------------------------------------------------------3974 3975define void @global_atomic_or_i64_noret(ptr addrspace(1) %ptr, i64 %in) {3976; SI-LABEL: global_atomic_or_i64_noret:3977; SI: ; %bb.0:3978; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3979; SI-NEXT: s_mov_b32 s6, 03980; SI-NEXT: s_mov_b32 s7, 0xf0003981; SI-NEXT: s_mov_b32 s4, s63982; SI-NEXT: s_mov_b32 s5, s63983; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr643984; SI-NEXT: s_mov_b64 s[8:9], 03985; SI-NEXT: .LBB60_1: ; %atomicrmw.start3986; SI-NEXT: ; =>This Inner Loop Header: Depth=13987; SI-NEXT: s_waitcnt vmcnt(0)3988; SI-NEXT: v_or_b32_e32 v5, v7, v33989; SI-NEXT: v_or_b32_e32 v4, v6, v23990; SI-NEXT: s_waitcnt expcnt(0)3991; SI-NEXT: v_mov_b32_e32 v11, v73992; SI-NEXT: v_mov_b32_e32 v10, v63993; SI-NEXT: v_mov_b32_e32 v9, v53994; SI-NEXT: v_mov_b32_e32 v8, v43995; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc3996; SI-NEXT: s_waitcnt vmcnt(0)3997; SI-NEXT: buffer_wbinvl13998; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]3999; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]4000; SI-NEXT: v_mov_b32_e32 v6, v84001; SI-NEXT: v_mov_b32_e32 v7, v94002; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]4003; SI-NEXT: s_cbranch_execnz .LBB60_14004; SI-NEXT: ; %bb.2: ; %atomicrmw.end4005; SI-NEXT: s_or_b64 exec, exec, s[8:9]4006; SI-NEXT: s_waitcnt expcnt(0)4007; SI-NEXT: s_setpc_b64 s[30:31]4008;4009; VI-LABEL: global_atomic_or_i64_noret:4010; VI: ; %bb.0:4011; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4012; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]4013; VI-NEXT: s_mov_b64 s[4:5], 04014; VI-NEXT: .LBB60_1: ; %atomicrmw.start4015; VI-NEXT: ; =>This Inner Loop Header: Depth=14016; VI-NEXT: s_waitcnt vmcnt(0)4017; VI-NEXT: v_or_b32_e32 v5, v7, v34018; VI-NEXT: v_or_b32_e32 v4, v6, v24019; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc4020; VI-NEXT: s_waitcnt vmcnt(0)4021; VI-NEXT: buffer_wbinvl1_vol4022; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]4023; VI-NEXT: v_mov_b32_e32 v7, v54024; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4025; VI-NEXT: v_mov_b32_e32 v6, v44026; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]4027; VI-NEXT: s_cbranch_execnz .LBB60_14028; VI-NEXT: ; %bb.2: ; %atomicrmw.end4029; VI-NEXT: s_or_b64 exec, exec, s[4:5]4030; VI-NEXT: s_setpc_b64 s[30:31]4031;4032; GFX9-LABEL: global_atomic_or_i64_noret:4033; GFX9: ; %bb.0:4034; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4035; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off4036; GFX9-NEXT: s_mov_b64 s[4:5], 04037; GFX9-NEXT: .LBB60_1: ; %atomicrmw.start4038; GFX9-NEXT: ; =>This Inner Loop Header: Depth=14039; GFX9-NEXT: s_waitcnt vmcnt(0)4040; GFX9-NEXT: v_or_b32_e32 v5, v7, v34041; GFX9-NEXT: v_or_b32_e32 v4, v6, v24042; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc4043; GFX9-NEXT: s_waitcnt vmcnt(0)4044; GFX9-NEXT: buffer_wbinvl1_vol4045; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]4046; GFX9-NEXT: v_mov_b32_e32 v7, v54047; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4048; GFX9-NEXT: v_mov_b32_e32 v6, v44049; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]4050; GFX9-NEXT: s_cbranch_execnz .LBB60_14051; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end4052; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]4053; GFX9-NEXT: s_setpc_b64 s[30:31]4054 %tmp0 = atomicrmw or ptr addrspace(1) %ptr, i64 %in seq_cst4055 ret void4056}4057 4058define void @global_atomic_or_i64_noret_offset(ptr addrspace(1) %out, i64 %in) {4059; SI-LABEL: global_atomic_or_i64_noret_offset:4060; SI: ; %bb.0:4061; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4062; SI-NEXT: s_mov_b32 s6, 04063; SI-NEXT: s_mov_b32 s7, 0xf0004064; SI-NEXT: s_mov_b32 s4, s64065; SI-NEXT: s_mov_b32 s5, s64066; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:324067; SI-NEXT: s_mov_b64 s[8:9], 04068; SI-NEXT: .LBB61_1: ; %atomicrmw.start4069; SI-NEXT: ; =>This Inner Loop Header: Depth=14070; SI-NEXT: s_waitcnt vmcnt(0)4071; SI-NEXT: v_or_b32_e32 v5, v7, v34072; SI-NEXT: v_or_b32_e32 v4, v6, v24073; SI-NEXT: s_waitcnt expcnt(0)4074; SI-NEXT: v_mov_b32_e32 v11, v74075; SI-NEXT: v_mov_b32_e32 v10, v64076; SI-NEXT: v_mov_b32_e32 v9, v54077; SI-NEXT: v_mov_b32_e32 v8, v44078; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32 glc4079; SI-NEXT: s_waitcnt vmcnt(0)4080; SI-NEXT: buffer_wbinvl14081; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]4082; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]4083; SI-NEXT: v_mov_b32_e32 v6, v84084; SI-NEXT: v_mov_b32_e32 v7, v94085; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]4086; SI-NEXT: s_cbranch_execnz .LBB61_14087; SI-NEXT: ; %bb.2: ; %atomicrmw.end4088; SI-NEXT: s_or_b64 exec, exec, s[8:9]4089; SI-NEXT: s_waitcnt expcnt(0)4090; SI-NEXT: s_setpc_b64 s[30:31]4091;4092; VI-LABEL: global_atomic_or_i64_noret_offset:4093; VI: ; %bb.0:4094; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4095; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v04096; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc4097; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]4098; VI-NEXT: s_mov_b64 s[4:5], 04099; VI-NEXT: .LBB61_1: ; %atomicrmw.start4100; VI-NEXT: ; =>This Inner Loop Header: Depth=14101; VI-NEXT: s_waitcnt vmcnt(0)4102; VI-NEXT: v_or_b32_e32 v5, v7, v34103; VI-NEXT: v_or_b32_e32 v4, v6, v24104; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc4105; VI-NEXT: s_waitcnt vmcnt(0)4106; VI-NEXT: buffer_wbinvl1_vol4107; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]4108; VI-NEXT: v_mov_b32_e32 v7, v54109; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4110; VI-NEXT: v_mov_b32_e32 v6, v44111; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]4112; VI-NEXT: s_cbranch_execnz .LBB61_14113; VI-NEXT: ; %bb.2: ; %atomicrmw.end4114; VI-NEXT: s_or_b64 exec, exec, s[4:5]4115; VI-NEXT: s_setpc_b64 s[30:31]4116;4117; GFX9-LABEL: global_atomic_or_i64_noret_offset:4118; GFX9: ; %bb.0:4119; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4120; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:324121; GFX9-NEXT: s_mov_b64 s[4:5], 04122; GFX9-NEXT: .LBB61_1: ; %atomicrmw.start4123; GFX9-NEXT: ; =>This Inner Loop Header: Depth=14124; GFX9-NEXT: s_waitcnt vmcnt(0)4125; GFX9-NEXT: v_or_b32_e32 v5, v7, v34126; GFX9-NEXT: v_or_b32_e32 v4, v6, v24127; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc4128; GFX9-NEXT: s_waitcnt vmcnt(0)4129; GFX9-NEXT: buffer_wbinvl1_vol4130; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]4131; GFX9-NEXT: v_mov_b32_e32 v7, v54132; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4133; GFX9-NEXT: v_mov_b32_e32 v6, v44134; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]4135; GFX9-NEXT: s_cbranch_execnz .LBB61_14136; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end4137; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]4138; GFX9-NEXT: s_setpc_b64 s[30:31]4139 %gep = getelementptr i64, ptr addrspace(1) %out, i64 44140 %tmp0 = atomicrmw or ptr addrspace(1) %gep, i64 %in seq_cst4141 ret void4142}4143 4144define i64 @global_atomic_or_i64_ret(ptr addrspace(1) %ptr, i64 %in) {4145; SI-LABEL: global_atomic_or_i64_ret:4146; SI: ; %bb.0:4147; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4148; SI-NEXT: v_mov_b32_e32 v6, v34149; SI-NEXT: v_mov_b32_e32 v7, v24150; SI-NEXT: v_mov_b32_e32 v5, v14151; SI-NEXT: v_mov_b32_e32 v4, v04152; SI-NEXT: s_mov_b32 s6, 04153; SI-NEXT: s_mov_b32 s7, 0xf0004154; SI-NEXT: s_mov_b32 s4, s64155; SI-NEXT: s_mov_b32 s5, s64156; SI-NEXT: buffer_load_dwordx2 v[10:11], v[4:5], s[4:7], 0 addr644157; SI-NEXT: s_mov_b64 s[8:9], 04158; SI-NEXT: .LBB62_1: ; %atomicrmw.start4159; SI-NEXT: ; =>This Inner Loop Header: Depth=14160; SI-NEXT: s_waitcnt vmcnt(0)4161; SI-NEXT: v_or_b32_e32 v9, v11, v64162; SI-NEXT: v_or_b32_e32 v8, v10, v74163; SI-NEXT: s_waitcnt expcnt(0)4164; SI-NEXT: v_mov_b32_e32 v0, v84165; SI-NEXT: v_mov_b32_e32 v1, v94166; SI-NEXT: v_mov_b32_e32 v2, v104167; SI-NEXT: v_mov_b32_e32 v3, v114168; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 glc4169; SI-NEXT: s_waitcnt vmcnt(0)4170; SI-NEXT: buffer_wbinvl14171; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]4172; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]4173; SI-NEXT: v_mov_b32_e32 v11, v14174; SI-NEXT: v_mov_b32_e32 v10, v04175; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]4176; SI-NEXT: s_cbranch_execnz .LBB62_14177; SI-NEXT: ; %bb.2: ; %atomicrmw.end4178; SI-NEXT: s_or_b64 exec, exec, s[8:9]4179; SI-NEXT: s_waitcnt expcnt(0)4180; SI-NEXT: s_setpc_b64 s[30:31]4181;4182; VI-LABEL: global_atomic_or_i64_ret:4183; VI: ; %bb.0:4184; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4185; VI-NEXT: flat_load_dwordx2 v[4:5], v[0:1]4186; VI-NEXT: s_mov_b64 s[4:5], 04187; VI-NEXT: .LBB62_1: ; %atomicrmw.start4188; VI-NEXT: ; =>This Inner Loop Header: Depth=14189; VI-NEXT: s_waitcnt vmcnt(0)4190; VI-NEXT: v_mov_b32_e32 v7, v54191; VI-NEXT: v_mov_b32_e32 v6, v44192; VI-NEXT: v_or_b32_e32 v5, v7, v34193; VI-NEXT: v_or_b32_e32 v4, v6, v24194; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc4195; VI-NEXT: s_waitcnt vmcnt(0)4196; VI-NEXT: buffer_wbinvl1_vol4197; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]4198; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4199; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]4200; VI-NEXT: s_cbranch_execnz .LBB62_14201; VI-NEXT: ; %bb.2: ; %atomicrmw.end4202; VI-NEXT: s_or_b64 exec, exec, s[4:5]4203; VI-NEXT: v_mov_b32_e32 v0, v44204; VI-NEXT: v_mov_b32_e32 v1, v54205; VI-NEXT: s_setpc_b64 s[30:31]4206;4207; GFX9-LABEL: global_atomic_or_i64_ret:4208; GFX9: ; %bb.0:4209; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4210; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off4211; GFX9-NEXT: s_mov_b64 s[4:5], 04212; GFX9-NEXT: .LBB62_1: ; %atomicrmw.start4213; GFX9-NEXT: ; =>This Inner Loop Header: Depth=14214; GFX9-NEXT: s_waitcnt vmcnt(0)4215; GFX9-NEXT: v_mov_b32_e32 v7, v54216; GFX9-NEXT: v_mov_b32_e32 v6, v44217; GFX9-NEXT: v_or_b32_e32 v5, v7, v34218; GFX9-NEXT: v_or_b32_e32 v4, v6, v24219; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc4220; GFX9-NEXT: s_waitcnt vmcnt(0)4221; GFX9-NEXT: buffer_wbinvl1_vol4222; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]4223; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4224; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]4225; GFX9-NEXT: s_cbranch_execnz .LBB62_14226; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end4227; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]4228; GFX9-NEXT: v_mov_b32_e32 v0, v44229; GFX9-NEXT: v_mov_b32_e32 v1, v54230; GFX9-NEXT: s_setpc_b64 s[30:31]4231 %result = atomicrmw or ptr addrspace(1) %ptr, i64 %in seq_cst4232 ret i64 %result4233}4234 4235define i64 @global_atomic_or_i64_ret_offset(ptr addrspace(1) %out, i64 %in) {4236; SI-LABEL: global_atomic_or_i64_ret_offset:4237; SI: ; %bb.0:4238; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4239; SI-NEXT: v_mov_b32_e32 v6, v34240; SI-NEXT: v_mov_b32_e32 v7, v24241; SI-NEXT: v_mov_b32_e32 v5, v14242; SI-NEXT: v_mov_b32_e32 v4, v04243; SI-NEXT: s_mov_b32 s6, 04244; SI-NEXT: s_mov_b32 s7, 0xf0004245; SI-NEXT: s_mov_b32 s4, s64246; SI-NEXT: s_mov_b32 s5, s64247; SI-NEXT: buffer_load_dwordx2 v[10:11], v[4:5], s[4:7], 0 addr64 offset:324248; SI-NEXT: s_mov_b64 s[8:9], 04249; SI-NEXT: .LBB63_1: ; %atomicrmw.start4250; SI-NEXT: ; =>This Inner Loop Header: Depth=14251; SI-NEXT: s_waitcnt vmcnt(0)4252; SI-NEXT: v_or_b32_e32 v9, v11, v64253; SI-NEXT: v_or_b32_e32 v8, v10, v74254; SI-NEXT: s_waitcnt expcnt(0)4255; SI-NEXT: v_mov_b32_e32 v0, v84256; SI-NEXT: v_mov_b32_e32 v1, v94257; SI-NEXT: v_mov_b32_e32 v2, v104258; SI-NEXT: v_mov_b32_e32 v3, v114259; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 offset:32 glc4260; SI-NEXT: s_waitcnt vmcnt(0)4261; SI-NEXT: buffer_wbinvl14262; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]4263; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]4264; SI-NEXT: v_mov_b32_e32 v11, v14265; SI-NEXT: v_mov_b32_e32 v10, v04266; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]4267; SI-NEXT: s_cbranch_execnz .LBB63_14268; SI-NEXT: ; %bb.2: ; %atomicrmw.end4269; SI-NEXT: s_or_b64 exec, exec, s[8:9]4270; SI-NEXT: s_waitcnt expcnt(0)4271; SI-NEXT: s_setpc_b64 s[30:31]4272;4273; VI-LABEL: global_atomic_or_i64_ret_offset:4274; VI: ; %bb.0:4275; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4276; VI-NEXT: v_add_u32_e32 v4, vcc, 32, v04277; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc4278; VI-NEXT: flat_load_dwordx2 v[0:1], v[4:5]4279; VI-NEXT: s_mov_b64 s[4:5], 04280; VI-NEXT: .LBB63_1: ; %atomicrmw.start4281; VI-NEXT: ; =>This Inner Loop Header: Depth=14282; VI-NEXT: s_waitcnt vmcnt(0)4283; VI-NEXT: v_mov_b32_e32 v9, v14284; VI-NEXT: v_mov_b32_e32 v8, v04285; VI-NEXT: v_or_b32_e32 v7, v9, v34286; VI-NEXT: v_or_b32_e32 v6, v8, v24287; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc4288; VI-NEXT: s_waitcnt vmcnt(0)4289; VI-NEXT: buffer_wbinvl1_vol4290; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]4291; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4292; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]4293; VI-NEXT: s_cbranch_execnz .LBB63_14294; VI-NEXT: ; %bb.2: ; %atomicrmw.end4295; VI-NEXT: s_or_b64 exec, exec, s[4:5]4296; VI-NEXT: s_setpc_b64 s[30:31]4297;4298; GFX9-LABEL: global_atomic_or_i64_ret_offset:4299; GFX9: ; %bb.0:4300; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4301; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:324302; GFX9-NEXT: s_mov_b64 s[4:5], 04303; GFX9-NEXT: .LBB63_1: ; %atomicrmw.start4304; GFX9-NEXT: ; =>This Inner Loop Header: Depth=14305; GFX9-NEXT: s_waitcnt vmcnt(0)4306; GFX9-NEXT: v_mov_b32_e32 v7, v54307; GFX9-NEXT: v_mov_b32_e32 v6, v44308; GFX9-NEXT: v_or_b32_e32 v5, v7, v34309; GFX9-NEXT: v_or_b32_e32 v4, v6, v24310; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc4311; GFX9-NEXT: s_waitcnt vmcnt(0)4312; GFX9-NEXT: buffer_wbinvl1_vol4313; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]4314; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4315; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]4316; GFX9-NEXT: s_cbranch_execnz .LBB63_14317; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end4318; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]4319; GFX9-NEXT: v_mov_b32_e32 v0, v44320; GFX9-NEXT: v_mov_b32_e32 v1, v54321; GFX9-NEXT: s_setpc_b64 s[30:31]4322 %gep = getelementptr i64, ptr addrspace(1) %out, i64 44323 %result = atomicrmw or ptr addrspace(1) %gep, i64 %in seq_cst4324 ret i64 %result4325}4326 4327define amdgpu_gfx void @global_atomic_or_i64_noret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {4328; SI-LABEL: global_atomic_or_i64_noret_scalar:4329; SI: ; %bb.0:4330; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4331; SI-NEXT: s_xor_saveexec_b64 s[34:35], -14332; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill4333; SI-NEXT: s_mov_b64 exec, s[34:35]4334; SI-NEXT: s_waitcnt expcnt(0)4335; SI-NEXT: v_writelane_b32 v8, s6, 04336; SI-NEXT: v_writelane_b32 v8, s7, 14337; SI-NEXT: s_mov_b32 s34, s74338; SI-NEXT: s_mov_b32 s35, s64339; SI-NEXT: s_mov_b32 s7, 0xf0004340; SI-NEXT: s_mov_b32 s6, -14341; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 04342; SI-NEXT: s_mov_b64 s[36:37], 04343; SI-NEXT: .LBB64_1: ; %atomicrmw.start4344; SI-NEXT: ; =>This Inner Loop Header: Depth=14345; SI-NEXT: s_waitcnt vmcnt(0)4346; SI-NEXT: v_or_b32_e32 v1, s34, v34347; SI-NEXT: v_or_b32_e32 v0, s35, v24348; SI-NEXT: s_waitcnt expcnt(0)4349; SI-NEXT: v_mov_b32_e32 v7, v34350; SI-NEXT: v_mov_b32_e32 v6, v24351; SI-NEXT: v_mov_b32_e32 v5, v14352; SI-NEXT: v_mov_b32_e32 v4, v04353; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 glc4354; SI-NEXT: s_waitcnt vmcnt(0)4355; SI-NEXT: buffer_wbinvl14356; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]4357; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]4358; SI-NEXT: v_mov_b32_e32 v2, v44359; SI-NEXT: v_mov_b32_e32 v3, v54360; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]4361; SI-NEXT: s_cbranch_execnz .LBB64_14362; SI-NEXT: ; %bb.2: ; %atomicrmw.end4363; SI-NEXT: s_or_b64 exec, exec, s[36:37]4364; SI-NEXT: v_readlane_b32 s7, v8, 14365; SI-NEXT: v_readlane_b32 s6, v8, 04366; SI-NEXT: s_xor_saveexec_b64 s[34:35], -14367; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload4368; SI-NEXT: s_mov_b64 exec, s[34:35]4369; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)4370; SI-NEXT: s_setpc_b64 s[30:31]4371;4372; VI-LABEL: global_atomic_or_i64_noret_scalar:4373; VI: ; %bb.0:4374; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4375; VI-NEXT: v_mov_b32_e32 v0, s44376; VI-NEXT: v_mov_b32_e32 v1, s54377; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]4378; VI-NEXT: v_mov_b32_e32 v4, s44379; VI-NEXT: s_mov_b64 s[34:35], 04380; VI-NEXT: v_mov_b32_e32 v5, s54381; VI-NEXT: .LBB64_1: ; %atomicrmw.start4382; VI-NEXT: ; =>This Inner Loop Header: Depth=14383; VI-NEXT: s_waitcnt vmcnt(0)4384; VI-NEXT: v_or_b32_e32 v1, s7, v34385; VI-NEXT: v_or_b32_e32 v0, s6, v24386; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4387; VI-NEXT: s_waitcnt vmcnt(0)4388; VI-NEXT: buffer_wbinvl1_vol4389; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4390; VI-NEXT: v_mov_b32_e32 v3, v14391; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4392; VI-NEXT: v_mov_b32_e32 v2, v04393; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]4394; VI-NEXT: s_cbranch_execnz .LBB64_14395; VI-NEXT: ; %bb.2: ; %atomicrmw.end4396; VI-NEXT: s_or_b64 exec, exec, s[34:35]4397; VI-NEXT: s_setpc_b64 s[30:31]4398;4399; GFX9-LABEL: global_atomic_or_i64_noret_scalar:4400; GFX9: ; %bb.0:4401; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4402; GFX9-NEXT: v_mov_b32_e32 v4, 04403; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5]4404; GFX9-NEXT: s_mov_b64 s[34:35], 04405; GFX9-NEXT: .LBB64_1: ; %atomicrmw.start4406; GFX9-NEXT: ; =>This Inner Loop Header: Depth=14407; GFX9-NEXT: s_waitcnt vmcnt(0)4408; GFX9-NEXT: v_or_b32_e32 v1, s7, v34409; GFX9-NEXT: v_or_b32_e32 v0, s6, v24410; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc4411; GFX9-NEXT: s_waitcnt vmcnt(0)4412; GFX9-NEXT: buffer_wbinvl1_vol4413; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4414; GFX9-NEXT: v_mov_b32_e32 v3, v14415; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4416; GFX9-NEXT: v_mov_b32_e32 v2, v04417; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]4418; GFX9-NEXT: s_cbranch_execnz .LBB64_14419; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end4420; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]4421; GFX9-NEXT: s_setpc_b64 s[30:31]4422 %tmp0 = atomicrmw or ptr addrspace(1) %ptr, i64 %in seq_cst4423 ret void4424}4425 4426define amdgpu_gfx void @global_atomic_or_i64_noret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {4427; SI-LABEL: global_atomic_or_i64_noret_offset_scalar:4428; SI: ; %bb.0:4429; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4430; SI-NEXT: s_xor_saveexec_b64 s[34:35], -14431; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill4432; SI-NEXT: s_mov_b64 exec, s[34:35]4433; SI-NEXT: s_waitcnt expcnt(0)4434; SI-NEXT: v_writelane_b32 v8, s6, 04435; SI-NEXT: v_writelane_b32 v8, s7, 14436; SI-NEXT: s_mov_b32 s34, s74437; SI-NEXT: s_mov_b32 s35, s64438; SI-NEXT: s_mov_b32 s7, 0xf0004439; SI-NEXT: s_mov_b32 s6, -14440; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:324441; SI-NEXT: s_mov_b64 s[36:37], 04442; SI-NEXT: .LBB65_1: ; %atomicrmw.start4443; SI-NEXT: ; =>This Inner Loop Header: Depth=14444; SI-NEXT: s_waitcnt vmcnt(0)4445; SI-NEXT: v_or_b32_e32 v1, s34, v34446; SI-NEXT: v_or_b32_e32 v0, s35, v24447; SI-NEXT: s_waitcnt expcnt(0)4448; SI-NEXT: v_mov_b32_e32 v7, v34449; SI-NEXT: v_mov_b32_e32 v6, v24450; SI-NEXT: v_mov_b32_e32 v5, v14451; SI-NEXT: v_mov_b32_e32 v4, v04452; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 offset:32 glc4453; SI-NEXT: s_waitcnt vmcnt(0)4454; SI-NEXT: buffer_wbinvl14455; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]4456; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]4457; SI-NEXT: v_mov_b32_e32 v2, v44458; SI-NEXT: v_mov_b32_e32 v3, v54459; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]4460; SI-NEXT: s_cbranch_execnz .LBB65_14461; SI-NEXT: ; %bb.2: ; %atomicrmw.end4462; SI-NEXT: s_or_b64 exec, exec, s[36:37]4463; SI-NEXT: v_readlane_b32 s7, v8, 14464; SI-NEXT: v_readlane_b32 s6, v8, 04465; SI-NEXT: s_xor_saveexec_b64 s[34:35], -14466; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload4467; SI-NEXT: s_mov_b64 exec, s[34:35]4468; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)4469; SI-NEXT: s_setpc_b64 s[30:31]4470;4471; VI-LABEL: global_atomic_or_i64_noret_offset_scalar:4472; VI: ; %bb.0:4473; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4474; VI-NEXT: s_add_u32 s34, s4, 324475; VI-NEXT: s_addc_u32 s35, s5, 04476; VI-NEXT: v_mov_b32_e32 v4, s344477; VI-NEXT: v_mov_b32_e32 v5, s354478; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]4479; VI-NEXT: s_mov_b64 s[34:35], 04480; VI-NEXT: .LBB65_1: ; %atomicrmw.start4481; VI-NEXT: ; =>This Inner Loop Header: Depth=14482; VI-NEXT: s_waitcnt vmcnt(0)4483; VI-NEXT: v_or_b32_e32 v1, s7, v34484; VI-NEXT: v_or_b32_e32 v0, s6, v24485; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4486; VI-NEXT: s_waitcnt vmcnt(0)4487; VI-NEXT: buffer_wbinvl1_vol4488; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4489; VI-NEXT: v_mov_b32_e32 v3, v14490; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4491; VI-NEXT: v_mov_b32_e32 v2, v04492; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]4493; VI-NEXT: s_cbranch_execnz .LBB65_14494; VI-NEXT: ; %bb.2: ; %atomicrmw.end4495; VI-NEXT: s_or_b64 exec, exec, s[34:35]4496; VI-NEXT: s_setpc_b64 s[30:31]4497;4498; GFX9-LABEL: global_atomic_or_i64_noret_offset_scalar:4499; GFX9: ; %bb.0:4500; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4501; GFX9-NEXT: v_mov_b32_e32 v4, 04502; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5] offset:324503; GFX9-NEXT: s_mov_b64 s[34:35], 04504; GFX9-NEXT: .LBB65_1: ; %atomicrmw.start4505; GFX9-NEXT: ; =>This Inner Loop Header: Depth=14506; GFX9-NEXT: s_waitcnt vmcnt(0)4507; GFX9-NEXT: v_or_b32_e32 v1, s7, v34508; GFX9-NEXT: v_or_b32_e32 v0, s6, v24509; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc4510; GFX9-NEXT: s_waitcnt vmcnt(0)4511; GFX9-NEXT: buffer_wbinvl1_vol4512; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4513; GFX9-NEXT: v_mov_b32_e32 v3, v14514; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4515; GFX9-NEXT: v_mov_b32_e32 v2, v04516; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]4517; GFX9-NEXT: s_cbranch_execnz .LBB65_14518; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end4519; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]4520; GFX9-NEXT: s_setpc_b64 s[30:31]4521 %gep = getelementptr i64, ptr addrspace(1) %out, i64 44522 %tmp0 = atomicrmw or ptr addrspace(1) %gep, i64 %in seq_cst4523 ret void4524}4525 4526define amdgpu_gfx i64 @global_atomic_or_i64_ret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {4527; SI-LABEL: global_atomic_or_i64_ret_scalar:4528; SI: ; %bb.0:4529; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4530; SI-NEXT: s_xor_saveexec_b64 s[34:35], -14531; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill4532; SI-NEXT: s_mov_b64 exec, s[34:35]4533; SI-NEXT: s_waitcnt expcnt(0)4534; SI-NEXT: v_writelane_b32 v6, s6, 04535; SI-NEXT: v_writelane_b32 v6, s7, 14536; SI-NEXT: s_mov_b32 s34, s74537; SI-NEXT: s_mov_b32 s35, s64538; SI-NEXT: s_mov_b32 s7, 0xf0004539; SI-NEXT: s_mov_b32 s6, -14540; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 04541; SI-NEXT: s_mov_b64 s[36:37], 04542; SI-NEXT: .LBB66_1: ; %atomicrmw.start4543; SI-NEXT: ; =>This Inner Loop Header: Depth=14544; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)4545; SI-NEXT: v_or_b32_e32 v3, s34, v54546; SI-NEXT: v_or_b32_e32 v2, s35, v44547; SI-NEXT: v_mov_b32_e32 v0, v24548; SI-NEXT: v_mov_b32_e32 v1, v34549; SI-NEXT: v_mov_b32_e32 v2, v44550; SI-NEXT: v_mov_b32_e32 v3, v54551; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc4552; SI-NEXT: s_waitcnt vmcnt(0)4553; SI-NEXT: buffer_wbinvl14554; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]4555; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]4556; SI-NEXT: v_mov_b32_e32 v5, v14557; SI-NEXT: v_mov_b32_e32 v4, v04558; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]4559; SI-NEXT: s_cbranch_execnz .LBB66_14560; SI-NEXT: ; %bb.2: ; %atomicrmw.end4561; SI-NEXT: s_or_b64 exec, exec, s[36:37]4562; SI-NEXT: v_readlane_b32 s7, v6, 14563; SI-NEXT: v_readlane_b32 s6, v6, 04564; SI-NEXT: s_xor_saveexec_b64 s[34:35], -14565; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload4566; SI-NEXT: s_mov_b64 exec, s[34:35]4567; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)4568; SI-NEXT: s_setpc_b64 s[30:31]4569;4570; VI-LABEL: global_atomic_or_i64_ret_scalar:4571; VI: ; %bb.0:4572; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4573; VI-NEXT: v_mov_b32_e32 v0, s44574; VI-NEXT: v_mov_b32_e32 v1, s54575; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]4576; VI-NEXT: v_mov_b32_e32 v2, s44577; VI-NEXT: s_mov_b64 s[34:35], 04578; VI-NEXT: v_mov_b32_e32 v3, s54579; VI-NEXT: .LBB66_1: ; %atomicrmw.start4580; VI-NEXT: ; =>This Inner Loop Header: Depth=14581; VI-NEXT: s_waitcnt vmcnt(0)4582; VI-NEXT: v_mov_b32_e32 v7, v14583; VI-NEXT: v_mov_b32_e32 v6, v04584; VI-NEXT: v_or_b32_e32 v5, s7, v74585; VI-NEXT: v_or_b32_e32 v4, s6, v64586; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc4587; VI-NEXT: s_waitcnt vmcnt(0)4588; VI-NEXT: buffer_wbinvl1_vol4589; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]4590; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4591; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]4592; VI-NEXT: s_cbranch_execnz .LBB66_14593; VI-NEXT: ; %bb.2: ; %atomicrmw.end4594; VI-NEXT: s_or_b64 exec, exec, s[34:35]4595; VI-NEXT: s_setpc_b64 s[30:31]4596;4597; GFX9-LABEL: global_atomic_or_i64_ret_scalar:4598; GFX9: ; %bb.0:4599; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4600; GFX9-NEXT: v_mov_b32_e32 v2, 04601; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]4602; GFX9-NEXT: s_mov_b64 s[34:35], 04603; GFX9-NEXT: .LBB66_1: ; %atomicrmw.start4604; GFX9-NEXT: ; =>This Inner Loop Header: Depth=14605; GFX9-NEXT: s_waitcnt vmcnt(0)4606; GFX9-NEXT: v_mov_b32_e32 v6, v14607; GFX9-NEXT: v_mov_b32_e32 v5, v04608; GFX9-NEXT: v_or_b32_e32 v4, s7, v64609; GFX9-NEXT: v_or_b32_e32 v3, s6, v54610; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] glc4611; GFX9-NEXT: s_waitcnt vmcnt(0)4612; GFX9-NEXT: buffer_wbinvl1_vol4613; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]4614; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4615; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]4616; GFX9-NEXT: s_cbranch_execnz .LBB66_14617; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end4618; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]4619; GFX9-NEXT: s_setpc_b64 s[30:31]4620 %result = atomicrmw or ptr addrspace(1) %ptr, i64 %in seq_cst4621 ret i64 %result4622}4623 4624define amdgpu_gfx i64 @global_atomic_or_i64_ret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {4625; SI-LABEL: global_atomic_or_i64_ret_offset_scalar:4626; SI: ; %bb.0:4627; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4628; SI-NEXT: s_xor_saveexec_b64 s[34:35], -14629; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill4630; SI-NEXT: s_mov_b64 exec, s[34:35]4631; SI-NEXT: s_waitcnt expcnt(0)4632; SI-NEXT: v_writelane_b32 v6, s6, 04633; SI-NEXT: v_writelane_b32 v6, s7, 14634; SI-NEXT: s_mov_b32 s34, s74635; SI-NEXT: s_mov_b32 s35, s64636; SI-NEXT: s_mov_b32 s7, 0xf0004637; SI-NEXT: s_mov_b32 s6, -14638; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0 offset:324639; SI-NEXT: s_mov_b64 s[36:37], 04640; SI-NEXT: .LBB67_1: ; %atomicrmw.start4641; SI-NEXT: ; =>This Inner Loop Header: Depth=14642; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)4643; SI-NEXT: v_or_b32_e32 v3, s34, v54644; SI-NEXT: v_or_b32_e32 v2, s35, v44645; SI-NEXT: v_mov_b32_e32 v0, v24646; SI-NEXT: v_mov_b32_e32 v1, v34647; SI-NEXT: v_mov_b32_e32 v2, v44648; SI-NEXT: v_mov_b32_e32 v3, v54649; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 offset:32 glc4650; SI-NEXT: s_waitcnt vmcnt(0)4651; SI-NEXT: buffer_wbinvl14652; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]4653; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]4654; SI-NEXT: v_mov_b32_e32 v5, v14655; SI-NEXT: v_mov_b32_e32 v4, v04656; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]4657; SI-NEXT: s_cbranch_execnz .LBB67_14658; SI-NEXT: ; %bb.2: ; %atomicrmw.end4659; SI-NEXT: s_or_b64 exec, exec, s[36:37]4660; SI-NEXT: v_readlane_b32 s7, v6, 14661; SI-NEXT: v_readlane_b32 s6, v6, 04662; SI-NEXT: s_xor_saveexec_b64 s[34:35], -14663; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload4664; SI-NEXT: s_mov_b64 exec, s[34:35]4665; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)4666; SI-NEXT: s_setpc_b64 s[30:31]4667;4668; VI-LABEL: global_atomic_or_i64_ret_offset_scalar:4669; VI: ; %bb.0:4670; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4671; VI-NEXT: s_add_u32 s34, s4, 324672; VI-NEXT: s_addc_u32 s35, s5, 04673; VI-NEXT: v_mov_b32_e32 v2, s344674; VI-NEXT: v_mov_b32_e32 v3, s354675; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]4676; VI-NEXT: s_mov_b64 s[34:35], 04677; VI-NEXT: .LBB67_1: ; %atomicrmw.start4678; VI-NEXT: ; =>This Inner Loop Header: Depth=14679; VI-NEXT: s_waitcnt vmcnt(0)4680; VI-NEXT: v_mov_b32_e32 v7, v14681; VI-NEXT: v_mov_b32_e32 v6, v04682; VI-NEXT: v_or_b32_e32 v5, s7, v74683; VI-NEXT: v_or_b32_e32 v4, s6, v64684; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc4685; VI-NEXT: s_waitcnt vmcnt(0)4686; VI-NEXT: buffer_wbinvl1_vol4687; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]4688; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4689; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]4690; VI-NEXT: s_cbranch_execnz .LBB67_14691; VI-NEXT: ; %bb.2: ; %atomicrmw.end4692; VI-NEXT: s_or_b64 exec, exec, s[34:35]4693; VI-NEXT: s_setpc_b64 s[30:31]4694;4695; GFX9-LABEL: global_atomic_or_i64_ret_offset_scalar:4696; GFX9: ; %bb.0:4697; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4698; GFX9-NEXT: v_mov_b32_e32 v2, 04699; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5] offset:324700; GFX9-NEXT: s_mov_b64 s[34:35], 04701; GFX9-NEXT: .LBB67_1: ; %atomicrmw.start4702; GFX9-NEXT: ; =>This Inner Loop Header: Depth=14703; GFX9-NEXT: s_waitcnt vmcnt(0)4704; GFX9-NEXT: v_mov_b32_e32 v6, v14705; GFX9-NEXT: v_mov_b32_e32 v5, v04706; GFX9-NEXT: v_or_b32_e32 v4, s7, v64707; GFX9-NEXT: v_or_b32_e32 v3, s6, v54708; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] offset:32 glc4709; GFX9-NEXT: s_waitcnt vmcnt(0)4710; GFX9-NEXT: buffer_wbinvl1_vol4711; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]4712; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4713; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]4714; GFX9-NEXT: s_cbranch_execnz .LBB67_14715; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end4716; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]4717; GFX9-NEXT: s_setpc_b64 s[30:31]4718 %gep = getelementptr i64, ptr addrspace(1) %out, i64 44719 %result = atomicrmw or ptr addrspace(1) %gep, i64 %in seq_cst4720 ret i64 %result4721}4722 4723define void @global_atomic_or_i64_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {4724; SI-LABEL: global_atomic_or_i64_noret_offset__amdgpu_no_remote_memory:4725; SI: ; %bb.0:4726; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4727; SI-NEXT: s_mov_b32 s6, 04728; SI-NEXT: s_mov_b32 s7, 0xf0004729; SI-NEXT: s_mov_b32 s4, s64730; SI-NEXT: s_mov_b32 s5, s64731; SI-NEXT: buffer_atomic_or_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:324732; SI-NEXT: s_waitcnt vmcnt(0)4733; SI-NEXT: buffer_wbinvl14734; SI-NEXT: s_waitcnt expcnt(0)4735; SI-NEXT: s_setpc_b64 s[30:31]4736;4737; VI-LABEL: global_atomic_or_i64_noret_offset__amdgpu_no_remote_memory:4738; VI: ; %bb.0:4739; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4740; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v04741; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc4742; VI-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]4743; VI-NEXT: s_waitcnt vmcnt(0)4744; VI-NEXT: buffer_wbinvl1_vol4745; VI-NEXT: s_setpc_b64 s[30:31]4746;4747; GFX9-LABEL: global_atomic_or_i64_noret_offset__amdgpu_no_remote_memory:4748; GFX9: ; %bb.0:4749; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4750; GFX9-NEXT: global_atomic_or_x2 v[0:1], v[2:3], off offset:324751; GFX9-NEXT: s_waitcnt vmcnt(0)4752; GFX9-NEXT: buffer_wbinvl1_vol4753; GFX9-NEXT: s_setpc_b64 s[30:31]4754 %gep = getelementptr i64, ptr addrspace(1) %out, i64 44755 %tmp0 = atomicrmw or ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !04756 ret void4757}4758 4759define i64 @global_atomic_or_i64_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {4760; SI-LABEL: global_atomic_or_i64_ret_offset__amdgpu_no_remote_memory:4761; SI: ; %bb.0:4762; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4763; SI-NEXT: s_mov_b32 s6, 04764; SI-NEXT: s_mov_b32 s7, 0xf0004765; SI-NEXT: s_mov_b32 s4, s64766; SI-NEXT: s_mov_b32 s5, s64767; SI-NEXT: buffer_atomic_or_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc4768; SI-NEXT: s_waitcnt vmcnt(0)4769; SI-NEXT: buffer_wbinvl14770; SI-NEXT: v_mov_b32_e32 v0, v24771; SI-NEXT: v_mov_b32_e32 v1, v34772; SI-NEXT: s_waitcnt expcnt(0)4773; SI-NEXT: s_setpc_b64 s[30:31]4774;4775; VI-LABEL: global_atomic_or_i64_ret_offset__amdgpu_no_remote_memory:4776; VI: ; %bb.0:4777; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4778; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v04779; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc4780; VI-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc4781; VI-NEXT: s_waitcnt vmcnt(0)4782; VI-NEXT: buffer_wbinvl1_vol4783; VI-NEXT: s_setpc_b64 s[30:31]4784;4785; GFX9-LABEL: global_atomic_or_i64_ret_offset__amdgpu_no_remote_memory:4786; GFX9: ; %bb.0:4787; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4788; GFX9-NEXT: global_atomic_or_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc4789; GFX9-NEXT: s_waitcnt vmcnt(0)4790; GFX9-NEXT: buffer_wbinvl1_vol4791; GFX9-NEXT: s_setpc_b64 s[30:31]4792 %gep = getelementptr i64, ptr addrspace(1) %out, i64 44793 %result = atomicrmw or ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !04794 ret i64 %result4795}4796 4797; ---------------------------------------------------------------------4798; atomicrmw xor4799; ---------------------------------------------------------------------4800 4801define void @global_atomic_xor_i64_noret(ptr addrspace(1) %ptr, i64 %in) {4802; SI-LABEL: global_atomic_xor_i64_noret:4803; SI: ; %bb.0:4804; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4805; SI-NEXT: s_mov_b32 s6, 04806; SI-NEXT: s_mov_b32 s7, 0xf0004807; SI-NEXT: s_mov_b32 s4, s64808; SI-NEXT: s_mov_b32 s5, s64809; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr644810; SI-NEXT: s_mov_b64 s[8:9], 04811; SI-NEXT: .LBB70_1: ; %atomicrmw.start4812; SI-NEXT: ; =>This Inner Loop Header: Depth=14813; SI-NEXT: s_waitcnt vmcnt(0)4814; SI-NEXT: v_xor_b32_e32 v5, v7, v34815; SI-NEXT: v_xor_b32_e32 v4, v6, v24816; SI-NEXT: s_waitcnt expcnt(0)4817; SI-NEXT: v_mov_b32_e32 v11, v74818; SI-NEXT: v_mov_b32_e32 v10, v64819; SI-NEXT: v_mov_b32_e32 v9, v54820; SI-NEXT: v_mov_b32_e32 v8, v44821; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc4822; SI-NEXT: s_waitcnt vmcnt(0)4823; SI-NEXT: buffer_wbinvl14824; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]4825; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]4826; SI-NEXT: v_mov_b32_e32 v6, v84827; SI-NEXT: v_mov_b32_e32 v7, v94828; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]4829; SI-NEXT: s_cbranch_execnz .LBB70_14830; SI-NEXT: ; %bb.2: ; %atomicrmw.end4831; SI-NEXT: s_or_b64 exec, exec, s[8:9]4832; SI-NEXT: s_waitcnt expcnt(0)4833; SI-NEXT: s_setpc_b64 s[30:31]4834;4835; VI-LABEL: global_atomic_xor_i64_noret:4836; VI: ; %bb.0:4837; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4838; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]4839; VI-NEXT: s_mov_b64 s[4:5], 04840; VI-NEXT: .LBB70_1: ; %atomicrmw.start4841; VI-NEXT: ; =>This Inner Loop Header: Depth=14842; VI-NEXT: s_waitcnt vmcnt(0)4843; VI-NEXT: v_xor_b32_e32 v5, v7, v34844; VI-NEXT: v_xor_b32_e32 v4, v6, v24845; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc4846; VI-NEXT: s_waitcnt vmcnt(0)4847; VI-NEXT: buffer_wbinvl1_vol4848; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]4849; VI-NEXT: v_mov_b32_e32 v7, v54850; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4851; VI-NEXT: v_mov_b32_e32 v6, v44852; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]4853; VI-NEXT: s_cbranch_execnz .LBB70_14854; VI-NEXT: ; %bb.2: ; %atomicrmw.end4855; VI-NEXT: s_or_b64 exec, exec, s[4:5]4856; VI-NEXT: s_setpc_b64 s[30:31]4857;4858; GFX9-LABEL: global_atomic_xor_i64_noret:4859; GFX9: ; %bb.0:4860; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4861; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off4862; GFX9-NEXT: s_mov_b64 s[4:5], 04863; GFX9-NEXT: .LBB70_1: ; %atomicrmw.start4864; GFX9-NEXT: ; =>This Inner Loop Header: Depth=14865; GFX9-NEXT: s_waitcnt vmcnt(0)4866; GFX9-NEXT: v_xor_b32_e32 v5, v7, v34867; GFX9-NEXT: v_xor_b32_e32 v4, v6, v24868; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc4869; GFX9-NEXT: s_waitcnt vmcnt(0)4870; GFX9-NEXT: buffer_wbinvl1_vol4871; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]4872; GFX9-NEXT: v_mov_b32_e32 v7, v54873; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4874; GFX9-NEXT: v_mov_b32_e32 v6, v44875; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]4876; GFX9-NEXT: s_cbranch_execnz .LBB70_14877; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end4878; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]4879; GFX9-NEXT: s_setpc_b64 s[30:31]4880 %tmp0 = atomicrmw xor ptr addrspace(1) %ptr, i64 %in seq_cst4881 ret void4882}4883 4884define void @global_atomic_xor_i64_noret_offset(ptr addrspace(1) %out, i64 %in) {4885; SI-LABEL: global_atomic_xor_i64_noret_offset:4886; SI: ; %bb.0:4887; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4888; SI-NEXT: s_mov_b32 s6, 04889; SI-NEXT: s_mov_b32 s7, 0xf0004890; SI-NEXT: s_mov_b32 s4, s64891; SI-NEXT: s_mov_b32 s5, s64892; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:324893; SI-NEXT: s_mov_b64 s[8:9], 04894; SI-NEXT: .LBB71_1: ; %atomicrmw.start4895; SI-NEXT: ; =>This Inner Loop Header: Depth=14896; SI-NEXT: s_waitcnt vmcnt(0)4897; SI-NEXT: v_xor_b32_e32 v5, v7, v34898; SI-NEXT: v_xor_b32_e32 v4, v6, v24899; SI-NEXT: s_waitcnt expcnt(0)4900; SI-NEXT: v_mov_b32_e32 v11, v74901; SI-NEXT: v_mov_b32_e32 v10, v64902; SI-NEXT: v_mov_b32_e32 v9, v54903; SI-NEXT: v_mov_b32_e32 v8, v44904; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32 glc4905; SI-NEXT: s_waitcnt vmcnt(0)4906; SI-NEXT: buffer_wbinvl14907; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]4908; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]4909; SI-NEXT: v_mov_b32_e32 v6, v84910; SI-NEXT: v_mov_b32_e32 v7, v94911; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]4912; SI-NEXT: s_cbranch_execnz .LBB71_14913; SI-NEXT: ; %bb.2: ; %atomicrmw.end4914; SI-NEXT: s_or_b64 exec, exec, s[8:9]4915; SI-NEXT: s_waitcnt expcnt(0)4916; SI-NEXT: s_setpc_b64 s[30:31]4917;4918; VI-LABEL: global_atomic_xor_i64_noret_offset:4919; VI: ; %bb.0:4920; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4921; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v04922; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc4923; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]4924; VI-NEXT: s_mov_b64 s[4:5], 04925; VI-NEXT: .LBB71_1: ; %atomicrmw.start4926; VI-NEXT: ; =>This Inner Loop Header: Depth=14927; VI-NEXT: s_waitcnt vmcnt(0)4928; VI-NEXT: v_xor_b32_e32 v5, v7, v34929; VI-NEXT: v_xor_b32_e32 v4, v6, v24930; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc4931; VI-NEXT: s_waitcnt vmcnt(0)4932; VI-NEXT: buffer_wbinvl1_vol4933; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]4934; VI-NEXT: v_mov_b32_e32 v7, v54935; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4936; VI-NEXT: v_mov_b32_e32 v6, v44937; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]4938; VI-NEXT: s_cbranch_execnz .LBB71_14939; VI-NEXT: ; %bb.2: ; %atomicrmw.end4940; VI-NEXT: s_or_b64 exec, exec, s[4:5]4941; VI-NEXT: s_setpc_b64 s[30:31]4942;4943; GFX9-LABEL: global_atomic_xor_i64_noret_offset:4944; GFX9: ; %bb.0:4945; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4946; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:324947; GFX9-NEXT: s_mov_b64 s[4:5], 04948; GFX9-NEXT: .LBB71_1: ; %atomicrmw.start4949; GFX9-NEXT: ; =>This Inner Loop Header: Depth=14950; GFX9-NEXT: s_waitcnt vmcnt(0)4951; GFX9-NEXT: v_xor_b32_e32 v5, v7, v34952; GFX9-NEXT: v_xor_b32_e32 v4, v6, v24953; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc4954; GFX9-NEXT: s_waitcnt vmcnt(0)4955; GFX9-NEXT: buffer_wbinvl1_vol4956; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]4957; GFX9-NEXT: v_mov_b32_e32 v7, v54958; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4959; GFX9-NEXT: v_mov_b32_e32 v6, v44960; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]4961; GFX9-NEXT: s_cbranch_execnz .LBB71_14962; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end4963; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]4964; GFX9-NEXT: s_setpc_b64 s[30:31]4965 %gep = getelementptr i64, ptr addrspace(1) %out, i64 44966 %tmp0 = atomicrmw xor ptr addrspace(1) %gep, i64 %in seq_cst4967 ret void4968}4969 4970define i64 @global_atomic_xor_i64_ret(ptr addrspace(1) %ptr, i64 %in) {4971; SI-LABEL: global_atomic_xor_i64_ret:4972; SI: ; %bb.0:4973; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4974; SI-NEXT: v_mov_b32_e32 v6, v34975; SI-NEXT: v_mov_b32_e32 v7, v24976; SI-NEXT: v_mov_b32_e32 v5, v14977; SI-NEXT: v_mov_b32_e32 v4, v04978; SI-NEXT: s_mov_b32 s6, 04979; SI-NEXT: s_mov_b32 s7, 0xf0004980; SI-NEXT: s_mov_b32 s4, s64981; SI-NEXT: s_mov_b32 s5, s64982; SI-NEXT: buffer_load_dwordx2 v[10:11], v[4:5], s[4:7], 0 addr644983; SI-NEXT: s_mov_b64 s[8:9], 04984; SI-NEXT: .LBB72_1: ; %atomicrmw.start4985; SI-NEXT: ; =>This Inner Loop Header: Depth=14986; SI-NEXT: s_waitcnt vmcnt(0)4987; SI-NEXT: v_xor_b32_e32 v9, v11, v64988; SI-NEXT: v_xor_b32_e32 v8, v10, v74989; SI-NEXT: s_waitcnt expcnt(0)4990; SI-NEXT: v_mov_b32_e32 v0, v84991; SI-NEXT: v_mov_b32_e32 v1, v94992; SI-NEXT: v_mov_b32_e32 v2, v104993; SI-NEXT: v_mov_b32_e32 v3, v114994; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 glc4995; SI-NEXT: s_waitcnt vmcnt(0)4996; SI-NEXT: buffer_wbinvl14997; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]4998; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]4999; SI-NEXT: v_mov_b32_e32 v11, v15000; SI-NEXT: v_mov_b32_e32 v10, v05001; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]5002; SI-NEXT: s_cbranch_execnz .LBB72_15003; SI-NEXT: ; %bb.2: ; %atomicrmw.end5004; SI-NEXT: s_or_b64 exec, exec, s[8:9]5005; SI-NEXT: s_waitcnt expcnt(0)5006; SI-NEXT: s_setpc_b64 s[30:31]5007;5008; VI-LABEL: global_atomic_xor_i64_ret:5009; VI: ; %bb.0:5010; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5011; VI-NEXT: flat_load_dwordx2 v[4:5], v[0:1]5012; VI-NEXT: s_mov_b64 s[4:5], 05013; VI-NEXT: .LBB72_1: ; %atomicrmw.start5014; VI-NEXT: ; =>This Inner Loop Header: Depth=15015; VI-NEXT: s_waitcnt vmcnt(0)5016; VI-NEXT: v_mov_b32_e32 v7, v55017; VI-NEXT: v_mov_b32_e32 v6, v45018; VI-NEXT: v_xor_b32_e32 v5, v7, v35019; VI-NEXT: v_xor_b32_e32 v4, v6, v25020; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc5021; VI-NEXT: s_waitcnt vmcnt(0)5022; VI-NEXT: buffer_wbinvl1_vol5023; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]5024; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5025; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]5026; VI-NEXT: s_cbranch_execnz .LBB72_15027; VI-NEXT: ; %bb.2: ; %atomicrmw.end5028; VI-NEXT: s_or_b64 exec, exec, s[4:5]5029; VI-NEXT: v_mov_b32_e32 v0, v45030; VI-NEXT: v_mov_b32_e32 v1, v55031; VI-NEXT: s_setpc_b64 s[30:31]5032;5033; GFX9-LABEL: global_atomic_xor_i64_ret:5034; GFX9: ; %bb.0:5035; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5036; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off5037; GFX9-NEXT: s_mov_b64 s[4:5], 05038; GFX9-NEXT: .LBB72_1: ; %atomicrmw.start5039; GFX9-NEXT: ; =>This Inner Loop Header: Depth=15040; GFX9-NEXT: s_waitcnt vmcnt(0)5041; GFX9-NEXT: v_mov_b32_e32 v7, v55042; GFX9-NEXT: v_mov_b32_e32 v6, v45043; GFX9-NEXT: v_xor_b32_e32 v5, v7, v35044; GFX9-NEXT: v_xor_b32_e32 v4, v6, v25045; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc5046; GFX9-NEXT: s_waitcnt vmcnt(0)5047; GFX9-NEXT: buffer_wbinvl1_vol5048; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]5049; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5050; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]5051; GFX9-NEXT: s_cbranch_execnz .LBB72_15052; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end5053; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]5054; GFX9-NEXT: v_mov_b32_e32 v0, v45055; GFX9-NEXT: v_mov_b32_e32 v1, v55056; GFX9-NEXT: s_setpc_b64 s[30:31]5057 %result = atomicrmw xor ptr addrspace(1) %ptr, i64 %in seq_cst5058 ret i64 %result5059}5060 5061define i64 @global_atomic_xor_i64_ret_offset(ptr addrspace(1) %out, i64 %in) {5062; SI-LABEL: global_atomic_xor_i64_ret_offset:5063; SI: ; %bb.0:5064; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5065; SI-NEXT: v_mov_b32_e32 v6, v35066; SI-NEXT: v_mov_b32_e32 v7, v25067; SI-NEXT: v_mov_b32_e32 v5, v15068; SI-NEXT: v_mov_b32_e32 v4, v05069; SI-NEXT: s_mov_b32 s6, 05070; SI-NEXT: s_mov_b32 s7, 0xf0005071; SI-NEXT: s_mov_b32 s4, s65072; SI-NEXT: s_mov_b32 s5, s65073; SI-NEXT: buffer_load_dwordx2 v[10:11], v[4:5], s[4:7], 0 addr64 offset:325074; SI-NEXT: s_mov_b64 s[8:9], 05075; SI-NEXT: .LBB73_1: ; %atomicrmw.start5076; SI-NEXT: ; =>This Inner Loop Header: Depth=15077; SI-NEXT: s_waitcnt vmcnt(0)5078; SI-NEXT: v_xor_b32_e32 v9, v11, v65079; SI-NEXT: v_xor_b32_e32 v8, v10, v75080; SI-NEXT: s_waitcnt expcnt(0)5081; SI-NEXT: v_mov_b32_e32 v0, v85082; SI-NEXT: v_mov_b32_e32 v1, v95083; SI-NEXT: v_mov_b32_e32 v2, v105084; SI-NEXT: v_mov_b32_e32 v3, v115085; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 offset:32 glc5086; SI-NEXT: s_waitcnt vmcnt(0)5087; SI-NEXT: buffer_wbinvl15088; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]5089; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]5090; SI-NEXT: v_mov_b32_e32 v11, v15091; SI-NEXT: v_mov_b32_e32 v10, v05092; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]5093; SI-NEXT: s_cbranch_execnz .LBB73_15094; SI-NEXT: ; %bb.2: ; %atomicrmw.end5095; SI-NEXT: s_or_b64 exec, exec, s[8:9]5096; SI-NEXT: s_waitcnt expcnt(0)5097; SI-NEXT: s_setpc_b64 s[30:31]5098;5099; VI-LABEL: global_atomic_xor_i64_ret_offset:5100; VI: ; %bb.0:5101; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5102; VI-NEXT: v_add_u32_e32 v4, vcc, 32, v05103; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc5104; VI-NEXT: flat_load_dwordx2 v[0:1], v[4:5]5105; VI-NEXT: s_mov_b64 s[4:5], 05106; VI-NEXT: .LBB73_1: ; %atomicrmw.start5107; VI-NEXT: ; =>This Inner Loop Header: Depth=15108; VI-NEXT: s_waitcnt vmcnt(0)5109; VI-NEXT: v_mov_b32_e32 v9, v15110; VI-NEXT: v_mov_b32_e32 v8, v05111; VI-NEXT: v_xor_b32_e32 v7, v9, v35112; VI-NEXT: v_xor_b32_e32 v6, v8, v25113; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc5114; VI-NEXT: s_waitcnt vmcnt(0)5115; VI-NEXT: buffer_wbinvl1_vol5116; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]5117; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5118; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]5119; VI-NEXT: s_cbranch_execnz .LBB73_15120; VI-NEXT: ; %bb.2: ; %atomicrmw.end5121; VI-NEXT: s_or_b64 exec, exec, s[4:5]5122; VI-NEXT: s_setpc_b64 s[30:31]5123;5124; GFX9-LABEL: global_atomic_xor_i64_ret_offset:5125; GFX9: ; %bb.0:5126; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5127; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:325128; GFX9-NEXT: s_mov_b64 s[4:5], 05129; GFX9-NEXT: .LBB73_1: ; %atomicrmw.start5130; GFX9-NEXT: ; =>This Inner Loop Header: Depth=15131; GFX9-NEXT: s_waitcnt vmcnt(0)5132; GFX9-NEXT: v_mov_b32_e32 v7, v55133; GFX9-NEXT: v_mov_b32_e32 v6, v45134; GFX9-NEXT: v_xor_b32_e32 v5, v7, v35135; GFX9-NEXT: v_xor_b32_e32 v4, v6, v25136; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc5137; GFX9-NEXT: s_waitcnt vmcnt(0)5138; GFX9-NEXT: buffer_wbinvl1_vol5139; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]5140; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5141; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]5142; GFX9-NEXT: s_cbranch_execnz .LBB73_15143; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end5144; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]5145; GFX9-NEXT: v_mov_b32_e32 v0, v45146; GFX9-NEXT: v_mov_b32_e32 v1, v55147; GFX9-NEXT: s_setpc_b64 s[30:31]5148 %gep = getelementptr i64, ptr addrspace(1) %out, i64 45149 %result = atomicrmw xor ptr addrspace(1) %gep, i64 %in seq_cst5150 ret i64 %result5151}5152 5153define amdgpu_gfx void @global_atomic_xor_i64_noret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {5154; SI-LABEL: global_atomic_xor_i64_noret_scalar:5155; SI: ; %bb.0:5156; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5157; SI-NEXT: s_xor_saveexec_b64 s[34:35], -15158; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill5159; SI-NEXT: s_mov_b64 exec, s[34:35]5160; SI-NEXT: s_waitcnt expcnt(0)5161; SI-NEXT: v_writelane_b32 v8, s6, 05162; SI-NEXT: v_writelane_b32 v8, s7, 15163; SI-NEXT: s_mov_b32 s34, s75164; SI-NEXT: s_mov_b32 s35, s65165; SI-NEXT: s_mov_b32 s7, 0xf0005166; SI-NEXT: s_mov_b32 s6, -15167; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 05168; SI-NEXT: s_mov_b64 s[36:37], 05169; SI-NEXT: .LBB74_1: ; %atomicrmw.start5170; SI-NEXT: ; =>This Inner Loop Header: Depth=15171; SI-NEXT: s_waitcnt vmcnt(0)5172; SI-NEXT: v_xor_b32_e32 v1, s34, v35173; SI-NEXT: v_xor_b32_e32 v0, s35, v25174; SI-NEXT: s_waitcnt expcnt(0)5175; SI-NEXT: v_mov_b32_e32 v7, v35176; SI-NEXT: v_mov_b32_e32 v6, v25177; SI-NEXT: v_mov_b32_e32 v5, v15178; SI-NEXT: v_mov_b32_e32 v4, v05179; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 glc5180; SI-NEXT: s_waitcnt vmcnt(0)5181; SI-NEXT: buffer_wbinvl15182; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]5183; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]5184; SI-NEXT: v_mov_b32_e32 v2, v45185; SI-NEXT: v_mov_b32_e32 v3, v55186; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]5187; SI-NEXT: s_cbranch_execnz .LBB74_15188; SI-NEXT: ; %bb.2: ; %atomicrmw.end5189; SI-NEXT: s_or_b64 exec, exec, s[36:37]5190; SI-NEXT: v_readlane_b32 s7, v8, 15191; SI-NEXT: v_readlane_b32 s6, v8, 05192; SI-NEXT: s_xor_saveexec_b64 s[34:35], -15193; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload5194; SI-NEXT: s_mov_b64 exec, s[34:35]5195; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)5196; SI-NEXT: s_setpc_b64 s[30:31]5197;5198; VI-LABEL: global_atomic_xor_i64_noret_scalar:5199; VI: ; %bb.0:5200; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5201; VI-NEXT: v_mov_b32_e32 v0, s45202; VI-NEXT: v_mov_b32_e32 v1, s55203; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]5204; VI-NEXT: v_mov_b32_e32 v4, s45205; VI-NEXT: s_mov_b64 s[34:35], 05206; VI-NEXT: v_mov_b32_e32 v5, s55207; VI-NEXT: .LBB74_1: ; %atomicrmw.start5208; VI-NEXT: ; =>This Inner Loop Header: Depth=15209; VI-NEXT: s_waitcnt vmcnt(0)5210; VI-NEXT: v_xor_b32_e32 v1, s7, v35211; VI-NEXT: v_xor_b32_e32 v0, s6, v25212; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc5213; VI-NEXT: s_waitcnt vmcnt(0)5214; VI-NEXT: buffer_wbinvl1_vol5215; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]5216; VI-NEXT: v_mov_b32_e32 v3, v15217; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5218; VI-NEXT: v_mov_b32_e32 v2, v05219; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]5220; VI-NEXT: s_cbranch_execnz .LBB74_15221; VI-NEXT: ; %bb.2: ; %atomicrmw.end5222; VI-NEXT: s_or_b64 exec, exec, s[34:35]5223; VI-NEXT: s_setpc_b64 s[30:31]5224;5225; GFX9-LABEL: global_atomic_xor_i64_noret_scalar:5226; GFX9: ; %bb.0:5227; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5228; GFX9-NEXT: v_mov_b32_e32 v4, 05229; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5]5230; GFX9-NEXT: s_mov_b64 s[34:35], 05231; GFX9-NEXT: .LBB74_1: ; %atomicrmw.start5232; GFX9-NEXT: ; =>This Inner Loop Header: Depth=15233; GFX9-NEXT: s_waitcnt vmcnt(0)5234; GFX9-NEXT: v_xor_b32_e32 v1, s7, v35235; GFX9-NEXT: v_xor_b32_e32 v0, s6, v25236; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc5237; GFX9-NEXT: s_waitcnt vmcnt(0)5238; GFX9-NEXT: buffer_wbinvl1_vol5239; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]5240; GFX9-NEXT: v_mov_b32_e32 v3, v15241; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5242; GFX9-NEXT: v_mov_b32_e32 v2, v05243; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]5244; GFX9-NEXT: s_cbranch_execnz .LBB74_15245; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end5246; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]5247; GFX9-NEXT: s_setpc_b64 s[30:31]5248 %tmp0 = atomicrmw xor ptr addrspace(1) %ptr, i64 %in seq_cst5249 ret void5250}5251 5252define amdgpu_gfx void @global_atomic_xor_i64_noret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {5253; SI-LABEL: global_atomic_xor_i64_noret_offset_scalar:5254; SI: ; %bb.0:5255; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5256; SI-NEXT: s_xor_saveexec_b64 s[34:35], -15257; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill5258; SI-NEXT: s_mov_b64 exec, s[34:35]5259; SI-NEXT: s_waitcnt expcnt(0)5260; SI-NEXT: v_writelane_b32 v8, s6, 05261; SI-NEXT: v_writelane_b32 v8, s7, 15262; SI-NEXT: s_mov_b32 s34, s75263; SI-NEXT: s_mov_b32 s35, s65264; SI-NEXT: s_mov_b32 s7, 0xf0005265; SI-NEXT: s_mov_b32 s6, -15266; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:325267; SI-NEXT: s_mov_b64 s[36:37], 05268; SI-NEXT: .LBB75_1: ; %atomicrmw.start5269; SI-NEXT: ; =>This Inner Loop Header: Depth=15270; SI-NEXT: s_waitcnt vmcnt(0)5271; SI-NEXT: v_xor_b32_e32 v1, s34, v35272; SI-NEXT: v_xor_b32_e32 v0, s35, v25273; SI-NEXT: s_waitcnt expcnt(0)5274; SI-NEXT: v_mov_b32_e32 v7, v35275; SI-NEXT: v_mov_b32_e32 v6, v25276; SI-NEXT: v_mov_b32_e32 v5, v15277; SI-NEXT: v_mov_b32_e32 v4, v05278; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 offset:32 glc5279; SI-NEXT: s_waitcnt vmcnt(0)5280; SI-NEXT: buffer_wbinvl15281; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]5282; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]5283; SI-NEXT: v_mov_b32_e32 v2, v45284; SI-NEXT: v_mov_b32_e32 v3, v55285; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]5286; SI-NEXT: s_cbranch_execnz .LBB75_15287; SI-NEXT: ; %bb.2: ; %atomicrmw.end5288; SI-NEXT: s_or_b64 exec, exec, s[36:37]5289; SI-NEXT: v_readlane_b32 s7, v8, 15290; SI-NEXT: v_readlane_b32 s6, v8, 05291; SI-NEXT: s_xor_saveexec_b64 s[34:35], -15292; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload5293; SI-NEXT: s_mov_b64 exec, s[34:35]5294; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)5295; SI-NEXT: s_setpc_b64 s[30:31]5296;5297; VI-LABEL: global_atomic_xor_i64_noret_offset_scalar:5298; VI: ; %bb.0:5299; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5300; VI-NEXT: s_add_u32 s34, s4, 325301; VI-NEXT: s_addc_u32 s35, s5, 05302; VI-NEXT: v_mov_b32_e32 v4, s345303; VI-NEXT: v_mov_b32_e32 v5, s355304; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]5305; VI-NEXT: s_mov_b64 s[34:35], 05306; VI-NEXT: .LBB75_1: ; %atomicrmw.start5307; VI-NEXT: ; =>This Inner Loop Header: Depth=15308; VI-NEXT: s_waitcnt vmcnt(0)5309; VI-NEXT: v_xor_b32_e32 v1, s7, v35310; VI-NEXT: v_xor_b32_e32 v0, s6, v25311; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc5312; VI-NEXT: s_waitcnt vmcnt(0)5313; VI-NEXT: buffer_wbinvl1_vol5314; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]5315; VI-NEXT: v_mov_b32_e32 v3, v15316; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5317; VI-NEXT: v_mov_b32_e32 v2, v05318; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]5319; VI-NEXT: s_cbranch_execnz .LBB75_15320; VI-NEXT: ; %bb.2: ; %atomicrmw.end5321; VI-NEXT: s_or_b64 exec, exec, s[34:35]5322; VI-NEXT: s_setpc_b64 s[30:31]5323;5324; GFX9-LABEL: global_atomic_xor_i64_noret_offset_scalar:5325; GFX9: ; %bb.0:5326; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5327; GFX9-NEXT: v_mov_b32_e32 v4, 05328; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5] offset:325329; GFX9-NEXT: s_mov_b64 s[34:35], 05330; GFX9-NEXT: .LBB75_1: ; %atomicrmw.start5331; GFX9-NEXT: ; =>This Inner Loop Header: Depth=15332; GFX9-NEXT: s_waitcnt vmcnt(0)5333; GFX9-NEXT: v_xor_b32_e32 v1, s7, v35334; GFX9-NEXT: v_xor_b32_e32 v0, s6, v25335; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc5336; GFX9-NEXT: s_waitcnt vmcnt(0)5337; GFX9-NEXT: buffer_wbinvl1_vol5338; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]5339; GFX9-NEXT: v_mov_b32_e32 v3, v15340; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5341; GFX9-NEXT: v_mov_b32_e32 v2, v05342; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]5343; GFX9-NEXT: s_cbranch_execnz .LBB75_15344; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end5345; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]5346; GFX9-NEXT: s_setpc_b64 s[30:31]5347 %gep = getelementptr i64, ptr addrspace(1) %out, i64 45348 %tmp0 = atomicrmw xor ptr addrspace(1) %gep, i64 %in seq_cst5349 ret void5350}5351 5352define amdgpu_gfx i64 @global_atomic_xor_i64_ret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {5353; SI-LABEL: global_atomic_xor_i64_ret_scalar:5354; SI: ; %bb.0:5355; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5356; SI-NEXT: s_xor_saveexec_b64 s[34:35], -15357; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill5358; SI-NEXT: s_mov_b64 exec, s[34:35]5359; SI-NEXT: s_waitcnt expcnt(0)5360; SI-NEXT: v_writelane_b32 v6, s6, 05361; SI-NEXT: v_writelane_b32 v6, s7, 15362; SI-NEXT: s_mov_b32 s34, s75363; SI-NEXT: s_mov_b32 s35, s65364; SI-NEXT: s_mov_b32 s7, 0xf0005365; SI-NEXT: s_mov_b32 s6, -15366; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 05367; SI-NEXT: s_mov_b64 s[36:37], 05368; SI-NEXT: .LBB76_1: ; %atomicrmw.start5369; SI-NEXT: ; =>This Inner Loop Header: Depth=15370; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)5371; SI-NEXT: v_xor_b32_e32 v3, s34, v55372; SI-NEXT: v_xor_b32_e32 v2, s35, v45373; SI-NEXT: v_mov_b32_e32 v0, v25374; SI-NEXT: v_mov_b32_e32 v1, v35375; SI-NEXT: v_mov_b32_e32 v2, v45376; SI-NEXT: v_mov_b32_e32 v3, v55377; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc5378; SI-NEXT: s_waitcnt vmcnt(0)5379; SI-NEXT: buffer_wbinvl15380; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]5381; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]5382; SI-NEXT: v_mov_b32_e32 v5, v15383; SI-NEXT: v_mov_b32_e32 v4, v05384; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]5385; SI-NEXT: s_cbranch_execnz .LBB76_15386; SI-NEXT: ; %bb.2: ; %atomicrmw.end5387; SI-NEXT: s_or_b64 exec, exec, s[36:37]5388; SI-NEXT: v_readlane_b32 s7, v6, 15389; SI-NEXT: v_readlane_b32 s6, v6, 05390; SI-NEXT: s_xor_saveexec_b64 s[34:35], -15391; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload5392; SI-NEXT: s_mov_b64 exec, s[34:35]5393; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)5394; SI-NEXT: s_setpc_b64 s[30:31]5395;5396; VI-LABEL: global_atomic_xor_i64_ret_scalar:5397; VI: ; %bb.0:5398; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5399; VI-NEXT: v_mov_b32_e32 v0, s45400; VI-NEXT: v_mov_b32_e32 v1, s55401; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]5402; VI-NEXT: v_mov_b32_e32 v2, s45403; VI-NEXT: s_mov_b64 s[34:35], 05404; VI-NEXT: v_mov_b32_e32 v3, s55405; VI-NEXT: .LBB76_1: ; %atomicrmw.start5406; VI-NEXT: ; =>This Inner Loop Header: Depth=15407; VI-NEXT: s_waitcnt vmcnt(0)5408; VI-NEXT: v_mov_b32_e32 v7, v15409; VI-NEXT: v_mov_b32_e32 v6, v05410; VI-NEXT: v_xor_b32_e32 v5, s7, v75411; VI-NEXT: v_xor_b32_e32 v4, s6, v65412; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc5413; VI-NEXT: s_waitcnt vmcnt(0)5414; VI-NEXT: buffer_wbinvl1_vol5415; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]5416; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5417; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]5418; VI-NEXT: s_cbranch_execnz .LBB76_15419; VI-NEXT: ; %bb.2: ; %atomicrmw.end5420; VI-NEXT: s_or_b64 exec, exec, s[34:35]5421; VI-NEXT: s_setpc_b64 s[30:31]5422;5423; GFX9-LABEL: global_atomic_xor_i64_ret_scalar:5424; GFX9: ; %bb.0:5425; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5426; GFX9-NEXT: v_mov_b32_e32 v2, 05427; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]5428; GFX9-NEXT: s_mov_b64 s[34:35], 05429; GFX9-NEXT: .LBB76_1: ; %atomicrmw.start5430; GFX9-NEXT: ; =>This Inner Loop Header: Depth=15431; GFX9-NEXT: s_waitcnt vmcnt(0)5432; GFX9-NEXT: v_mov_b32_e32 v6, v15433; GFX9-NEXT: v_mov_b32_e32 v5, v05434; GFX9-NEXT: v_xor_b32_e32 v4, s7, v65435; GFX9-NEXT: v_xor_b32_e32 v3, s6, v55436; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] glc5437; GFX9-NEXT: s_waitcnt vmcnt(0)5438; GFX9-NEXT: buffer_wbinvl1_vol5439; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]5440; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5441; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]5442; GFX9-NEXT: s_cbranch_execnz .LBB76_15443; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end5444; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]5445; GFX9-NEXT: s_setpc_b64 s[30:31]5446 %result = atomicrmw xor ptr addrspace(1) %ptr, i64 %in seq_cst5447 ret i64 %result5448}5449 5450define amdgpu_gfx i64 @global_atomic_xor_i64_ret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {5451; SI-LABEL: global_atomic_xor_i64_ret_offset_scalar:5452; SI: ; %bb.0:5453; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5454; SI-NEXT: s_xor_saveexec_b64 s[34:35], -15455; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill5456; SI-NEXT: s_mov_b64 exec, s[34:35]5457; SI-NEXT: s_waitcnt expcnt(0)5458; SI-NEXT: v_writelane_b32 v6, s6, 05459; SI-NEXT: v_writelane_b32 v6, s7, 15460; SI-NEXT: s_mov_b32 s34, s75461; SI-NEXT: s_mov_b32 s35, s65462; SI-NEXT: s_mov_b32 s7, 0xf0005463; SI-NEXT: s_mov_b32 s6, -15464; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0 offset:325465; SI-NEXT: s_mov_b64 s[36:37], 05466; SI-NEXT: .LBB77_1: ; %atomicrmw.start5467; SI-NEXT: ; =>This Inner Loop Header: Depth=15468; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)5469; SI-NEXT: v_xor_b32_e32 v3, s34, v55470; SI-NEXT: v_xor_b32_e32 v2, s35, v45471; SI-NEXT: v_mov_b32_e32 v0, v25472; SI-NEXT: v_mov_b32_e32 v1, v35473; SI-NEXT: v_mov_b32_e32 v2, v45474; SI-NEXT: v_mov_b32_e32 v3, v55475; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 offset:32 glc5476; SI-NEXT: s_waitcnt vmcnt(0)5477; SI-NEXT: buffer_wbinvl15478; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]5479; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]5480; SI-NEXT: v_mov_b32_e32 v5, v15481; SI-NEXT: v_mov_b32_e32 v4, v05482; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]5483; SI-NEXT: s_cbranch_execnz .LBB77_15484; SI-NEXT: ; %bb.2: ; %atomicrmw.end5485; SI-NEXT: s_or_b64 exec, exec, s[36:37]5486; SI-NEXT: v_readlane_b32 s7, v6, 15487; SI-NEXT: v_readlane_b32 s6, v6, 05488; SI-NEXT: s_xor_saveexec_b64 s[34:35], -15489; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload5490; SI-NEXT: s_mov_b64 exec, s[34:35]5491; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)5492; SI-NEXT: s_setpc_b64 s[30:31]5493;5494; VI-LABEL: global_atomic_xor_i64_ret_offset_scalar:5495; VI: ; %bb.0:5496; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5497; VI-NEXT: s_add_u32 s34, s4, 325498; VI-NEXT: s_addc_u32 s35, s5, 05499; VI-NEXT: v_mov_b32_e32 v2, s345500; VI-NEXT: v_mov_b32_e32 v3, s355501; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]5502; VI-NEXT: s_mov_b64 s[34:35], 05503; VI-NEXT: .LBB77_1: ; %atomicrmw.start5504; VI-NEXT: ; =>This Inner Loop Header: Depth=15505; VI-NEXT: s_waitcnt vmcnt(0)5506; VI-NEXT: v_mov_b32_e32 v7, v15507; VI-NEXT: v_mov_b32_e32 v6, v05508; VI-NEXT: v_xor_b32_e32 v5, s7, v75509; VI-NEXT: v_xor_b32_e32 v4, s6, v65510; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc5511; VI-NEXT: s_waitcnt vmcnt(0)5512; VI-NEXT: buffer_wbinvl1_vol5513; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]5514; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5515; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]5516; VI-NEXT: s_cbranch_execnz .LBB77_15517; VI-NEXT: ; %bb.2: ; %atomicrmw.end5518; VI-NEXT: s_or_b64 exec, exec, s[34:35]5519; VI-NEXT: s_setpc_b64 s[30:31]5520;5521; GFX9-LABEL: global_atomic_xor_i64_ret_offset_scalar:5522; GFX9: ; %bb.0:5523; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5524; GFX9-NEXT: v_mov_b32_e32 v2, 05525; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5] offset:325526; GFX9-NEXT: s_mov_b64 s[34:35], 05527; GFX9-NEXT: .LBB77_1: ; %atomicrmw.start5528; GFX9-NEXT: ; =>This Inner Loop Header: Depth=15529; GFX9-NEXT: s_waitcnt vmcnt(0)5530; GFX9-NEXT: v_mov_b32_e32 v6, v15531; GFX9-NEXT: v_mov_b32_e32 v5, v05532; GFX9-NEXT: v_xor_b32_e32 v4, s7, v65533; GFX9-NEXT: v_xor_b32_e32 v3, s6, v55534; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] offset:32 glc5535; GFX9-NEXT: s_waitcnt vmcnt(0)5536; GFX9-NEXT: buffer_wbinvl1_vol5537; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]5538; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5539; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]5540; GFX9-NEXT: s_cbranch_execnz .LBB77_15541; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end5542; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]5543; GFX9-NEXT: s_setpc_b64 s[30:31]5544 %gep = getelementptr i64, ptr addrspace(1) %out, i64 45545 %result = atomicrmw xor ptr addrspace(1) %gep, i64 %in seq_cst5546 ret i64 %result5547}5548 5549define void @global_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {5550; SI-LABEL: global_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory:5551; SI: ; %bb.0:5552; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5553; SI-NEXT: s_mov_b32 s6, 05554; SI-NEXT: s_mov_b32 s7, 0xf0005555; SI-NEXT: s_mov_b32 s4, s65556; SI-NEXT: s_mov_b32 s5, s65557; SI-NEXT: buffer_atomic_xor_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:325558; SI-NEXT: s_waitcnt vmcnt(0)5559; SI-NEXT: buffer_wbinvl15560; SI-NEXT: s_waitcnt expcnt(0)5561; SI-NEXT: s_setpc_b64 s[30:31]5562;5563; VI-LABEL: global_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory:5564; VI: ; %bb.0:5565; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5566; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v05567; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc5568; VI-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]5569; VI-NEXT: s_waitcnt vmcnt(0)5570; VI-NEXT: buffer_wbinvl1_vol5571; VI-NEXT: s_setpc_b64 s[30:31]5572;5573; GFX9-LABEL: global_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory:5574; GFX9: ; %bb.0:5575; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5576; GFX9-NEXT: global_atomic_xor_x2 v[0:1], v[2:3], off offset:325577; GFX9-NEXT: s_waitcnt vmcnt(0)5578; GFX9-NEXT: buffer_wbinvl1_vol5579; GFX9-NEXT: s_setpc_b64 s[30:31]5580 %gep = getelementptr i64, ptr addrspace(1) %out, i64 45581 %tmp0 = atomicrmw xor ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !05582 ret void5583}5584 5585define i64 @global_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {5586; SI-LABEL: global_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory:5587; SI: ; %bb.0:5588; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5589; SI-NEXT: s_mov_b32 s6, 05590; SI-NEXT: s_mov_b32 s7, 0xf0005591; SI-NEXT: s_mov_b32 s4, s65592; SI-NEXT: s_mov_b32 s5, s65593; SI-NEXT: buffer_atomic_xor_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc5594; SI-NEXT: s_waitcnt vmcnt(0)5595; SI-NEXT: buffer_wbinvl15596; SI-NEXT: v_mov_b32_e32 v0, v25597; SI-NEXT: v_mov_b32_e32 v1, v35598; SI-NEXT: s_waitcnt expcnt(0)5599; SI-NEXT: s_setpc_b64 s[30:31]5600;5601; VI-LABEL: global_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory:5602; VI: ; %bb.0:5603; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5604; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v05605; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc5606; VI-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc5607; VI-NEXT: s_waitcnt vmcnt(0)5608; VI-NEXT: buffer_wbinvl1_vol5609; VI-NEXT: s_setpc_b64 s[30:31]5610;5611; GFX9-LABEL: global_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory:5612; GFX9: ; %bb.0:5613; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5614; GFX9-NEXT: global_atomic_xor_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc5615; GFX9-NEXT: s_waitcnt vmcnt(0)5616; GFX9-NEXT: buffer_wbinvl1_vol5617; GFX9-NEXT: s_setpc_b64 s[30:31]5618 %gep = getelementptr i64, ptr addrspace(1) %out, i64 45619 %result = atomicrmw xor ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !05620 ret i64 %result5621}5622 5623; ---------------------------------------------------------------------5624; atomicrmw max5625; ---------------------------------------------------------------------5626 5627define void @global_atomic_max_i64_noret(ptr addrspace(1) %ptr, i64 %in) {5628; SI-LABEL: global_atomic_max_i64_noret:5629; SI: ; %bb.0:5630; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5631; SI-NEXT: s_mov_b32 s6, 05632; SI-NEXT: s_mov_b32 s7, 0xf0005633; SI-NEXT: s_mov_b32 s4, s65634; SI-NEXT: s_mov_b32 s5, s65635; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr645636; SI-NEXT: s_mov_b64 s[8:9], 05637; SI-NEXT: .LBB80_1: ; %atomicrmw.start5638; SI-NEXT: ; =>This Inner Loop Header: Depth=15639; SI-NEXT: s_waitcnt vmcnt(0)5640; SI-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]5641; SI-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc5642; SI-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc5643; SI-NEXT: s_waitcnt expcnt(0)5644; SI-NEXT: v_mov_b32_e32 v11, v75645; SI-NEXT: v_mov_b32_e32 v10, v65646; SI-NEXT: v_mov_b32_e32 v9, v55647; SI-NEXT: v_mov_b32_e32 v8, v45648; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc5649; SI-NEXT: s_waitcnt vmcnt(0)5650; SI-NEXT: buffer_wbinvl15651; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]5652; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]5653; SI-NEXT: v_mov_b32_e32 v6, v85654; SI-NEXT: v_mov_b32_e32 v7, v95655; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]5656; SI-NEXT: s_cbranch_execnz .LBB80_15657; SI-NEXT: ; %bb.2: ; %atomicrmw.end5658; SI-NEXT: s_or_b64 exec, exec, s[8:9]5659; SI-NEXT: s_waitcnt expcnt(0)5660; SI-NEXT: s_setpc_b64 s[30:31]5661;5662; VI-LABEL: global_atomic_max_i64_noret:5663; VI: ; %bb.0:5664; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5665; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]5666; VI-NEXT: s_mov_b64 s[4:5], 05667; VI-NEXT: .LBB80_1: ; %atomicrmw.start5668; VI-NEXT: ; =>This Inner Loop Header: Depth=15669; VI-NEXT: s_waitcnt vmcnt(0)5670; VI-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]5671; VI-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc5672; VI-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc5673; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc5674; VI-NEXT: s_waitcnt vmcnt(0)5675; VI-NEXT: buffer_wbinvl1_vol5676; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]5677; VI-NEXT: v_mov_b32_e32 v7, v55678; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5679; VI-NEXT: v_mov_b32_e32 v6, v45680; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]5681; VI-NEXT: s_cbranch_execnz .LBB80_15682; VI-NEXT: ; %bb.2: ; %atomicrmw.end5683; VI-NEXT: s_or_b64 exec, exec, s[4:5]5684; VI-NEXT: s_setpc_b64 s[30:31]5685;5686; GFX9-LABEL: global_atomic_max_i64_noret:5687; GFX9: ; %bb.0:5688; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5689; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off5690; GFX9-NEXT: s_mov_b64 s[4:5], 05691; GFX9-NEXT: .LBB80_1: ; %atomicrmw.start5692; GFX9-NEXT: ; =>This Inner Loop Header: Depth=15693; GFX9-NEXT: s_waitcnt vmcnt(0)5694; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]5695; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc5696; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc5697; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc5698; GFX9-NEXT: s_waitcnt vmcnt(0)5699; GFX9-NEXT: buffer_wbinvl1_vol5700; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]5701; GFX9-NEXT: v_mov_b32_e32 v7, v55702; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5703; GFX9-NEXT: v_mov_b32_e32 v6, v45704; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]5705; GFX9-NEXT: s_cbranch_execnz .LBB80_15706; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end5707; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]5708; GFX9-NEXT: s_setpc_b64 s[30:31]5709 %tmp0 = atomicrmw max ptr addrspace(1) %ptr, i64 %in seq_cst5710 ret void5711}5712 5713define void @global_atomic_max_i64_noret_offset(ptr addrspace(1) %out, i64 %in) {5714; SI-LABEL: global_atomic_max_i64_noret_offset:5715; SI: ; %bb.0:5716; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5717; SI-NEXT: s_mov_b32 s6, 05718; SI-NEXT: s_mov_b32 s7, 0xf0005719; SI-NEXT: s_mov_b32 s4, s65720; SI-NEXT: s_mov_b32 s5, s65721; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:325722; SI-NEXT: s_mov_b64 s[8:9], 05723; SI-NEXT: .LBB81_1: ; %atomicrmw.start5724; SI-NEXT: ; =>This Inner Loop Header: Depth=15725; SI-NEXT: s_waitcnt vmcnt(0)5726; SI-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]5727; SI-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc5728; SI-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc5729; SI-NEXT: s_waitcnt expcnt(0)5730; SI-NEXT: v_mov_b32_e32 v11, v75731; SI-NEXT: v_mov_b32_e32 v10, v65732; SI-NEXT: v_mov_b32_e32 v9, v55733; SI-NEXT: v_mov_b32_e32 v8, v45734; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32 glc5735; SI-NEXT: s_waitcnt vmcnt(0)5736; SI-NEXT: buffer_wbinvl15737; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]5738; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]5739; SI-NEXT: v_mov_b32_e32 v6, v85740; SI-NEXT: v_mov_b32_e32 v7, v95741; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]5742; SI-NEXT: s_cbranch_execnz .LBB81_15743; SI-NEXT: ; %bb.2: ; %atomicrmw.end5744; SI-NEXT: s_or_b64 exec, exec, s[8:9]5745; SI-NEXT: s_waitcnt expcnt(0)5746; SI-NEXT: s_setpc_b64 s[30:31]5747;5748; VI-LABEL: global_atomic_max_i64_noret_offset:5749; VI: ; %bb.0:5750; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5751; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v05752; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc5753; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]5754; VI-NEXT: s_mov_b64 s[4:5], 05755; VI-NEXT: .LBB81_1: ; %atomicrmw.start5756; VI-NEXT: ; =>This Inner Loop Header: Depth=15757; VI-NEXT: s_waitcnt vmcnt(0)5758; VI-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]5759; VI-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc5760; VI-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc5761; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc5762; VI-NEXT: s_waitcnt vmcnt(0)5763; VI-NEXT: buffer_wbinvl1_vol5764; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]5765; VI-NEXT: v_mov_b32_e32 v7, v55766; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5767; VI-NEXT: v_mov_b32_e32 v6, v45768; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]5769; VI-NEXT: s_cbranch_execnz .LBB81_15770; VI-NEXT: ; %bb.2: ; %atomicrmw.end5771; VI-NEXT: s_or_b64 exec, exec, s[4:5]5772; VI-NEXT: s_setpc_b64 s[30:31]5773;5774; GFX9-LABEL: global_atomic_max_i64_noret_offset:5775; GFX9: ; %bb.0:5776; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5777; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:325778; GFX9-NEXT: s_mov_b64 s[4:5], 05779; GFX9-NEXT: .LBB81_1: ; %atomicrmw.start5780; GFX9-NEXT: ; =>This Inner Loop Header: Depth=15781; GFX9-NEXT: s_waitcnt vmcnt(0)5782; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]5783; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc5784; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc5785; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc5786; GFX9-NEXT: s_waitcnt vmcnt(0)5787; GFX9-NEXT: buffer_wbinvl1_vol5788; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]5789; GFX9-NEXT: v_mov_b32_e32 v7, v55790; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5791; GFX9-NEXT: v_mov_b32_e32 v6, v45792; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]5793; GFX9-NEXT: s_cbranch_execnz .LBB81_15794; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end5795; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]5796; GFX9-NEXT: s_setpc_b64 s[30:31]5797 %gep = getelementptr i64, ptr addrspace(1) %out, i64 45798 %tmp0 = atomicrmw max ptr addrspace(1) %gep, i64 %in seq_cst5799 ret void5800}5801 5802define i64 @global_atomic_max_i64_ret(ptr addrspace(1) %ptr, i64 %in) {5803; SI-LABEL: global_atomic_max_i64_ret:5804; SI: ; %bb.0:5805; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5806; SI-NEXT: v_mov_b32_e32 v5, v35807; SI-NEXT: v_mov_b32_e32 v4, v25808; SI-NEXT: v_mov_b32_e32 v7, v15809; SI-NEXT: v_mov_b32_e32 v6, v05810; SI-NEXT: s_mov_b32 s6, 05811; SI-NEXT: s_mov_b32 s7, 0xf0005812; SI-NEXT: s_mov_b32 s4, s65813; SI-NEXT: s_mov_b32 s5, s65814; SI-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[4:7], 0 addr645815; SI-NEXT: s_mov_b64 s[8:9], 05816; SI-NEXT: .LBB82_1: ; %atomicrmw.start5817; SI-NEXT: ; =>This Inner Loop Header: Depth=15818; SI-NEXT: s_waitcnt vmcnt(0)5819; SI-NEXT: v_cmp_gt_i64_e32 vcc, v[10:11], v[4:5]5820; SI-NEXT: v_cndmask_b32_e32 v9, v5, v11, vcc5821; SI-NEXT: v_cndmask_b32_e32 v8, v4, v10, vcc5822; SI-NEXT: s_waitcnt expcnt(0)5823; SI-NEXT: v_mov_b32_e32 v0, v85824; SI-NEXT: v_mov_b32_e32 v1, v95825; SI-NEXT: v_mov_b32_e32 v2, v105826; SI-NEXT: v_mov_b32_e32 v3, v115827; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc5828; SI-NEXT: s_waitcnt vmcnt(0)5829; SI-NEXT: buffer_wbinvl15830; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]5831; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]5832; SI-NEXT: v_mov_b32_e32 v11, v15833; SI-NEXT: v_mov_b32_e32 v10, v05834; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]5835; SI-NEXT: s_cbranch_execnz .LBB82_15836; SI-NEXT: ; %bb.2: ; %atomicrmw.end5837; SI-NEXT: s_or_b64 exec, exec, s[8:9]5838; SI-NEXT: s_waitcnt expcnt(0)5839; SI-NEXT: s_setpc_b64 s[30:31]5840;5841; VI-LABEL: global_atomic_max_i64_ret:5842; VI: ; %bb.0:5843; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5844; VI-NEXT: flat_load_dwordx2 v[4:5], v[0:1]5845; VI-NEXT: s_mov_b64 s[4:5], 05846; VI-NEXT: .LBB82_1: ; %atomicrmw.start5847; VI-NEXT: ; =>This Inner Loop Header: Depth=15848; VI-NEXT: s_waitcnt vmcnt(0)5849; VI-NEXT: v_mov_b32_e32 v7, v55850; VI-NEXT: v_mov_b32_e32 v6, v45851; VI-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]5852; VI-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc5853; VI-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc5854; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc5855; VI-NEXT: s_waitcnt vmcnt(0)5856; VI-NEXT: buffer_wbinvl1_vol5857; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]5858; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5859; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]5860; VI-NEXT: s_cbranch_execnz .LBB82_15861; VI-NEXT: ; %bb.2: ; %atomicrmw.end5862; VI-NEXT: s_or_b64 exec, exec, s[4:5]5863; VI-NEXT: v_mov_b32_e32 v0, v45864; VI-NEXT: v_mov_b32_e32 v1, v55865; VI-NEXT: s_setpc_b64 s[30:31]5866;5867; GFX9-LABEL: global_atomic_max_i64_ret:5868; GFX9: ; %bb.0:5869; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5870; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off5871; GFX9-NEXT: s_mov_b64 s[4:5], 05872; GFX9-NEXT: .LBB82_1: ; %atomicrmw.start5873; GFX9-NEXT: ; =>This Inner Loop Header: Depth=15874; GFX9-NEXT: s_waitcnt vmcnt(0)5875; GFX9-NEXT: v_mov_b32_e32 v7, v55876; GFX9-NEXT: v_mov_b32_e32 v6, v45877; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]5878; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc5879; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc5880; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc5881; GFX9-NEXT: s_waitcnt vmcnt(0)5882; GFX9-NEXT: buffer_wbinvl1_vol5883; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]5884; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5885; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]5886; GFX9-NEXT: s_cbranch_execnz .LBB82_15887; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end5888; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]5889; GFX9-NEXT: v_mov_b32_e32 v0, v45890; GFX9-NEXT: v_mov_b32_e32 v1, v55891; GFX9-NEXT: s_setpc_b64 s[30:31]5892 %result = atomicrmw max ptr addrspace(1) %ptr, i64 %in seq_cst5893 ret i64 %result5894}5895 5896define i64 @global_atomic_max_i64_ret_offset(ptr addrspace(1) %out, i64 %in) {5897; SI-LABEL: global_atomic_max_i64_ret_offset:5898; SI: ; %bb.0:5899; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5900; SI-NEXT: v_mov_b32_e32 v5, v35901; SI-NEXT: v_mov_b32_e32 v4, v25902; SI-NEXT: v_mov_b32_e32 v7, v15903; SI-NEXT: v_mov_b32_e32 v6, v05904; SI-NEXT: s_mov_b32 s6, 05905; SI-NEXT: s_mov_b32 s7, 0xf0005906; SI-NEXT: s_mov_b32 s4, s65907; SI-NEXT: s_mov_b32 s5, s65908; SI-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[4:7], 0 addr64 offset:325909; SI-NEXT: s_mov_b64 s[8:9], 05910; SI-NEXT: .LBB83_1: ; %atomicrmw.start5911; SI-NEXT: ; =>This Inner Loop Header: Depth=15912; SI-NEXT: s_waitcnt vmcnt(0)5913; SI-NEXT: v_cmp_gt_i64_e32 vcc, v[10:11], v[4:5]5914; SI-NEXT: v_cndmask_b32_e32 v9, v5, v11, vcc5915; SI-NEXT: v_cndmask_b32_e32 v8, v4, v10, vcc5916; SI-NEXT: s_waitcnt expcnt(0)5917; SI-NEXT: v_mov_b32_e32 v0, v85918; SI-NEXT: v_mov_b32_e32 v1, v95919; SI-NEXT: v_mov_b32_e32 v2, v105920; SI-NEXT: v_mov_b32_e32 v3, v115921; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 offset:32 glc5922; SI-NEXT: s_waitcnt vmcnt(0)5923; SI-NEXT: buffer_wbinvl15924; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]5925; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]5926; SI-NEXT: v_mov_b32_e32 v11, v15927; SI-NEXT: v_mov_b32_e32 v10, v05928; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]5929; SI-NEXT: s_cbranch_execnz .LBB83_15930; SI-NEXT: ; %bb.2: ; %atomicrmw.end5931; SI-NEXT: s_or_b64 exec, exec, s[8:9]5932; SI-NEXT: s_waitcnt expcnt(0)5933; SI-NEXT: s_setpc_b64 s[30:31]5934;5935; VI-LABEL: global_atomic_max_i64_ret_offset:5936; VI: ; %bb.0:5937; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5938; VI-NEXT: v_add_u32_e32 v4, vcc, 32, v05939; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc5940; VI-NEXT: flat_load_dwordx2 v[0:1], v[4:5]5941; VI-NEXT: s_mov_b64 s[4:5], 05942; VI-NEXT: .LBB83_1: ; %atomicrmw.start5943; VI-NEXT: ; =>This Inner Loop Header: Depth=15944; VI-NEXT: s_waitcnt vmcnt(0)5945; VI-NEXT: v_mov_b32_e32 v9, v15946; VI-NEXT: v_mov_b32_e32 v8, v05947; VI-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]5948; VI-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc5949; VI-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc5950; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc5951; VI-NEXT: s_waitcnt vmcnt(0)5952; VI-NEXT: buffer_wbinvl1_vol5953; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]5954; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5955; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]5956; VI-NEXT: s_cbranch_execnz .LBB83_15957; VI-NEXT: ; %bb.2: ; %atomicrmw.end5958; VI-NEXT: s_or_b64 exec, exec, s[4:5]5959; VI-NEXT: s_setpc_b64 s[30:31]5960;5961; GFX9-LABEL: global_atomic_max_i64_ret_offset:5962; GFX9: ; %bb.0:5963; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5964; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:325965; GFX9-NEXT: s_mov_b64 s[4:5], 05966; GFX9-NEXT: .LBB83_1: ; %atomicrmw.start5967; GFX9-NEXT: ; =>This Inner Loop Header: Depth=15968; GFX9-NEXT: s_waitcnt vmcnt(0)5969; GFX9-NEXT: v_mov_b32_e32 v7, v55970; GFX9-NEXT: v_mov_b32_e32 v6, v45971; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]5972; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc5973; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc5974; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc5975; GFX9-NEXT: s_waitcnt vmcnt(0)5976; GFX9-NEXT: buffer_wbinvl1_vol5977; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]5978; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5979; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]5980; GFX9-NEXT: s_cbranch_execnz .LBB83_15981; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end5982; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]5983; GFX9-NEXT: v_mov_b32_e32 v0, v45984; GFX9-NEXT: v_mov_b32_e32 v1, v55985; GFX9-NEXT: s_setpc_b64 s[30:31]5986 %gep = getelementptr i64, ptr addrspace(1) %out, i64 45987 %result = atomicrmw max ptr addrspace(1) %gep, i64 %in seq_cst5988 ret i64 %result5989}5990 5991define amdgpu_gfx void @global_atomic_max_i64_noret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {5992; SI-LABEL: global_atomic_max_i64_noret_scalar:5993; SI: ; %bb.0:5994; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5995; SI-NEXT: s_xor_saveexec_b64 s[34:35], -15996; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill5997; SI-NEXT: s_mov_b64 exec, s[34:35]5998; SI-NEXT: s_waitcnt expcnt(0)5999; SI-NEXT: v_writelane_b32 v10, s6, 06000; SI-NEXT: v_writelane_b32 v10, s7, 16001; SI-NEXT: s_mov_b32 s35, s76002; SI-NEXT: s_mov_b32 s34, s66003; SI-NEXT: s_mov_b32 s7, 0xf0006004; SI-NEXT: s_mov_b32 s6, -16005; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 06006; SI-NEXT: s_mov_b64 s[36:37], 06007; SI-NEXT: v_mov_b32_e32 v4, s356008; SI-NEXT: v_mov_b32_e32 v5, s346009; SI-NEXT: .LBB84_1: ; %atomicrmw.start6010; SI-NEXT: ; =>This Inner Loop Header: Depth=16011; SI-NEXT: s_waitcnt vmcnt(0)6012; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[34:35], v[2:3]6013; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc6014; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc6015; SI-NEXT: s_waitcnt expcnt(0)6016; SI-NEXT: v_mov_b32_e32 v9, v36017; SI-NEXT: v_mov_b32_e32 v8, v26018; SI-NEXT: v_mov_b32_e32 v7, v16019; SI-NEXT: v_mov_b32_e32 v6, v06020; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 glc6021; SI-NEXT: s_waitcnt vmcnt(0)6022; SI-NEXT: buffer_wbinvl16023; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]6024; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]6025; SI-NEXT: v_mov_b32_e32 v2, v66026; SI-NEXT: v_mov_b32_e32 v3, v76027; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]6028; SI-NEXT: s_cbranch_execnz .LBB84_16029; SI-NEXT: ; %bb.2: ; %atomicrmw.end6030; SI-NEXT: s_or_b64 exec, exec, s[36:37]6031; SI-NEXT: v_readlane_b32 s7, v10, 16032; SI-NEXT: v_readlane_b32 s6, v10, 06033; SI-NEXT: s_xor_saveexec_b64 s[34:35], -16034; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload6035; SI-NEXT: s_mov_b64 exec, s[34:35]6036; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)6037; SI-NEXT: s_setpc_b64 s[30:31]6038;6039; VI-LABEL: global_atomic_max_i64_noret_scalar:6040; VI: ; %bb.0:6041; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6042; VI-NEXT: v_mov_b32_e32 v0, s46043; VI-NEXT: v_mov_b32_e32 v1, s56044; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]6045; VI-NEXT: v_mov_b32_e32 v4, s46046; VI-NEXT: s_mov_b64 s[34:35], 06047; VI-NEXT: v_mov_b32_e32 v6, s76048; VI-NEXT: v_mov_b32_e32 v7, s66049; VI-NEXT: v_mov_b32_e32 v5, s56050; VI-NEXT: .LBB84_1: ; %atomicrmw.start6051; VI-NEXT: ; =>This Inner Loop Header: Depth=16052; VI-NEXT: s_waitcnt vmcnt(0)6053; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]6054; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc6055; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc6056; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc6057; VI-NEXT: s_waitcnt vmcnt(0)6058; VI-NEXT: buffer_wbinvl1_vol6059; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6060; VI-NEXT: v_mov_b32_e32 v3, v16061; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]6062; VI-NEXT: v_mov_b32_e32 v2, v06063; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]6064; VI-NEXT: s_cbranch_execnz .LBB84_16065; VI-NEXT: ; %bb.2: ; %atomicrmw.end6066; VI-NEXT: s_or_b64 exec, exec, s[34:35]6067; VI-NEXT: s_setpc_b64 s[30:31]6068;6069; GFX9-LABEL: global_atomic_max_i64_noret_scalar:6070; GFX9: ; %bb.0:6071; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6072; GFX9-NEXT: v_mov_b32_e32 v4, 06073; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5]6074; GFX9-NEXT: s_mov_b64 s[34:35], 06075; GFX9-NEXT: v_mov_b32_e32 v5, s76076; GFX9-NEXT: v_mov_b32_e32 v6, s66077; GFX9-NEXT: .LBB84_1: ; %atomicrmw.start6078; GFX9-NEXT: ; =>This Inner Loop Header: Depth=16079; GFX9-NEXT: s_waitcnt vmcnt(0)6080; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]6081; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc6082; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc6083; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc6084; GFX9-NEXT: s_waitcnt vmcnt(0)6085; GFX9-NEXT: buffer_wbinvl1_vol6086; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6087; GFX9-NEXT: v_mov_b32_e32 v3, v16088; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]6089; GFX9-NEXT: v_mov_b32_e32 v2, v06090; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]6091; GFX9-NEXT: s_cbranch_execnz .LBB84_16092; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end6093; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]6094; GFX9-NEXT: s_setpc_b64 s[30:31]6095 %tmp0 = atomicrmw max ptr addrspace(1) %ptr, i64 %in seq_cst6096 ret void6097}6098 6099define amdgpu_gfx void @global_atomic_max_i64_noret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {6100; SI-LABEL: global_atomic_max_i64_noret_offset_scalar:6101; SI: ; %bb.0:6102; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6103; SI-NEXT: s_xor_saveexec_b64 s[34:35], -16104; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill6105; SI-NEXT: s_mov_b64 exec, s[34:35]6106; SI-NEXT: s_waitcnt expcnt(0)6107; SI-NEXT: v_writelane_b32 v10, s6, 06108; SI-NEXT: v_writelane_b32 v10, s7, 16109; SI-NEXT: s_mov_b32 s35, s76110; SI-NEXT: s_mov_b32 s34, s66111; SI-NEXT: s_mov_b32 s7, 0xf0006112; SI-NEXT: s_mov_b32 s6, -16113; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:326114; SI-NEXT: s_mov_b64 s[36:37], 06115; SI-NEXT: v_mov_b32_e32 v4, s356116; SI-NEXT: v_mov_b32_e32 v5, s346117; SI-NEXT: .LBB85_1: ; %atomicrmw.start6118; SI-NEXT: ; =>This Inner Loop Header: Depth=16119; SI-NEXT: s_waitcnt vmcnt(0)6120; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[34:35], v[2:3]6121; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc6122; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc6123; SI-NEXT: s_waitcnt expcnt(0)6124; SI-NEXT: v_mov_b32_e32 v9, v36125; SI-NEXT: v_mov_b32_e32 v8, v26126; SI-NEXT: v_mov_b32_e32 v7, v16127; SI-NEXT: v_mov_b32_e32 v6, v06128; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 offset:32 glc6129; SI-NEXT: s_waitcnt vmcnt(0)6130; SI-NEXT: buffer_wbinvl16131; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]6132; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]6133; SI-NEXT: v_mov_b32_e32 v2, v66134; SI-NEXT: v_mov_b32_e32 v3, v76135; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]6136; SI-NEXT: s_cbranch_execnz .LBB85_16137; SI-NEXT: ; %bb.2: ; %atomicrmw.end6138; SI-NEXT: s_or_b64 exec, exec, s[36:37]6139; SI-NEXT: v_readlane_b32 s7, v10, 16140; SI-NEXT: v_readlane_b32 s6, v10, 06141; SI-NEXT: s_xor_saveexec_b64 s[34:35], -16142; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload6143; SI-NEXT: s_mov_b64 exec, s[34:35]6144; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)6145; SI-NEXT: s_setpc_b64 s[30:31]6146;6147; VI-LABEL: global_atomic_max_i64_noret_offset_scalar:6148; VI: ; %bb.0:6149; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6150; VI-NEXT: s_add_u32 s34, s4, 326151; VI-NEXT: s_addc_u32 s35, s5, 06152; VI-NEXT: v_mov_b32_e32 v4, s346153; VI-NEXT: v_mov_b32_e32 v5, s356154; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]6155; VI-NEXT: s_mov_b64 s[34:35], 06156; VI-NEXT: v_mov_b32_e32 v6, s76157; VI-NEXT: v_mov_b32_e32 v7, s66158; VI-NEXT: .LBB85_1: ; %atomicrmw.start6159; VI-NEXT: ; =>This Inner Loop Header: Depth=16160; VI-NEXT: s_waitcnt vmcnt(0)6161; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]6162; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc6163; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc6164; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc6165; VI-NEXT: s_waitcnt vmcnt(0)6166; VI-NEXT: buffer_wbinvl1_vol6167; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6168; VI-NEXT: v_mov_b32_e32 v3, v16169; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]6170; VI-NEXT: v_mov_b32_e32 v2, v06171; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]6172; VI-NEXT: s_cbranch_execnz .LBB85_16173; VI-NEXT: ; %bb.2: ; %atomicrmw.end6174; VI-NEXT: s_or_b64 exec, exec, s[34:35]6175; VI-NEXT: s_setpc_b64 s[30:31]6176;6177; GFX9-LABEL: global_atomic_max_i64_noret_offset_scalar:6178; GFX9: ; %bb.0:6179; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6180; GFX9-NEXT: v_mov_b32_e32 v4, 06181; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5] offset:326182; GFX9-NEXT: s_mov_b64 s[34:35], 06183; GFX9-NEXT: v_mov_b32_e32 v5, s76184; GFX9-NEXT: v_mov_b32_e32 v6, s66185; GFX9-NEXT: .LBB85_1: ; %atomicrmw.start6186; GFX9-NEXT: ; =>This Inner Loop Header: Depth=16187; GFX9-NEXT: s_waitcnt vmcnt(0)6188; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]6189; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc6190; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc6191; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc6192; GFX9-NEXT: s_waitcnt vmcnt(0)6193; GFX9-NEXT: buffer_wbinvl1_vol6194; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6195; GFX9-NEXT: v_mov_b32_e32 v3, v16196; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]6197; GFX9-NEXT: v_mov_b32_e32 v2, v06198; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]6199; GFX9-NEXT: s_cbranch_execnz .LBB85_16200; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end6201; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]6202; GFX9-NEXT: s_setpc_b64 s[30:31]6203 %gep = getelementptr i64, ptr addrspace(1) %out, i64 46204 %tmp0 = atomicrmw max ptr addrspace(1) %gep, i64 %in seq_cst6205 ret void6206}6207 6208define amdgpu_gfx i64 @global_atomic_max_i64_ret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {6209; SI-LABEL: global_atomic_max_i64_ret_scalar:6210; SI: ; %bb.0:6211; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6212; SI-NEXT: s_xor_saveexec_b64 s[34:35], -16213; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill6214; SI-NEXT: s_mov_b64 exec, s[34:35]6215; SI-NEXT: s_waitcnt expcnt(0)6216; SI-NEXT: v_writelane_b32 v8, s6, 06217; SI-NEXT: v_writelane_b32 v8, s7, 16218; SI-NEXT: s_mov_b32 s35, s76219; SI-NEXT: s_mov_b32 s34, s66220; SI-NEXT: s_mov_b32 s7, 0xf0006221; SI-NEXT: s_mov_b32 s6, -16222; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 06223; SI-NEXT: s_mov_b64 s[36:37], 06224; SI-NEXT: v_mov_b32_e32 v6, s356225; SI-NEXT: v_mov_b32_e32 v7, s346226; SI-NEXT: .LBB86_1: ; %atomicrmw.start6227; SI-NEXT: ; =>This Inner Loop Header: Depth=16228; SI-NEXT: s_waitcnt vmcnt(0)6229; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[34:35], v[4:5]6230; SI-NEXT: s_waitcnt expcnt(0)6231; SI-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc6232; SI-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc6233; SI-NEXT: v_mov_b32_e32 v0, v26234; SI-NEXT: v_mov_b32_e32 v1, v36235; SI-NEXT: v_mov_b32_e32 v2, v46236; SI-NEXT: v_mov_b32_e32 v3, v56237; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc6238; SI-NEXT: s_waitcnt vmcnt(0)6239; SI-NEXT: buffer_wbinvl16240; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]6241; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]6242; SI-NEXT: v_mov_b32_e32 v5, v16243; SI-NEXT: v_mov_b32_e32 v4, v06244; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]6245; SI-NEXT: s_cbranch_execnz .LBB86_16246; SI-NEXT: ; %bb.2: ; %atomicrmw.end6247; SI-NEXT: s_or_b64 exec, exec, s[36:37]6248; SI-NEXT: v_readlane_b32 s7, v8, 16249; SI-NEXT: v_readlane_b32 s6, v8, 06250; SI-NEXT: s_xor_saveexec_b64 s[34:35], -16251; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload6252; SI-NEXT: s_mov_b64 exec, s[34:35]6253; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)6254; SI-NEXT: s_setpc_b64 s[30:31]6255;6256; VI-LABEL: global_atomic_max_i64_ret_scalar:6257; VI: ; %bb.0:6258; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6259; VI-NEXT: v_mov_b32_e32 v0, s46260; VI-NEXT: v_mov_b32_e32 v1, s56261; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]6262; VI-NEXT: v_mov_b32_e32 v2, s46263; VI-NEXT: s_mov_b64 s[34:35], 06264; VI-NEXT: v_mov_b32_e32 v4, s76265; VI-NEXT: v_mov_b32_e32 v5, s66266; VI-NEXT: v_mov_b32_e32 v3, s56267; VI-NEXT: .LBB86_1: ; %atomicrmw.start6268; VI-NEXT: ; =>This Inner Loop Header: Depth=16269; VI-NEXT: s_waitcnt vmcnt(0)6270; VI-NEXT: v_mov_b32_e32 v9, v16271; VI-NEXT: v_mov_b32_e32 v8, v06272; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]6273; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc6274; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc6275; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc6276; VI-NEXT: s_waitcnt vmcnt(0)6277; VI-NEXT: buffer_wbinvl1_vol6278; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]6279; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]6280; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]6281; VI-NEXT: s_cbranch_execnz .LBB86_16282; VI-NEXT: ; %bb.2: ; %atomicrmw.end6283; VI-NEXT: s_or_b64 exec, exec, s[34:35]6284; VI-NEXT: s_setpc_b64 s[30:31]6285;6286; GFX9-LABEL: global_atomic_max_i64_ret_scalar:6287; GFX9: ; %bb.0:6288; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6289; GFX9-NEXT: v_mov_b32_e32 v2, 06290; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]6291; GFX9-NEXT: s_mov_b64 s[34:35], 06292; GFX9-NEXT: v_mov_b32_e32 v3, s76293; GFX9-NEXT: v_mov_b32_e32 v4, s66294; GFX9-NEXT: .LBB86_1: ; %atomicrmw.start6295; GFX9-NEXT: ; =>This Inner Loop Header: Depth=16296; GFX9-NEXT: s_waitcnt vmcnt(0)6297; GFX9-NEXT: v_mov_b32_e32 v8, v16298; GFX9-NEXT: v_mov_b32_e32 v7, v06299; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[7:8]6300; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc6301; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc6302; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] glc6303; GFX9-NEXT: s_waitcnt vmcnt(0)6304; GFX9-NEXT: buffer_wbinvl1_vol6305; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]6306; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]6307; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]6308; GFX9-NEXT: s_cbranch_execnz .LBB86_16309; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end6310; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]6311; GFX9-NEXT: s_setpc_b64 s[30:31]6312 %result = atomicrmw max ptr addrspace(1) %ptr, i64 %in seq_cst6313 ret i64 %result6314}6315 6316define amdgpu_gfx i64 @global_atomic_max_i64_ret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {6317; SI-LABEL: global_atomic_max_i64_ret_offset_scalar:6318; SI: ; %bb.0:6319; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6320; SI-NEXT: s_xor_saveexec_b64 s[34:35], -16321; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill6322; SI-NEXT: s_mov_b64 exec, s[34:35]6323; SI-NEXT: s_waitcnt expcnt(0)6324; SI-NEXT: v_writelane_b32 v8, s6, 06325; SI-NEXT: v_writelane_b32 v8, s7, 16326; SI-NEXT: s_mov_b32 s35, s76327; SI-NEXT: s_mov_b32 s34, s66328; SI-NEXT: s_mov_b32 s7, 0xf0006329; SI-NEXT: s_mov_b32 s6, -16330; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0 offset:326331; SI-NEXT: s_mov_b64 s[36:37], 06332; SI-NEXT: v_mov_b32_e32 v6, s356333; SI-NEXT: v_mov_b32_e32 v7, s346334; SI-NEXT: .LBB87_1: ; %atomicrmw.start6335; SI-NEXT: ; =>This Inner Loop Header: Depth=16336; SI-NEXT: s_waitcnt vmcnt(0)6337; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[34:35], v[4:5]6338; SI-NEXT: s_waitcnt expcnt(0)6339; SI-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc6340; SI-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc6341; SI-NEXT: v_mov_b32_e32 v0, v26342; SI-NEXT: v_mov_b32_e32 v1, v36343; SI-NEXT: v_mov_b32_e32 v2, v46344; SI-NEXT: v_mov_b32_e32 v3, v56345; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 offset:32 glc6346; SI-NEXT: s_waitcnt vmcnt(0)6347; SI-NEXT: buffer_wbinvl16348; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]6349; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]6350; SI-NEXT: v_mov_b32_e32 v5, v16351; SI-NEXT: v_mov_b32_e32 v4, v06352; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]6353; SI-NEXT: s_cbranch_execnz .LBB87_16354; SI-NEXT: ; %bb.2: ; %atomicrmw.end6355; SI-NEXT: s_or_b64 exec, exec, s[36:37]6356; SI-NEXT: v_readlane_b32 s7, v8, 16357; SI-NEXT: v_readlane_b32 s6, v8, 06358; SI-NEXT: s_xor_saveexec_b64 s[34:35], -16359; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload6360; SI-NEXT: s_mov_b64 exec, s[34:35]6361; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)6362; SI-NEXT: s_setpc_b64 s[30:31]6363;6364; VI-LABEL: global_atomic_max_i64_ret_offset_scalar:6365; VI: ; %bb.0:6366; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6367; VI-NEXT: s_add_u32 s34, s4, 326368; VI-NEXT: s_addc_u32 s35, s5, 06369; VI-NEXT: v_mov_b32_e32 v2, s346370; VI-NEXT: v_mov_b32_e32 v3, s356371; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]6372; VI-NEXT: s_mov_b64 s[34:35], 06373; VI-NEXT: v_mov_b32_e32 v4, s76374; VI-NEXT: v_mov_b32_e32 v5, s66375; VI-NEXT: .LBB87_1: ; %atomicrmw.start6376; VI-NEXT: ; =>This Inner Loop Header: Depth=16377; VI-NEXT: s_waitcnt vmcnt(0)6378; VI-NEXT: v_mov_b32_e32 v9, v16379; VI-NEXT: v_mov_b32_e32 v8, v06380; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]6381; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc6382; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc6383; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc6384; VI-NEXT: s_waitcnt vmcnt(0)6385; VI-NEXT: buffer_wbinvl1_vol6386; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]6387; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]6388; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]6389; VI-NEXT: s_cbranch_execnz .LBB87_16390; VI-NEXT: ; %bb.2: ; %atomicrmw.end6391; VI-NEXT: s_or_b64 exec, exec, s[34:35]6392; VI-NEXT: s_setpc_b64 s[30:31]6393;6394; GFX9-LABEL: global_atomic_max_i64_ret_offset_scalar:6395; GFX9: ; %bb.0:6396; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6397; GFX9-NEXT: v_mov_b32_e32 v2, 06398; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5] offset:326399; GFX9-NEXT: s_mov_b64 s[34:35], 06400; GFX9-NEXT: v_mov_b32_e32 v3, s76401; GFX9-NEXT: v_mov_b32_e32 v4, s66402; GFX9-NEXT: .LBB87_1: ; %atomicrmw.start6403; GFX9-NEXT: ; =>This Inner Loop Header: Depth=16404; GFX9-NEXT: s_waitcnt vmcnt(0)6405; GFX9-NEXT: v_mov_b32_e32 v8, v16406; GFX9-NEXT: v_mov_b32_e32 v7, v06407; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[7:8]6408; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc6409; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc6410; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] offset:32 glc6411; GFX9-NEXT: s_waitcnt vmcnt(0)6412; GFX9-NEXT: buffer_wbinvl1_vol6413; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]6414; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]6415; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]6416; GFX9-NEXT: s_cbranch_execnz .LBB87_16417; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end6418; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]6419; GFX9-NEXT: s_setpc_b64 s[30:31]6420 %gep = getelementptr i64, ptr addrspace(1) %out, i64 46421 %result = atomicrmw max ptr addrspace(1) %gep, i64 %in seq_cst6422 ret i64 %result6423}6424 6425define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr addrspace(1) %out, i64 %in, i64 %index) {6426; SI-LABEL: atomic_max_i64_addr64_offset:6427; SI: ; %bb.0: ; %entry6428; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd6429; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96430; SI-NEXT: s_waitcnt lgkmcnt(0)6431; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 36432; SI-NEXT: s_add_u32 s4, s0, s46433; SI-NEXT: s_addc_u32 s5, s1, s56434; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x86435; SI-NEXT: s_mov_b64 s[0:1], 06436; SI-NEXT: s_mov_b32 s7, 0xf0006437; SI-NEXT: v_mov_b32_e32 v4, s36438; SI-NEXT: v_mov_b32_e32 v5, s26439; SI-NEXT: s_waitcnt lgkmcnt(0)6440; SI-NEXT: v_mov_b32_e32 v2, s86441; SI-NEXT: v_mov_b32_e32 v3, s96442; SI-NEXT: s_mov_b32 s6, -16443; SI-NEXT: .LBB88_1: ; %atomicrmw.start6444; SI-NEXT: ; =>This Inner Loop Header: Depth=16445; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]6446; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc6447; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc6448; SI-NEXT: s_waitcnt expcnt(0)6449; SI-NEXT: v_mov_b32_e32 v9, v36450; SI-NEXT: v_mov_b32_e32 v8, v26451; SI-NEXT: v_mov_b32_e32 v7, v16452; SI-NEXT: v_mov_b32_e32 v6, v06453; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 offset:32 glc6454; SI-NEXT: s_waitcnt vmcnt(0)6455; SI-NEXT: buffer_wbinvl16456; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]6457; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]6458; SI-NEXT: v_mov_b32_e32 v2, v66459; SI-NEXT: v_mov_b32_e32 v3, v76460; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]6461; SI-NEXT: s_cbranch_execnz .LBB88_16462; SI-NEXT: ; %bb.2: ; %atomicrmw.end6463; SI-NEXT: s_endpgm6464;6465; VI-LABEL: atomic_max_i64_addr64_offset:6466; VI: ; %bb.0: ; %entry6467; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346468; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246469; VI-NEXT: s_mov_b64 s[4:5], 06470; VI-NEXT: s_waitcnt lgkmcnt(0)6471; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 36472; VI-NEXT: s_add_u32 s0, s0, s66473; VI-NEXT: s_addc_u32 s1, s1, s76474; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x206475; VI-NEXT: s_add_u32 s0, s0, 326476; VI-NEXT: s_addc_u32 s1, s1, 06477; VI-NEXT: v_mov_b32_e32 v5, s16478; VI-NEXT: v_mov_b32_e32 v6, s36479; VI-NEXT: s_waitcnt lgkmcnt(0)6480; VI-NEXT: v_mov_b32_e32 v2, s66481; VI-NEXT: v_mov_b32_e32 v7, s26482; VI-NEXT: v_mov_b32_e32 v3, s76483; VI-NEXT: v_mov_b32_e32 v4, s06484; VI-NEXT: .LBB88_1: ; %atomicrmw.start6485; VI-NEXT: ; =>This Inner Loop Header: Depth=16486; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]6487; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc6488; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc6489; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc6490; VI-NEXT: s_waitcnt vmcnt(0)6491; VI-NEXT: buffer_wbinvl1_vol6492; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6493; VI-NEXT: v_mov_b32_e32 v3, v16494; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6495; VI-NEXT: v_mov_b32_e32 v2, v06496; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]6497; VI-NEXT: s_cbranch_execnz .LBB88_16498; VI-NEXT: ; %bb.2: ; %atomicrmw.end6499; VI-NEXT: s_endpgm6500;6501; GFX9-LABEL: atomic_max_i64_addr64_offset:6502; GFX9: ; %bb.0: ; %entry6503; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346504; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246505; GFX9-NEXT: v_mov_b32_e32 v6, 06506; GFX9-NEXT: s_waitcnt lgkmcnt(0)6507; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 36508; GFX9-NEXT: s_add_u32 s0, s0, s46509; GFX9-NEXT: s_addc_u32 s1, s1, s56510; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x206511; GFX9-NEXT: s_mov_b64 s[4:5], 06512; GFX9-NEXT: v_mov_b32_e32 v4, s36513; GFX9-NEXT: v_mov_b32_e32 v5, s26514; GFX9-NEXT: s_waitcnt lgkmcnt(0)6515; GFX9-NEXT: v_mov_b32_e32 v2, s66516; GFX9-NEXT: v_mov_b32_e32 v3, s76517; GFX9-NEXT: .LBB88_1: ; %atomicrmw.start6518; GFX9-NEXT: ; =>This Inner Loop Header: Depth=16519; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]6520; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc6521; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc6522; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] offset:32 glc6523; GFX9-NEXT: s_waitcnt vmcnt(0)6524; GFX9-NEXT: buffer_wbinvl1_vol6525; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6526; GFX9-NEXT: v_mov_b32_e32 v3, v16527; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6528; GFX9-NEXT: v_mov_b32_e32 v2, v06529; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]6530; GFX9-NEXT: s_cbranch_execnz .LBB88_16531; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end6532; GFX9-NEXT: s_endpgm6533entry:6534 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index6535 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 46536 %tmp0 = atomicrmw max ptr addrspace(1) %gep, i64 %in seq_cst6537 ret void6538}6539 6540define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {6541; SI-LABEL: atomic_max_i64_ret_addr64_offset:6542; SI: ; %bb.0: ; %entry6543; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x96544; SI-NEXT: s_waitcnt lgkmcnt(0)6545; SI-NEXT: s_lshl_b64 s[6:7], s[6:7], 36546; SI-NEXT: s_add_u32 s8, s0, s66547; SI-NEXT: s_addc_u32 s9, s1, s76548; SI-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x86549; SI-NEXT: s_mov_b64 s[0:1], 06550; SI-NEXT: s_mov_b32 s11, 0xf0006551; SI-NEXT: v_mov_b32_e32 v8, s56552; SI-NEXT: v_mov_b32_e32 v9, s46553; SI-NEXT: s_waitcnt lgkmcnt(0)6554; SI-NEXT: v_mov_b32_e32 v2, s66555; SI-NEXT: v_mov_b32_e32 v3, s76556; SI-NEXT: s_mov_b32 s10, -16557; SI-NEXT: .LBB89_1: ; %atomicrmw.start6558; SI-NEXT: ; =>This Inner Loop Header: Depth=16559; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]6560; SI-NEXT: v_cndmask_b32_e32 v1, v8, v3, vcc6561; SI-NEXT: v_cndmask_b32_e32 v0, v9, v2, vcc6562; SI-NEXT: s_waitcnt expcnt(0)6563; SI-NEXT: v_mov_b32_e32 v7, v36564; SI-NEXT: v_mov_b32_e32 v6, v26565; SI-NEXT: v_mov_b32_e32 v5, v16566; SI-NEXT: v_mov_b32_e32 v4, v06567; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[8:11], 0 offset:32 glc6568; SI-NEXT: s_waitcnt vmcnt(0)6569; SI-NEXT: buffer_wbinvl16570; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]6571; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]6572; SI-NEXT: v_mov_b32_e32 v2, v46573; SI-NEXT: v_mov_b32_e32 v3, v56574; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]6575; SI-NEXT: s_cbranch_execnz .LBB89_16576; SI-NEXT: ; %bb.2: ; %atomicrmw.end6577; SI-NEXT: s_or_b64 exec, exec, s[0:1]6578; SI-NEXT: s_mov_b32 s7, 0xf0006579; SI-NEXT: s_mov_b32 s6, -16580; SI-NEXT: s_mov_b32 s4, s26581; SI-NEXT: s_mov_b32 s5, s36582; SI-NEXT: buffer_store_dwordx2 v[4:5], off, s[4:7], 06583; SI-NEXT: s_endpgm6584;6585; VI-LABEL: atomic_max_i64_ret_addr64_offset:6586; VI: ; %bb.0: ; %entry6587; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x246588; VI-NEXT: s_mov_b64 s[8:9], 06589; VI-NEXT: s_waitcnt lgkmcnt(0)6590; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 36591; VI-NEXT: s_add_u32 s0, s0, s66592; VI-NEXT: s_addc_u32 s1, s1, s76593; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x206594; VI-NEXT: s_add_u32 s0, s0, 326595; VI-NEXT: s_addc_u32 s1, s1, 06596; VI-NEXT: v_mov_b32_e32 v0, s06597; VI-NEXT: v_mov_b32_e32 v4, s56598; VI-NEXT: s_waitcnt lgkmcnt(0)6599; VI-NEXT: v_mov_b32_e32 v2, s66600; VI-NEXT: v_mov_b32_e32 v5, s46601; VI-NEXT: v_mov_b32_e32 v3, s76602; VI-NEXT: v_mov_b32_e32 v1, s16603; VI-NEXT: .LBB89_1: ; %atomicrmw.start6604; VI-NEXT: ; =>This Inner Loop Header: Depth=16605; VI-NEXT: v_mov_b32_e32 v9, v36606; VI-NEXT: v_mov_b32_e32 v8, v26607; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]6608; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc6609; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc6610; VI-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc6611; VI-NEXT: s_waitcnt vmcnt(0)6612; VI-NEXT: buffer_wbinvl1_vol6613; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]6614; VI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]6615; VI-NEXT: s_andn2_b64 exec, exec, s[8:9]6616; VI-NEXT: s_cbranch_execnz .LBB89_16617; VI-NEXT: ; %bb.2: ; %atomicrmw.end6618; VI-NEXT: s_or_b64 exec, exec, s[8:9]6619; VI-NEXT: v_mov_b32_e32 v0, s26620; VI-NEXT: v_mov_b32_e32 v1, s36621; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]6622; VI-NEXT: s_endpgm6623;6624; GFX9-LABEL: atomic_max_i64_ret_addr64_offset:6625; GFX9: ; %bb.0: ; %entry6626; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x246627; GFX9-NEXT: s_mov_b64 s[2:3], 06628; GFX9-NEXT: v_mov_b32_e32 v4, 06629; GFX9-NEXT: s_waitcnt lgkmcnt(0)6630; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 36631; GFX9-NEXT: s_add_u32 s0, s8, s06632; GFX9-NEXT: s_addc_u32 s1, s9, s16633; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x206634; GFX9-NEXT: v_mov_b32_e32 v2, s136635; GFX9-NEXT: v_mov_b32_e32 v3, s126636; GFX9-NEXT: s_waitcnt lgkmcnt(0)6637; GFX9-NEXT: v_mov_b32_e32 v0, s46638; GFX9-NEXT: v_mov_b32_e32 v1, s56639; GFX9-NEXT: .LBB89_1: ; %atomicrmw.start6640; GFX9-NEXT: ; =>This Inner Loop Header: Depth=16641; GFX9-NEXT: v_mov_b32_e32 v8, v16642; GFX9-NEXT: v_mov_b32_e32 v7, v06643; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[7:8]6644; GFX9-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc6645; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc6646; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[5:8], s[0:1] offset:32 glc6647; GFX9-NEXT: s_waitcnt vmcnt(0)6648; GFX9-NEXT: buffer_wbinvl1_vol6649; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]6650; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]6651; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]6652; GFX9-NEXT: s_cbranch_execnz .LBB89_16653; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end6654; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]6655; GFX9-NEXT: v_mov_b32_e32 v2, 06656; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]6657; GFX9-NEXT: s_endpgm6658entry:6659 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index6660 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 46661 %tmp0 = atomicrmw max ptr addrspace(1) %gep, i64 %in seq_cst6662 store i64 %tmp0, ptr addrspace(1) %out26663 ret void6664}6665 6666define amdgpu_kernel void @atomic_max_i64_addr64(ptr addrspace(1) %out, i64 %in, i64 %index) {6667; SI-LABEL: atomic_max_i64_addr64:6668; SI: ; %bb.0: ; %entry6669; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd6670; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96671; SI-NEXT: s_waitcnt lgkmcnt(0)6672; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 36673; SI-NEXT: s_add_u32 s4, s0, s46674; SI-NEXT: s_addc_u32 s5, s1, s56675; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x06676; SI-NEXT: s_mov_b64 s[0:1], 06677; SI-NEXT: s_mov_b32 s7, 0xf0006678; SI-NEXT: v_mov_b32_e32 v4, s36679; SI-NEXT: v_mov_b32_e32 v5, s26680; SI-NEXT: s_waitcnt lgkmcnt(0)6681; SI-NEXT: v_mov_b32_e32 v2, s86682; SI-NEXT: v_mov_b32_e32 v3, s96683; SI-NEXT: s_mov_b32 s6, -16684; SI-NEXT: .LBB90_1: ; %atomicrmw.start6685; SI-NEXT: ; =>This Inner Loop Header: Depth=16686; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]6687; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc6688; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc6689; SI-NEXT: s_waitcnt expcnt(0)6690; SI-NEXT: v_mov_b32_e32 v9, v36691; SI-NEXT: v_mov_b32_e32 v8, v26692; SI-NEXT: v_mov_b32_e32 v7, v16693; SI-NEXT: v_mov_b32_e32 v6, v06694; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 glc6695; SI-NEXT: s_waitcnt vmcnt(0)6696; SI-NEXT: buffer_wbinvl16697; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]6698; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]6699; SI-NEXT: v_mov_b32_e32 v2, v66700; SI-NEXT: v_mov_b32_e32 v3, v76701; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]6702; SI-NEXT: s_cbranch_execnz .LBB90_16703; SI-NEXT: ; %bb.2: ; %atomicrmw.end6704; SI-NEXT: s_endpgm6705;6706; VI-LABEL: atomic_max_i64_addr64:6707; VI: ; %bb.0: ; %entry6708; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346709; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246710; VI-NEXT: s_waitcnt lgkmcnt(0)6711; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 36712; VI-NEXT: s_add_u32 s4, s0, s46713; VI-NEXT: s_addc_u32 s5, s1, s56714; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x06715; VI-NEXT: v_mov_b32_e32 v4, s46716; VI-NEXT: s_mov_b64 s[0:1], 06717; VI-NEXT: v_mov_b32_e32 v6, s36718; VI-NEXT: v_mov_b32_e32 v7, s26719; VI-NEXT: s_waitcnt lgkmcnt(0)6720; VI-NEXT: v_mov_b32_e32 v2, s66721; VI-NEXT: v_mov_b32_e32 v3, s76722; VI-NEXT: v_mov_b32_e32 v5, s56723; VI-NEXT: .LBB90_1: ; %atomicrmw.start6724; VI-NEXT: ; =>This Inner Loop Header: Depth=16725; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]6726; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc6727; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc6728; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc6729; VI-NEXT: s_waitcnt vmcnt(0)6730; VI-NEXT: buffer_wbinvl1_vol6731; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6732; VI-NEXT: v_mov_b32_e32 v3, v16733; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]6734; VI-NEXT: v_mov_b32_e32 v2, v06735; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]6736; VI-NEXT: s_cbranch_execnz .LBB90_16737; VI-NEXT: ; %bb.2: ; %atomicrmw.end6738; VI-NEXT: s_endpgm6739;6740; GFX9-LABEL: atomic_max_i64_addr64:6741; GFX9: ; %bb.0: ; %entry6742; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346743; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246744; GFX9-NEXT: v_mov_b32_e32 v6, 06745; GFX9-NEXT: s_waitcnt lgkmcnt(0)6746; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 36747; GFX9-NEXT: s_add_u32 s0, s0, s46748; GFX9-NEXT: s_addc_u32 s1, s1, s56749; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x06750; GFX9-NEXT: s_mov_b64 s[4:5], 06751; GFX9-NEXT: v_mov_b32_e32 v4, s36752; GFX9-NEXT: v_mov_b32_e32 v5, s26753; GFX9-NEXT: s_waitcnt lgkmcnt(0)6754; GFX9-NEXT: v_mov_b32_e32 v2, s66755; GFX9-NEXT: v_mov_b32_e32 v3, s76756; GFX9-NEXT: .LBB90_1: ; %atomicrmw.start6757; GFX9-NEXT: ; =>This Inner Loop Header: Depth=16758; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]6759; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc6760; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc6761; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc6762; GFX9-NEXT: s_waitcnt vmcnt(0)6763; GFX9-NEXT: buffer_wbinvl1_vol6764; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6765; GFX9-NEXT: v_mov_b32_e32 v3, v16766; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6767; GFX9-NEXT: v_mov_b32_e32 v2, v06768; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]6769; GFX9-NEXT: s_cbranch_execnz .LBB90_16770; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end6771; GFX9-NEXT: s_endpgm6772entry:6773 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index6774 %tmp0 = atomicrmw max ptr addrspace(1) %ptr, i64 %in seq_cst6775 ret void6776}6777 6778define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {6779; SI-LABEL: atomic_max_i64_ret_addr64:6780; SI: ; %bb.0: ; %entry6781; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x96782; SI-NEXT: s_waitcnt lgkmcnt(0)6783; SI-NEXT: s_lshl_b64 s[6:7], s[6:7], 36784; SI-NEXT: s_add_u32 s8, s0, s66785; SI-NEXT: s_addc_u32 s9, s1, s76786; SI-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x06787; SI-NEXT: s_mov_b64 s[0:1], 06788; SI-NEXT: s_mov_b32 s11, 0xf0006789; SI-NEXT: v_mov_b32_e32 v8, s56790; SI-NEXT: v_mov_b32_e32 v9, s46791; SI-NEXT: s_waitcnt lgkmcnt(0)6792; SI-NEXT: v_mov_b32_e32 v2, s66793; SI-NEXT: v_mov_b32_e32 v3, s76794; SI-NEXT: s_mov_b32 s10, -16795; SI-NEXT: .LBB91_1: ; %atomicrmw.start6796; SI-NEXT: ; =>This Inner Loop Header: Depth=16797; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]6798; SI-NEXT: v_cndmask_b32_e32 v1, v8, v3, vcc6799; SI-NEXT: v_cndmask_b32_e32 v0, v9, v2, vcc6800; SI-NEXT: s_waitcnt expcnt(0)6801; SI-NEXT: v_mov_b32_e32 v7, v36802; SI-NEXT: v_mov_b32_e32 v6, v26803; SI-NEXT: v_mov_b32_e32 v5, v16804; SI-NEXT: v_mov_b32_e32 v4, v06805; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[8:11], 0 glc6806; SI-NEXT: s_waitcnt vmcnt(0)6807; SI-NEXT: buffer_wbinvl16808; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]6809; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]6810; SI-NEXT: v_mov_b32_e32 v2, v46811; SI-NEXT: v_mov_b32_e32 v3, v56812; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]6813; SI-NEXT: s_cbranch_execnz .LBB91_16814; SI-NEXT: ; %bb.2: ; %atomicrmw.end6815; SI-NEXT: s_or_b64 exec, exec, s[0:1]6816; SI-NEXT: s_mov_b32 s7, 0xf0006817; SI-NEXT: s_mov_b32 s6, -16818; SI-NEXT: s_mov_b32 s4, s26819; SI-NEXT: s_mov_b32 s5, s36820; SI-NEXT: buffer_store_dwordx2 v[4:5], off, s[4:7], 06821; SI-NEXT: s_endpgm6822;6823; VI-LABEL: atomic_max_i64_ret_addr64:6824; VI: ; %bb.0: ; %entry6825; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x246826; VI-NEXT: s_waitcnt lgkmcnt(0)6827; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 36828; VI-NEXT: s_add_u32 s6, s0, s66829; VI-NEXT: s_addc_u32 s7, s1, s76830; VI-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x06831; VI-NEXT: v_mov_b32_e32 v0, s66832; VI-NEXT: s_mov_b64 s[0:1], 06833; VI-NEXT: v_mov_b32_e32 v4, s56834; VI-NEXT: v_mov_b32_e32 v5, s46835; VI-NEXT: s_waitcnt lgkmcnt(0)6836; VI-NEXT: v_mov_b32_e32 v2, s86837; VI-NEXT: v_mov_b32_e32 v3, s96838; VI-NEXT: v_mov_b32_e32 v1, s76839; VI-NEXT: .LBB91_1: ; %atomicrmw.start6840; VI-NEXT: ; =>This Inner Loop Header: Depth=16841; VI-NEXT: v_mov_b32_e32 v9, v36842; VI-NEXT: v_mov_b32_e32 v8, v26843; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]6844; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc6845; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc6846; VI-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc6847; VI-NEXT: s_waitcnt vmcnt(0)6848; VI-NEXT: buffer_wbinvl1_vol6849; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]6850; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]6851; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]6852; VI-NEXT: s_cbranch_execnz .LBB91_16853; VI-NEXT: ; %bb.2: ; %atomicrmw.end6854; VI-NEXT: s_or_b64 exec, exec, s[0:1]6855; VI-NEXT: v_mov_b32_e32 v0, s26856; VI-NEXT: v_mov_b32_e32 v1, s36857; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]6858; VI-NEXT: s_endpgm6859;6860; GFX9-LABEL: atomic_max_i64_ret_addr64:6861; GFX9: ; %bb.0: ; %entry6862; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x246863; GFX9-NEXT: s_mov_b64 s[2:3], 06864; GFX9-NEXT: v_mov_b32_e32 v4, 06865; GFX9-NEXT: s_waitcnt lgkmcnt(0)6866; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 36867; GFX9-NEXT: s_add_u32 s0, s8, s06868; GFX9-NEXT: s_addc_u32 s1, s9, s16869; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x06870; GFX9-NEXT: v_mov_b32_e32 v2, s136871; GFX9-NEXT: v_mov_b32_e32 v3, s126872; GFX9-NEXT: s_waitcnt lgkmcnt(0)6873; GFX9-NEXT: v_mov_b32_e32 v0, s46874; GFX9-NEXT: v_mov_b32_e32 v1, s56875; GFX9-NEXT: .LBB91_1: ; %atomicrmw.start6876; GFX9-NEXT: ; =>This Inner Loop Header: Depth=16877; GFX9-NEXT: v_mov_b32_e32 v8, v16878; GFX9-NEXT: v_mov_b32_e32 v7, v06879; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[7:8]6880; GFX9-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc6881; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc6882; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[5:8], s[0:1] glc6883; GFX9-NEXT: s_waitcnt vmcnt(0)6884; GFX9-NEXT: buffer_wbinvl1_vol6885; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]6886; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]6887; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]6888; GFX9-NEXT: s_cbranch_execnz .LBB91_16889; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end6890; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]6891; GFX9-NEXT: v_mov_b32_e32 v2, 06892; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]6893; GFX9-NEXT: s_endpgm6894entry:6895 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index6896 %tmp0 = atomicrmw max ptr addrspace(1) %ptr, i64 %in seq_cst6897 store i64 %tmp0, ptr addrspace(1) %out26898 ret void6899}6900 6901define void @global_atomic_max_i64_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {6902; SI-LABEL: global_atomic_max_i64_noret_offset__amdgpu_no_remote_memory:6903; SI: ; %bb.0:6904; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6905; SI-NEXT: s_mov_b32 s6, 06906; SI-NEXT: s_mov_b32 s7, 0xf0006907; SI-NEXT: s_mov_b32 s4, s66908; SI-NEXT: s_mov_b32 s5, s66909; SI-NEXT: buffer_atomic_smax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:326910; SI-NEXT: s_waitcnt vmcnt(0)6911; SI-NEXT: buffer_wbinvl16912; SI-NEXT: s_waitcnt expcnt(0)6913; SI-NEXT: s_setpc_b64 s[30:31]6914;6915; VI-LABEL: global_atomic_max_i64_noret_offset__amdgpu_no_remote_memory:6916; VI: ; %bb.0:6917; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6918; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v06919; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc6920; VI-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3]6921; VI-NEXT: s_waitcnt vmcnt(0)6922; VI-NEXT: buffer_wbinvl1_vol6923; VI-NEXT: s_setpc_b64 s[30:31]6924;6925; GFX9-LABEL: global_atomic_max_i64_noret_offset__amdgpu_no_remote_memory:6926; GFX9: ; %bb.0:6927; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6928; GFX9-NEXT: global_atomic_smax_x2 v[0:1], v[2:3], off offset:326929; GFX9-NEXT: s_waitcnt vmcnt(0)6930; GFX9-NEXT: buffer_wbinvl1_vol6931; GFX9-NEXT: s_setpc_b64 s[30:31]6932 %gep = getelementptr i64, ptr addrspace(1) %out, i64 46933 %tmp0 = atomicrmw max ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !06934 ret void6935}6936 6937define i64 @global_atomic_max_i64_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {6938; SI-LABEL: global_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:6939; SI: ; %bb.0:6940; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6941; SI-NEXT: s_mov_b32 s6, 06942; SI-NEXT: s_mov_b32 s7, 0xf0006943; SI-NEXT: s_mov_b32 s4, s66944; SI-NEXT: s_mov_b32 s5, s66945; SI-NEXT: buffer_atomic_smax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc6946; SI-NEXT: s_waitcnt vmcnt(0)6947; SI-NEXT: buffer_wbinvl16948; SI-NEXT: v_mov_b32_e32 v0, v26949; SI-NEXT: v_mov_b32_e32 v1, v36950; SI-NEXT: s_waitcnt expcnt(0)6951; SI-NEXT: s_setpc_b64 s[30:31]6952;6953; VI-LABEL: global_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:6954; VI: ; %bb.0:6955; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6956; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v06957; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc6958; VI-NEXT: flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc6959; VI-NEXT: s_waitcnt vmcnt(0)6960; VI-NEXT: buffer_wbinvl1_vol6961; VI-NEXT: s_setpc_b64 s[30:31]6962;6963; GFX9-LABEL: global_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:6964; GFX9: ; %bb.0:6965; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6966; GFX9-NEXT: global_atomic_smax_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc6967; GFX9-NEXT: s_waitcnt vmcnt(0)6968; GFX9-NEXT: buffer_wbinvl1_vol6969; GFX9-NEXT: s_setpc_b64 s[30:31]6970 %gep = getelementptr i64, ptr addrspace(1) %out, i64 46971 %result = atomicrmw max ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !06972 ret i64 %result6973}6974 6975; ---------------------------------------------------------------------6976; atomicrmw umax6977; ---------------------------------------------------------------------6978 6979define void @global_atomic_umax_i64_noret(ptr addrspace(1) %ptr, i64 %in) {6980; SI-LABEL: global_atomic_umax_i64_noret:6981; SI: ; %bb.0:6982; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6983; SI-NEXT: s_mov_b32 s6, 06984; SI-NEXT: s_mov_b32 s7, 0xf0006985; SI-NEXT: s_mov_b32 s4, s66986; SI-NEXT: s_mov_b32 s5, s66987; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr646988; SI-NEXT: s_mov_b64 s[8:9], 06989; SI-NEXT: .LBB94_1: ; %atomicrmw.start6990; SI-NEXT: ; =>This Inner Loop Header: Depth=16991; SI-NEXT: s_waitcnt vmcnt(0)6992; SI-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]6993; SI-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc6994; SI-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc6995; SI-NEXT: s_waitcnt expcnt(0)6996; SI-NEXT: v_mov_b32_e32 v11, v76997; SI-NEXT: v_mov_b32_e32 v10, v66998; SI-NEXT: v_mov_b32_e32 v9, v56999; SI-NEXT: v_mov_b32_e32 v8, v47000; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc7001; SI-NEXT: s_waitcnt vmcnt(0)7002; SI-NEXT: buffer_wbinvl17003; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]7004; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]7005; SI-NEXT: v_mov_b32_e32 v6, v87006; SI-NEXT: v_mov_b32_e32 v7, v97007; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]7008; SI-NEXT: s_cbranch_execnz .LBB94_17009; SI-NEXT: ; %bb.2: ; %atomicrmw.end7010; SI-NEXT: s_or_b64 exec, exec, s[8:9]7011; SI-NEXT: s_waitcnt expcnt(0)7012; SI-NEXT: s_setpc_b64 s[30:31]7013;7014; VI-LABEL: global_atomic_umax_i64_noret:7015; VI: ; %bb.0:7016; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7017; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]7018; VI-NEXT: s_mov_b64 s[4:5], 07019; VI-NEXT: .LBB94_1: ; %atomicrmw.start7020; VI-NEXT: ; =>This Inner Loop Header: Depth=17021; VI-NEXT: s_waitcnt vmcnt(0)7022; VI-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]7023; VI-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc7024; VI-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc7025; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc7026; VI-NEXT: s_waitcnt vmcnt(0)7027; VI-NEXT: buffer_wbinvl1_vol7028; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]7029; VI-NEXT: v_mov_b32_e32 v7, v57030; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7031; VI-NEXT: v_mov_b32_e32 v6, v47032; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]7033; VI-NEXT: s_cbranch_execnz .LBB94_17034; VI-NEXT: ; %bb.2: ; %atomicrmw.end7035; VI-NEXT: s_or_b64 exec, exec, s[4:5]7036; VI-NEXT: s_setpc_b64 s[30:31]7037;7038; GFX9-LABEL: global_atomic_umax_i64_noret:7039; GFX9: ; %bb.0:7040; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7041; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off7042; GFX9-NEXT: s_mov_b64 s[4:5], 07043; GFX9-NEXT: .LBB94_1: ; %atomicrmw.start7044; GFX9-NEXT: ; =>This Inner Loop Header: Depth=17045; GFX9-NEXT: s_waitcnt vmcnt(0)7046; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]7047; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc7048; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc7049; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc7050; GFX9-NEXT: s_waitcnt vmcnt(0)7051; GFX9-NEXT: buffer_wbinvl1_vol7052; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]7053; GFX9-NEXT: v_mov_b32_e32 v7, v57054; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7055; GFX9-NEXT: v_mov_b32_e32 v6, v47056; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]7057; GFX9-NEXT: s_cbranch_execnz .LBB94_17058; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end7059; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]7060; GFX9-NEXT: s_setpc_b64 s[30:31]7061 %tmp0 = atomicrmw umax ptr addrspace(1) %ptr, i64 %in seq_cst7062 ret void7063}7064 7065define void @global_atomic_umax_i64_noret_offset(ptr addrspace(1) %out, i64 %in) {7066; SI-LABEL: global_atomic_umax_i64_noret_offset:7067; SI: ; %bb.0:7068; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7069; SI-NEXT: s_mov_b32 s6, 07070; SI-NEXT: s_mov_b32 s7, 0xf0007071; SI-NEXT: s_mov_b32 s4, s67072; SI-NEXT: s_mov_b32 s5, s67073; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:327074; SI-NEXT: s_mov_b64 s[8:9], 07075; SI-NEXT: .LBB95_1: ; %atomicrmw.start7076; SI-NEXT: ; =>This Inner Loop Header: Depth=17077; SI-NEXT: s_waitcnt vmcnt(0)7078; SI-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]7079; SI-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc7080; SI-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc7081; SI-NEXT: s_waitcnt expcnt(0)7082; SI-NEXT: v_mov_b32_e32 v11, v77083; SI-NEXT: v_mov_b32_e32 v10, v67084; SI-NEXT: v_mov_b32_e32 v9, v57085; SI-NEXT: v_mov_b32_e32 v8, v47086; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32 glc7087; SI-NEXT: s_waitcnt vmcnt(0)7088; SI-NEXT: buffer_wbinvl17089; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]7090; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]7091; SI-NEXT: v_mov_b32_e32 v6, v87092; SI-NEXT: v_mov_b32_e32 v7, v97093; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]7094; SI-NEXT: s_cbranch_execnz .LBB95_17095; SI-NEXT: ; %bb.2: ; %atomicrmw.end7096; SI-NEXT: s_or_b64 exec, exec, s[8:9]7097; SI-NEXT: s_waitcnt expcnt(0)7098; SI-NEXT: s_setpc_b64 s[30:31]7099;7100; VI-LABEL: global_atomic_umax_i64_noret_offset:7101; VI: ; %bb.0:7102; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7103; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v07104; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc7105; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]7106; VI-NEXT: s_mov_b64 s[4:5], 07107; VI-NEXT: .LBB95_1: ; %atomicrmw.start7108; VI-NEXT: ; =>This Inner Loop Header: Depth=17109; VI-NEXT: s_waitcnt vmcnt(0)7110; VI-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]7111; VI-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc7112; VI-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc7113; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc7114; VI-NEXT: s_waitcnt vmcnt(0)7115; VI-NEXT: buffer_wbinvl1_vol7116; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]7117; VI-NEXT: v_mov_b32_e32 v7, v57118; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7119; VI-NEXT: v_mov_b32_e32 v6, v47120; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]7121; VI-NEXT: s_cbranch_execnz .LBB95_17122; VI-NEXT: ; %bb.2: ; %atomicrmw.end7123; VI-NEXT: s_or_b64 exec, exec, s[4:5]7124; VI-NEXT: s_setpc_b64 s[30:31]7125;7126; GFX9-LABEL: global_atomic_umax_i64_noret_offset:7127; GFX9: ; %bb.0:7128; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7129; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:327130; GFX9-NEXT: s_mov_b64 s[4:5], 07131; GFX9-NEXT: .LBB95_1: ; %atomicrmw.start7132; GFX9-NEXT: ; =>This Inner Loop Header: Depth=17133; GFX9-NEXT: s_waitcnt vmcnt(0)7134; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]7135; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc7136; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc7137; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc7138; GFX9-NEXT: s_waitcnt vmcnt(0)7139; GFX9-NEXT: buffer_wbinvl1_vol7140; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]7141; GFX9-NEXT: v_mov_b32_e32 v7, v57142; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7143; GFX9-NEXT: v_mov_b32_e32 v6, v47144; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]7145; GFX9-NEXT: s_cbranch_execnz .LBB95_17146; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end7147; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]7148; GFX9-NEXT: s_setpc_b64 s[30:31]7149 %gep = getelementptr i64, ptr addrspace(1) %out, i64 47150 %tmp0 = atomicrmw umax ptr addrspace(1) %gep, i64 %in seq_cst7151 ret void7152}7153 7154define i64 @global_atomic_umax_i64_ret(ptr addrspace(1) %ptr, i64 %in) {7155; SI-LABEL: global_atomic_umax_i64_ret:7156; SI: ; %bb.0:7157; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7158; SI-NEXT: v_mov_b32_e32 v5, v37159; SI-NEXT: v_mov_b32_e32 v4, v27160; SI-NEXT: v_mov_b32_e32 v7, v17161; SI-NEXT: v_mov_b32_e32 v6, v07162; SI-NEXT: s_mov_b32 s6, 07163; SI-NEXT: s_mov_b32 s7, 0xf0007164; SI-NEXT: s_mov_b32 s4, s67165; SI-NEXT: s_mov_b32 s5, s67166; SI-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[4:7], 0 addr647167; SI-NEXT: s_mov_b64 s[8:9], 07168; SI-NEXT: .LBB96_1: ; %atomicrmw.start7169; SI-NEXT: ; =>This Inner Loop Header: Depth=17170; SI-NEXT: s_waitcnt vmcnt(0)7171; SI-NEXT: v_cmp_gt_u64_e32 vcc, v[10:11], v[4:5]7172; SI-NEXT: v_cndmask_b32_e32 v9, v5, v11, vcc7173; SI-NEXT: v_cndmask_b32_e32 v8, v4, v10, vcc7174; SI-NEXT: s_waitcnt expcnt(0)7175; SI-NEXT: v_mov_b32_e32 v0, v87176; SI-NEXT: v_mov_b32_e32 v1, v97177; SI-NEXT: v_mov_b32_e32 v2, v107178; SI-NEXT: v_mov_b32_e32 v3, v117179; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc7180; SI-NEXT: s_waitcnt vmcnt(0)7181; SI-NEXT: buffer_wbinvl17182; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]7183; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]7184; SI-NEXT: v_mov_b32_e32 v11, v17185; SI-NEXT: v_mov_b32_e32 v10, v07186; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]7187; SI-NEXT: s_cbranch_execnz .LBB96_17188; SI-NEXT: ; %bb.2: ; %atomicrmw.end7189; SI-NEXT: s_or_b64 exec, exec, s[8:9]7190; SI-NEXT: s_waitcnt expcnt(0)7191; SI-NEXT: s_setpc_b64 s[30:31]7192;7193; VI-LABEL: global_atomic_umax_i64_ret:7194; VI: ; %bb.0:7195; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7196; VI-NEXT: flat_load_dwordx2 v[4:5], v[0:1]7197; VI-NEXT: s_mov_b64 s[4:5], 07198; VI-NEXT: .LBB96_1: ; %atomicrmw.start7199; VI-NEXT: ; =>This Inner Loop Header: Depth=17200; VI-NEXT: s_waitcnt vmcnt(0)7201; VI-NEXT: v_mov_b32_e32 v7, v57202; VI-NEXT: v_mov_b32_e32 v6, v47203; VI-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]7204; VI-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc7205; VI-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc7206; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc7207; VI-NEXT: s_waitcnt vmcnt(0)7208; VI-NEXT: buffer_wbinvl1_vol7209; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]7210; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7211; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]7212; VI-NEXT: s_cbranch_execnz .LBB96_17213; VI-NEXT: ; %bb.2: ; %atomicrmw.end7214; VI-NEXT: s_or_b64 exec, exec, s[4:5]7215; VI-NEXT: v_mov_b32_e32 v0, v47216; VI-NEXT: v_mov_b32_e32 v1, v57217; VI-NEXT: s_setpc_b64 s[30:31]7218;7219; GFX9-LABEL: global_atomic_umax_i64_ret:7220; GFX9: ; %bb.0:7221; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7222; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off7223; GFX9-NEXT: s_mov_b64 s[4:5], 07224; GFX9-NEXT: .LBB96_1: ; %atomicrmw.start7225; GFX9-NEXT: ; =>This Inner Loop Header: Depth=17226; GFX9-NEXT: s_waitcnt vmcnt(0)7227; GFX9-NEXT: v_mov_b32_e32 v7, v57228; GFX9-NEXT: v_mov_b32_e32 v6, v47229; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]7230; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc7231; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc7232; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc7233; GFX9-NEXT: s_waitcnt vmcnt(0)7234; GFX9-NEXT: buffer_wbinvl1_vol7235; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]7236; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7237; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]7238; GFX9-NEXT: s_cbranch_execnz .LBB96_17239; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end7240; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]7241; GFX9-NEXT: v_mov_b32_e32 v0, v47242; GFX9-NEXT: v_mov_b32_e32 v1, v57243; GFX9-NEXT: s_setpc_b64 s[30:31]7244 %result = atomicrmw umax ptr addrspace(1) %ptr, i64 %in seq_cst7245 ret i64 %result7246}7247 7248define i64 @global_atomic_umax_i64_ret_offset(ptr addrspace(1) %out, i64 %in) {7249; SI-LABEL: global_atomic_umax_i64_ret_offset:7250; SI: ; %bb.0:7251; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7252; SI-NEXT: v_mov_b32_e32 v5, v37253; SI-NEXT: v_mov_b32_e32 v4, v27254; SI-NEXT: v_mov_b32_e32 v7, v17255; SI-NEXT: v_mov_b32_e32 v6, v07256; SI-NEXT: s_mov_b32 s6, 07257; SI-NEXT: s_mov_b32 s7, 0xf0007258; SI-NEXT: s_mov_b32 s4, s67259; SI-NEXT: s_mov_b32 s5, s67260; SI-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[4:7], 0 addr64 offset:327261; SI-NEXT: s_mov_b64 s[8:9], 07262; SI-NEXT: .LBB97_1: ; %atomicrmw.start7263; SI-NEXT: ; =>This Inner Loop Header: Depth=17264; SI-NEXT: s_waitcnt vmcnt(0)7265; SI-NEXT: v_cmp_gt_u64_e32 vcc, v[10:11], v[4:5]7266; SI-NEXT: v_cndmask_b32_e32 v9, v5, v11, vcc7267; SI-NEXT: v_cndmask_b32_e32 v8, v4, v10, vcc7268; SI-NEXT: s_waitcnt expcnt(0)7269; SI-NEXT: v_mov_b32_e32 v0, v87270; SI-NEXT: v_mov_b32_e32 v1, v97271; SI-NEXT: v_mov_b32_e32 v2, v107272; SI-NEXT: v_mov_b32_e32 v3, v117273; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 offset:32 glc7274; SI-NEXT: s_waitcnt vmcnt(0)7275; SI-NEXT: buffer_wbinvl17276; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]7277; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]7278; SI-NEXT: v_mov_b32_e32 v11, v17279; SI-NEXT: v_mov_b32_e32 v10, v07280; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]7281; SI-NEXT: s_cbranch_execnz .LBB97_17282; SI-NEXT: ; %bb.2: ; %atomicrmw.end7283; SI-NEXT: s_or_b64 exec, exec, s[8:9]7284; SI-NEXT: s_waitcnt expcnt(0)7285; SI-NEXT: s_setpc_b64 s[30:31]7286;7287; VI-LABEL: global_atomic_umax_i64_ret_offset:7288; VI: ; %bb.0:7289; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7290; VI-NEXT: v_add_u32_e32 v4, vcc, 32, v07291; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc7292; VI-NEXT: flat_load_dwordx2 v[0:1], v[4:5]7293; VI-NEXT: s_mov_b64 s[4:5], 07294; VI-NEXT: .LBB97_1: ; %atomicrmw.start7295; VI-NEXT: ; =>This Inner Loop Header: Depth=17296; VI-NEXT: s_waitcnt vmcnt(0)7297; VI-NEXT: v_mov_b32_e32 v9, v17298; VI-NEXT: v_mov_b32_e32 v8, v07299; VI-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]7300; VI-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc7301; VI-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc7302; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc7303; VI-NEXT: s_waitcnt vmcnt(0)7304; VI-NEXT: buffer_wbinvl1_vol7305; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]7306; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7307; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]7308; VI-NEXT: s_cbranch_execnz .LBB97_17309; VI-NEXT: ; %bb.2: ; %atomicrmw.end7310; VI-NEXT: s_or_b64 exec, exec, s[4:5]7311; VI-NEXT: s_setpc_b64 s[30:31]7312;7313; GFX9-LABEL: global_atomic_umax_i64_ret_offset:7314; GFX9: ; %bb.0:7315; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7316; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:327317; GFX9-NEXT: s_mov_b64 s[4:5], 07318; GFX9-NEXT: .LBB97_1: ; %atomicrmw.start7319; GFX9-NEXT: ; =>This Inner Loop Header: Depth=17320; GFX9-NEXT: s_waitcnt vmcnt(0)7321; GFX9-NEXT: v_mov_b32_e32 v7, v57322; GFX9-NEXT: v_mov_b32_e32 v6, v47323; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]7324; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc7325; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc7326; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc7327; GFX9-NEXT: s_waitcnt vmcnt(0)7328; GFX9-NEXT: buffer_wbinvl1_vol7329; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]7330; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7331; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]7332; GFX9-NEXT: s_cbranch_execnz .LBB97_17333; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end7334; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]7335; GFX9-NEXT: v_mov_b32_e32 v0, v47336; GFX9-NEXT: v_mov_b32_e32 v1, v57337; GFX9-NEXT: s_setpc_b64 s[30:31]7338 %gep = getelementptr i64, ptr addrspace(1) %out, i64 47339 %result = atomicrmw umax ptr addrspace(1) %gep, i64 %in seq_cst7340 ret i64 %result7341}7342 7343define amdgpu_gfx void @global_atomic_umax_i64_noret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {7344; SI-LABEL: global_atomic_umax_i64_noret_scalar:7345; SI: ; %bb.0:7346; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7347; SI-NEXT: s_xor_saveexec_b64 s[34:35], -17348; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill7349; SI-NEXT: s_mov_b64 exec, s[34:35]7350; SI-NEXT: s_waitcnt expcnt(0)7351; SI-NEXT: v_writelane_b32 v10, s6, 07352; SI-NEXT: v_writelane_b32 v10, s7, 17353; SI-NEXT: s_mov_b32 s35, s77354; SI-NEXT: s_mov_b32 s34, s67355; SI-NEXT: s_mov_b32 s7, 0xf0007356; SI-NEXT: s_mov_b32 s6, -17357; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 07358; SI-NEXT: s_mov_b64 s[36:37], 07359; SI-NEXT: v_mov_b32_e32 v4, s357360; SI-NEXT: v_mov_b32_e32 v5, s347361; SI-NEXT: .LBB98_1: ; %atomicrmw.start7362; SI-NEXT: ; =>This Inner Loop Header: Depth=17363; SI-NEXT: s_waitcnt vmcnt(0)7364; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[34:35], v[2:3]7365; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc7366; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc7367; SI-NEXT: s_waitcnt expcnt(0)7368; SI-NEXT: v_mov_b32_e32 v9, v37369; SI-NEXT: v_mov_b32_e32 v8, v27370; SI-NEXT: v_mov_b32_e32 v7, v17371; SI-NEXT: v_mov_b32_e32 v6, v07372; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 glc7373; SI-NEXT: s_waitcnt vmcnt(0)7374; SI-NEXT: buffer_wbinvl17375; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]7376; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]7377; SI-NEXT: v_mov_b32_e32 v2, v67378; SI-NEXT: v_mov_b32_e32 v3, v77379; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]7380; SI-NEXT: s_cbranch_execnz .LBB98_17381; SI-NEXT: ; %bb.2: ; %atomicrmw.end7382; SI-NEXT: s_or_b64 exec, exec, s[36:37]7383; SI-NEXT: v_readlane_b32 s7, v10, 17384; SI-NEXT: v_readlane_b32 s6, v10, 07385; SI-NEXT: s_xor_saveexec_b64 s[34:35], -17386; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload7387; SI-NEXT: s_mov_b64 exec, s[34:35]7388; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)7389; SI-NEXT: s_setpc_b64 s[30:31]7390;7391; VI-LABEL: global_atomic_umax_i64_noret_scalar:7392; VI: ; %bb.0:7393; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7394; VI-NEXT: v_mov_b32_e32 v0, s47395; VI-NEXT: v_mov_b32_e32 v1, s57396; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]7397; VI-NEXT: v_mov_b32_e32 v4, s47398; VI-NEXT: s_mov_b64 s[34:35], 07399; VI-NEXT: v_mov_b32_e32 v6, s77400; VI-NEXT: v_mov_b32_e32 v7, s67401; VI-NEXT: v_mov_b32_e32 v5, s57402; VI-NEXT: .LBB98_1: ; %atomicrmw.start7403; VI-NEXT: ; =>This Inner Loop Header: Depth=17404; VI-NEXT: s_waitcnt vmcnt(0)7405; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]7406; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc7407; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc7408; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc7409; VI-NEXT: s_waitcnt vmcnt(0)7410; VI-NEXT: buffer_wbinvl1_vol7411; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]7412; VI-NEXT: v_mov_b32_e32 v3, v17413; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]7414; VI-NEXT: v_mov_b32_e32 v2, v07415; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]7416; VI-NEXT: s_cbranch_execnz .LBB98_17417; VI-NEXT: ; %bb.2: ; %atomicrmw.end7418; VI-NEXT: s_or_b64 exec, exec, s[34:35]7419; VI-NEXT: s_setpc_b64 s[30:31]7420;7421; GFX9-LABEL: global_atomic_umax_i64_noret_scalar:7422; GFX9: ; %bb.0:7423; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7424; GFX9-NEXT: v_mov_b32_e32 v4, 07425; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5]7426; GFX9-NEXT: s_mov_b64 s[34:35], 07427; GFX9-NEXT: v_mov_b32_e32 v5, s77428; GFX9-NEXT: v_mov_b32_e32 v6, s67429; GFX9-NEXT: .LBB98_1: ; %atomicrmw.start7430; GFX9-NEXT: ; =>This Inner Loop Header: Depth=17431; GFX9-NEXT: s_waitcnt vmcnt(0)7432; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]7433; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc7434; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc7435; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc7436; GFX9-NEXT: s_waitcnt vmcnt(0)7437; GFX9-NEXT: buffer_wbinvl1_vol7438; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]7439; GFX9-NEXT: v_mov_b32_e32 v3, v17440; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]7441; GFX9-NEXT: v_mov_b32_e32 v2, v07442; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]7443; GFX9-NEXT: s_cbranch_execnz .LBB98_17444; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end7445; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]7446; GFX9-NEXT: s_setpc_b64 s[30:31]7447 %tmp0 = atomicrmw umax ptr addrspace(1) %ptr, i64 %in seq_cst7448 ret void7449}7450 7451define amdgpu_gfx void @global_atomic_umax_i64_noret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {7452; SI-LABEL: global_atomic_umax_i64_noret_offset_scalar:7453; SI: ; %bb.0:7454; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7455; SI-NEXT: s_xor_saveexec_b64 s[34:35], -17456; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill7457; SI-NEXT: s_mov_b64 exec, s[34:35]7458; SI-NEXT: s_waitcnt expcnt(0)7459; SI-NEXT: v_writelane_b32 v10, s6, 07460; SI-NEXT: v_writelane_b32 v10, s7, 17461; SI-NEXT: s_mov_b32 s35, s77462; SI-NEXT: s_mov_b32 s34, s67463; SI-NEXT: s_mov_b32 s7, 0xf0007464; SI-NEXT: s_mov_b32 s6, -17465; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:327466; SI-NEXT: s_mov_b64 s[36:37], 07467; SI-NEXT: v_mov_b32_e32 v4, s357468; SI-NEXT: v_mov_b32_e32 v5, s347469; SI-NEXT: .LBB99_1: ; %atomicrmw.start7470; SI-NEXT: ; =>This Inner Loop Header: Depth=17471; SI-NEXT: s_waitcnt vmcnt(0)7472; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[34:35], v[2:3]7473; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc7474; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc7475; SI-NEXT: s_waitcnt expcnt(0)7476; SI-NEXT: v_mov_b32_e32 v9, v37477; SI-NEXT: v_mov_b32_e32 v8, v27478; SI-NEXT: v_mov_b32_e32 v7, v17479; SI-NEXT: v_mov_b32_e32 v6, v07480; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 offset:32 glc7481; SI-NEXT: s_waitcnt vmcnt(0)7482; SI-NEXT: buffer_wbinvl17483; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]7484; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]7485; SI-NEXT: v_mov_b32_e32 v2, v67486; SI-NEXT: v_mov_b32_e32 v3, v77487; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]7488; SI-NEXT: s_cbranch_execnz .LBB99_17489; SI-NEXT: ; %bb.2: ; %atomicrmw.end7490; SI-NEXT: s_or_b64 exec, exec, s[36:37]7491; SI-NEXT: v_readlane_b32 s7, v10, 17492; SI-NEXT: v_readlane_b32 s6, v10, 07493; SI-NEXT: s_xor_saveexec_b64 s[34:35], -17494; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload7495; SI-NEXT: s_mov_b64 exec, s[34:35]7496; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)7497; SI-NEXT: s_setpc_b64 s[30:31]7498;7499; VI-LABEL: global_atomic_umax_i64_noret_offset_scalar:7500; VI: ; %bb.0:7501; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7502; VI-NEXT: s_add_u32 s34, s4, 327503; VI-NEXT: s_addc_u32 s35, s5, 07504; VI-NEXT: v_mov_b32_e32 v4, s347505; VI-NEXT: v_mov_b32_e32 v5, s357506; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]7507; VI-NEXT: s_mov_b64 s[34:35], 07508; VI-NEXT: v_mov_b32_e32 v6, s77509; VI-NEXT: v_mov_b32_e32 v7, s67510; VI-NEXT: .LBB99_1: ; %atomicrmw.start7511; VI-NEXT: ; =>This Inner Loop Header: Depth=17512; VI-NEXT: s_waitcnt vmcnt(0)7513; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]7514; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc7515; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc7516; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc7517; VI-NEXT: s_waitcnt vmcnt(0)7518; VI-NEXT: buffer_wbinvl1_vol7519; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]7520; VI-NEXT: v_mov_b32_e32 v3, v17521; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]7522; VI-NEXT: v_mov_b32_e32 v2, v07523; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]7524; VI-NEXT: s_cbranch_execnz .LBB99_17525; VI-NEXT: ; %bb.2: ; %atomicrmw.end7526; VI-NEXT: s_or_b64 exec, exec, s[34:35]7527; VI-NEXT: s_setpc_b64 s[30:31]7528;7529; GFX9-LABEL: global_atomic_umax_i64_noret_offset_scalar:7530; GFX9: ; %bb.0:7531; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7532; GFX9-NEXT: v_mov_b32_e32 v4, 07533; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5] offset:327534; GFX9-NEXT: s_mov_b64 s[34:35], 07535; GFX9-NEXT: v_mov_b32_e32 v5, s77536; GFX9-NEXT: v_mov_b32_e32 v6, s67537; GFX9-NEXT: .LBB99_1: ; %atomicrmw.start7538; GFX9-NEXT: ; =>This Inner Loop Header: Depth=17539; GFX9-NEXT: s_waitcnt vmcnt(0)7540; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]7541; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc7542; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc7543; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc7544; GFX9-NEXT: s_waitcnt vmcnt(0)7545; GFX9-NEXT: buffer_wbinvl1_vol7546; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]7547; GFX9-NEXT: v_mov_b32_e32 v3, v17548; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]7549; GFX9-NEXT: v_mov_b32_e32 v2, v07550; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]7551; GFX9-NEXT: s_cbranch_execnz .LBB99_17552; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end7553; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]7554; GFX9-NEXT: s_setpc_b64 s[30:31]7555 %gep = getelementptr i64, ptr addrspace(1) %out, i64 47556 %tmp0 = atomicrmw umax ptr addrspace(1) %gep, i64 %in seq_cst7557 ret void7558}7559 7560define amdgpu_gfx i64 @global_atomic_umax_i64_ret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {7561; SI-LABEL: global_atomic_umax_i64_ret_scalar:7562; SI: ; %bb.0:7563; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7564; SI-NEXT: s_xor_saveexec_b64 s[34:35], -17565; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill7566; SI-NEXT: s_mov_b64 exec, s[34:35]7567; SI-NEXT: s_waitcnt expcnt(0)7568; SI-NEXT: v_writelane_b32 v8, s6, 07569; SI-NEXT: v_writelane_b32 v8, s7, 17570; SI-NEXT: s_mov_b32 s35, s77571; SI-NEXT: s_mov_b32 s34, s67572; SI-NEXT: s_mov_b32 s7, 0xf0007573; SI-NEXT: s_mov_b32 s6, -17574; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 07575; SI-NEXT: s_mov_b64 s[36:37], 07576; SI-NEXT: v_mov_b32_e32 v6, s357577; SI-NEXT: v_mov_b32_e32 v7, s347578; SI-NEXT: .LBB100_1: ; %atomicrmw.start7579; SI-NEXT: ; =>This Inner Loop Header: Depth=17580; SI-NEXT: s_waitcnt vmcnt(0)7581; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[34:35], v[4:5]7582; SI-NEXT: s_waitcnt expcnt(0)7583; SI-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc7584; SI-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc7585; SI-NEXT: v_mov_b32_e32 v0, v27586; SI-NEXT: v_mov_b32_e32 v1, v37587; SI-NEXT: v_mov_b32_e32 v2, v47588; SI-NEXT: v_mov_b32_e32 v3, v57589; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc7590; SI-NEXT: s_waitcnt vmcnt(0)7591; SI-NEXT: buffer_wbinvl17592; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]7593; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]7594; SI-NEXT: v_mov_b32_e32 v5, v17595; SI-NEXT: v_mov_b32_e32 v4, v07596; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]7597; SI-NEXT: s_cbranch_execnz .LBB100_17598; SI-NEXT: ; %bb.2: ; %atomicrmw.end7599; SI-NEXT: s_or_b64 exec, exec, s[36:37]7600; SI-NEXT: v_readlane_b32 s7, v8, 17601; SI-NEXT: v_readlane_b32 s6, v8, 07602; SI-NEXT: s_xor_saveexec_b64 s[34:35], -17603; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload7604; SI-NEXT: s_mov_b64 exec, s[34:35]7605; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)7606; SI-NEXT: s_setpc_b64 s[30:31]7607;7608; VI-LABEL: global_atomic_umax_i64_ret_scalar:7609; VI: ; %bb.0:7610; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7611; VI-NEXT: v_mov_b32_e32 v0, s47612; VI-NEXT: v_mov_b32_e32 v1, s57613; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]7614; VI-NEXT: v_mov_b32_e32 v2, s47615; VI-NEXT: s_mov_b64 s[34:35], 07616; VI-NEXT: v_mov_b32_e32 v4, s77617; VI-NEXT: v_mov_b32_e32 v5, s67618; VI-NEXT: v_mov_b32_e32 v3, s57619; VI-NEXT: .LBB100_1: ; %atomicrmw.start7620; VI-NEXT: ; =>This Inner Loop Header: Depth=17621; VI-NEXT: s_waitcnt vmcnt(0)7622; VI-NEXT: v_mov_b32_e32 v9, v17623; VI-NEXT: v_mov_b32_e32 v8, v07624; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]7625; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc7626; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc7627; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc7628; VI-NEXT: s_waitcnt vmcnt(0)7629; VI-NEXT: buffer_wbinvl1_vol7630; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]7631; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]7632; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]7633; VI-NEXT: s_cbranch_execnz .LBB100_17634; VI-NEXT: ; %bb.2: ; %atomicrmw.end7635; VI-NEXT: s_or_b64 exec, exec, s[34:35]7636; VI-NEXT: s_setpc_b64 s[30:31]7637;7638; GFX9-LABEL: global_atomic_umax_i64_ret_scalar:7639; GFX9: ; %bb.0:7640; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7641; GFX9-NEXT: v_mov_b32_e32 v2, 07642; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]7643; GFX9-NEXT: s_mov_b64 s[34:35], 07644; GFX9-NEXT: v_mov_b32_e32 v3, s77645; GFX9-NEXT: v_mov_b32_e32 v4, s67646; GFX9-NEXT: .LBB100_1: ; %atomicrmw.start7647; GFX9-NEXT: ; =>This Inner Loop Header: Depth=17648; GFX9-NEXT: s_waitcnt vmcnt(0)7649; GFX9-NEXT: v_mov_b32_e32 v8, v17650; GFX9-NEXT: v_mov_b32_e32 v7, v07651; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[7:8]7652; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc7653; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc7654; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] glc7655; GFX9-NEXT: s_waitcnt vmcnt(0)7656; GFX9-NEXT: buffer_wbinvl1_vol7657; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]7658; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]7659; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]7660; GFX9-NEXT: s_cbranch_execnz .LBB100_17661; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end7662; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]7663; GFX9-NEXT: s_setpc_b64 s[30:31]7664 %result = atomicrmw umax ptr addrspace(1) %ptr, i64 %in seq_cst7665 ret i64 %result7666}7667 7668define amdgpu_gfx i64 @global_atomic_umax_i64_ret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {7669; SI-LABEL: global_atomic_umax_i64_ret_offset_scalar:7670; SI: ; %bb.0:7671; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7672; SI-NEXT: s_xor_saveexec_b64 s[34:35], -17673; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill7674; SI-NEXT: s_mov_b64 exec, s[34:35]7675; SI-NEXT: s_waitcnt expcnt(0)7676; SI-NEXT: v_writelane_b32 v8, s6, 07677; SI-NEXT: v_writelane_b32 v8, s7, 17678; SI-NEXT: s_mov_b32 s35, s77679; SI-NEXT: s_mov_b32 s34, s67680; SI-NEXT: s_mov_b32 s7, 0xf0007681; SI-NEXT: s_mov_b32 s6, -17682; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0 offset:327683; SI-NEXT: s_mov_b64 s[36:37], 07684; SI-NEXT: v_mov_b32_e32 v6, s357685; SI-NEXT: v_mov_b32_e32 v7, s347686; SI-NEXT: .LBB101_1: ; %atomicrmw.start7687; SI-NEXT: ; =>This Inner Loop Header: Depth=17688; SI-NEXT: s_waitcnt vmcnt(0)7689; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[34:35], v[4:5]7690; SI-NEXT: s_waitcnt expcnt(0)7691; SI-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc7692; SI-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc7693; SI-NEXT: v_mov_b32_e32 v0, v27694; SI-NEXT: v_mov_b32_e32 v1, v37695; SI-NEXT: v_mov_b32_e32 v2, v47696; SI-NEXT: v_mov_b32_e32 v3, v57697; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 offset:32 glc7698; SI-NEXT: s_waitcnt vmcnt(0)7699; SI-NEXT: buffer_wbinvl17700; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]7701; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]7702; SI-NEXT: v_mov_b32_e32 v5, v17703; SI-NEXT: v_mov_b32_e32 v4, v07704; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]7705; SI-NEXT: s_cbranch_execnz .LBB101_17706; SI-NEXT: ; %bb.2: ; %atomicrmw.end7707; SI-NEXT: s_or_b64 exec, exec, s[36:37]7708; SI-NEXT: v_readlane_b32 s7, v8, 17709; SI-NEXT: v_readlane_b32 s6, v8, 07710; SI-NEXT: s_xor_saveexec_b64 s[34:35], -17711; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload7712; SI-NEXT: s_mov_b64 exec, s[34:35]7713; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)7714; SI-NEXT: s_setpc_b64 s[30:31]7715;7716; VI-LABEL: global_atomic_umax_i64_ret_offset_scalar:7717; VI: ; %bb.0:7718; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7719; VI-NEXT: s_add_u32 s34, s4, 327720; VI-NEXT: s_addc_u32 s35, s5, 07721; VI-NEXT: v_mov_b32_e32 v2, s347722; VI-NEXT: v_mov_b32_e32 v3, s357723; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]7724; VI-NEXT: s_mov_b64 s[34:35], 07725; VI-NEXT: v_mov_b32_e32 v4, s77726; VI-NEXT: v_mov_b32_e32 v5, s67727; VI-NEXT: .LBB101_1: ; %atomicrmw.start7728; VI-NEXT: ; =>This Inner Loop Header: Depth=17729; VI-NEXT: s_waitcnt vmcnt(0)7730; VI-NEXT: v_mov_b32_e32 v9, v17731; VI-NEXT: v_mov_b32_e32 v8, v07732; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]7733; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc7734; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc7735; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc7736; VI-NEXT: s_waitcnt vmcnt(0)7737; VI-NEXT: buffer_wbinvl1_vol7738; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]7739; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]7740; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]7741; VI-NEXT: s_cbranch_execnz .LBB101_17742; VI-NEXT: ; %bb.2: ; %atomicrmw.end7743; VI-NEXT: s_or_b64 exec, exec, s[34:35]7744; VI-NEXT: s_setpc_b64 s[30:31]7745;7746; GFX9-LABEL: global_atomic_umax_i64_ret_offset_scalar:7747; GFX9: ; %bb.0:7748; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7749; GFX9-NEXT: v_mov_b32_e32 v2, 07750; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5] offset:327751; GFX9-NEXT: s_mov_b64 s[34:35], 07752; GFX9-NEXT: v_mov_b32_e32 v3, s77753; GFX9-NEXT: v_mov_b32_e32 v4, s67754; GFX9-NEXT: .LBB101_1: ; %atomicrmw.start7755; GFX9-NEXT: ; =>This Inner Loop Header: Depth=17756; GFX9-NEXT: s_waitcnt vmcnt(0)7757; GFX9-NEXT: v_mov_b32_e32 v8, v17758; GFX9-NEXT: v_mov_b32_e32 v7, v07759; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[7:8]7760; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc7761; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc7762; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] offset:32 glc7763; GFX9-NEXT: s_waitcnt vmcnt(0)7764; GFX9-NEXT: buffer_wbinvl1_vol7765; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]7766; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]7767; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]7768; GFX9-NEXT: s_cbranch_execnz .LBB101_17769; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end7770; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]7771; GFX9-NEXT: s_setpc_b64 s[30:31]7772 %gep = getelementptr i64, ptr addrspace(1) %out, i64 47773 %result = atomicrmw umax ptr addrspace(1) %gep, i64 %in seq_cst7774 ret i64 %result7775}7776 7777define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr addrspace(1) %out, i64 %in, i64 %index) {7778; SI-LABEL: atomic_umax_i64_addr64_offset:7779; SI: ; %bb.0: ; %entry7780; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd7781; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x97782; SI-NEXT: s_waitcnt lgkmcnt(0)7783; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 37784; SI-NEXT: s_add_u32 s4, s0, s47785; SI-NEXT: s_addc_u32 s5, s1, s57786; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x87787; SI-NEXT: s_mov_b64 s[0:1], 07788; SI-NEXT: s_mov_b32 s7, 0xf0007789; SI-NEXT: v_mov_b32_e32 v4, s37790; SI-NEXT: v_mov_b32_e32 v5, s27791; SI-NEXT: s_waitcnt lgkmcnt(0)7792; SI-NEXT: v_mov_b32_e32 v2, s87793; SI-NEXT: v_mov_b32_e32 v3, s97794; SI-NEXT: s_mov_b32 s6, -17795; SI-NEXT: .LBB102_1: ; %atomicrmw.start7796; SI-NEXT: ; =>This Inner Loop Header: Depth=17797; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]7798; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc7799; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc7800; SI-NEXT: s_waitcnt expcnt(0)7801; SI-NEXT: v_mov_b32_e32 v9, v37802; SI-NEXT: v_mov_b32_e32 v8, v27803; SI-NEXT: v_mov_b32_e32 v7, v17804; SI-NEXT: v_mov_b32_e32 v6, v07805; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 offset:32 glc7806; SI-NEXT: s_waitcnt vmcnt(0)7807; SI-NEXT: buffer_wbinvl17808; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]7809; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]7810; SI-NEXT: v_mov_b32_e32 v2, v67811; SI-NEXT: v_mov_b32_e32 v3, v77812; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]7813; SI-NEXT: s_cbranch_execnz .LBB102_17814; SI-NEXT: ; %bb.2: ; %atomicrmw.end7815; SI-NEXT: s_endpgm7816;7817; VI-LABEL: atomic_umax_i64_addr64_offset:7818; VI: ; %bb.0: ; %entry7819; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x347820; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247821; VI-NEXT: s_mov_b64 s[4:5], 07822; VI-NEXT: s_waitcnt lgkmcnt(0)7823; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 37824; VI-NEXT: s_add_u32 s0, s0, s67825; VI-NEXT: s_addc_u32 s1, s1, s77826; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x207827; VI-NEXT: s_add_u32 s0, s0, 327828; VI-NEXT: s_addc_u32 s1, s1, 07829; VI-NEXT: v_mov_b32_e32 v5, s17830; VI-NEXT: v_mov_b32_e32 v6, s37831; VI-NEXT: s_waitcnt lgkmcnt(0)7832; VI-NEXT: v_mov_b32_e32 v2, s67833; VI-NEXT: v_mov_b32_e32 v7, s27834; VI-NEXT: v_mov_b32_e32 v3, s77835; VI-NEXT: v_mov_b32_e32 v4, s07836; VI-NEXT: .LBB102_1: ; %atomicrmw.start7837; VI-NEXT: ; =>This Inner Loop Header: Depth=17838; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]7839; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc7840; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc7841; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc7842; VI-NEXT: s_waitcnt vmcnt(0)7843; VI-NEXT: buffer_wbinvl1_vol7844; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]7845; VI-NEXT: v_mov_b32_e32 v3, v17846; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7847; VI-NEXT: v_mov_b32_e32 v2, v07848; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]7849; VI-NEXT: s_cbranch_execnz .LBB102_17850; VI-NEXT: ; %bb.2: ; %atomicrmw.end7851; VI-NEXT: s_endpgm7852;7853; GFX9-LABEL: atomic_umax_i64_addr64_offset:7854; GFX9: ; %bb.0: ; %entry7855; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x347856; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247857; GFX9-NEXT: v_mov_b32_e32 v6, 07858; GFX9-NEXT: s_waitcnt lgkmcnt(0)7859; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 37860; GFX9-NEXT: s_add_u32 s0, s0, s47861; GFX9-NEXT: s_addc_u32 s1, s1, s57862; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x207863; GFX9-NEXT: s_mov_b64 s[4:5], 07864; GFX9-NEXT: v_mov_b32_e32 v4, s37865; GFX9-NEXT: v_mov_b32_e32 v5, s27866; GFX9-NEXT: s_waitcnt lgkmcnt(0)7867; GFX9-NEXT: v_mov_b32_e32 v2, s67868; GFX9-NEXT: v_mov_b32_e32 v3, s77869; GFX9-NEXT: .LBB102_1: ; %atomicrmw.start7870; GFX9-NEXT: ; =>This Inner Loop Header: Depth=17871; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]7872; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc7873; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc7874; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] offset:32 glc7875; GFX9-NEXT: s_waitcnt vmcnt(0)7876; GFX9-NEXT: buffer_wbinvl1_vol7877; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]7878; GFX9-NEXT: v_mov_b32_e32 v3, v17879; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7880; GFX9-NEXT: v_mov_b32_e32 v2, v07881; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]7882; GFX9-NEXT: s_cbranch_execnz .LBB102_17883; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end7884; GFX9-NEXT: s_endpgm7885entry:7886 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index7887 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 47888 %tmp0 = atomicrmw umax ptr addrspace(1) %gep, i64 %in seq_cst7889 ret void7890}7891 7892define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {7893; SI-LABEL: atomic_umax_i64_ret_addr64_offset:7894; SI: ; %bb.0: ; %entry7895; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x97896; SI-NEXT: s_waitcnt lgkmcnt(0)7897; SI-NEXT: s_lshl_b64 s[6:7], s[6:7], 37898; SI-NEXT: s_add_u32 s8, s0, s67899; SI-NEXT: s_addc_u32 s9, s1, s77900; SI-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x87901; SI-NEXT: s_mov_b64 s[0:1], 07902; SI-NEXT: s_mov_b32 s11, 0xf0007903; SI-NEXT: v_mov_b32_e32 v8, s57904; SI-NEXT: v_mov_b32_e32 v9, s47905; SI-NEXT: s_waitcnt lgkmcnt(0)7906; SI-NEXT: v_mov_b32_e32 v2, s67907; SI-NEXT: v_mov_b32_e32 v3, s77908; SI-NEXT: s_mov_b32 s10, -17909; SI-NEXT: .LBB103_1: ; %atomicrmw.start7910; SI-NEXT: ; =>This Inner Loop Header: Depth=17911; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]7912; SI-NEXT: v_cndmask_b32_e32 v1, v8, v3, vcc7913; SI-NEXT: v_cndmask_b32_e32 v0, v9, v2, vcc7914; SI-NEXT: s_waitcnt expcnt(0)7915; SI-NEXT: v_mov_b32_e32 v7, v37916; SI-NEXT: v_mov_b32_e32 v6, v27917; SI-NEXT: v_mov_b32_e32 v5, v17918; SI-NEXT: v_mov_b32_e32 v4, v07919; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[8:11], 0 offset:32 glc7920; SI-NEXT: s_waitcnt vmcnt(0)7921; SI-NEXT: buffer_wbinvl17922; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]7923; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]7924; SI-NEXT: v_mov_b32_e32 v2, v47925; SI-NEXT: v_mov_b32_e32 v3, v57926; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]7927; SI-NEXT: s_cbranch_execnz .LBB103_17928; SI-NEXT: ; %bb.2: ; %atomicrmw.end7929; SI-NEXT: s_or_b64 exec, exec, s[0:1]7930; SI-NEXT: s_mov_b32 s7, 0xf0007931; SI-NEXT: s_mov_b32 s6, -17932; SI-NEXT: s_mov_b32 s4, s27933; SI-NEXT: s_mov_b32 s5, s37934; SI-NEXT: buffer_store_dwordx2 v[4:5], off, s[4:7], 07935; SI-NEXT: s_endpgm7936;7937; VI-LABEL: atomic_umax_i64_ret_addr64_offset:7938; VI: ; %bb.0: ; %entry7939; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x247940; VI-NEXT: s_mov_b64 s[8:9], 07941; VI-NEXT: s_waitcnt lgkmcnt(0)7942; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 37943; VI-NEXT: s_add_u32 s0, s0, s67944; VI-NEXT: s_addc_u32 s1, s1, s77945; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x207946; VI-NEXT: s_add_u32 s0, s0, 327947; VI-NEXT: s_addc_u32 s1, s1, 07948; VI-NEXT: v_mov_b32_e32 v0, s07949; VI-NEXT: v_mov_b32_e32 v4, s57950; VI-NEXT: s_waitcnt lgkmcnt(0)7951; VI-NEXT: v_mov_b32_e32 v2, s67952; VI-NEXT: v_mov_b32_e32 v5, s47953; VI-NEXT: v_mov_b32_e32 v3, s77954; VI-NEXT: v_mov_b32_e32 v1, s17955; VI-NEXT: .LBB103_1: ; %atomicrmw.start7956; VI-NEXT: ; =>This Inner Loop Header: Depth=17957; VI-NEXT: v_mov_b32_e32 v9, v37958; VI-NEXT: v_mov_b32_e32 v8, v27959; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]7960; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc7961; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc7962; VI-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc7963; VI-NEXT: s_waitcnt vmcnt(0)7964; VI-NEXT: buffer_wbinvl1_vol7965; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]7966; VI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]7967; VI-NEXT: s_andn2_b64 exec, exec, s[8:9]7968; VI-NEXT: s_cbranch_execnz .LBB103_17969; VI-NEXT: ; %bb.2: ; %atomicrmw.end7970; VI-NEXT: s_or_b64 exec, exec, s[8:9]7971; VI-NEXT: v_mov_b32_e32 v0, s27972; VI-NEXT: v_mov_b32_e32 v1, s37973; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]7974; VI-NEXT: s_endpgm7975;7976; GFX9-LABEL: atomic_umax_i64_ret_addr64_offset:7977; GFX9: ; %bb.0: ; %entry7978; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x247979; GFX9-NEXT: s_mov_b64 s[2:3], 07980; GFX9-NEXT: v_mov_b32_e32 v4, 07981; GFX9-NEXT: s_waitcnt lgkmcnt(0)7982; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 37983; GFX9-NEXT: s_add_u32 s0, s8, s07984; GFX9-NEXT: s_addc_u32 s1, s9, s17985; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x207986; GFX9-NEXT: v_mov_b32_e32 v2, s137987; GFX9-NEXT: v_mov_b32_e32 v3, s127988; GFX9-NEXT: s_waitcnt lgkmcnt(0)7989; GFX9-NEXT: v_mov_b32_e32 v0, s47990; GFX9-NEXT: v_mov_b32_e32 v1, s57991; GFX9-NEXT: .LBB103_1: ; %atomicrmw.start7992; GFX9-NEXT: ; =>This Inner Loop Header: Depth=17993; GFX9-NEXT: v_mov_b32_e32 v8, v17994; GFX9-NEXT: v_mov_b32_e32 v7, v07995; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[7:8]7996; GFX9-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc7997; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc7998; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[5:8], s[0:1] offset:32 glc7999; GFX9-NEXT: s_waitcnt vmcnt(0)8000; GFX9-NEXT: buffer_wbinvl1_vol8001; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]8002; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]8003; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]8004; GFX9-NEXT: s_cbranch_execnz .LBB103_18005; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end8006; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]8007; GFX9-NEXT: v_mov_b32_e32 v2, 08008; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]8009; GFX9-NEXT: s_endpgm8010entry:8011 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index8012 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 48013 %tmp0 = atomicrmw umax ptr addrspace(1) %gep, i64 %in seq_cst8014 store i64 %tmp0, ptr addrspace(1) %out28015 ret void8016}8017 8018define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {8019; SI-LABEL: atomic_umax_i64_ret_addr64:8020; SI: ; %bb.0: ; %entry8021; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x98022; SI-NEXT: s_waitcnt lgkmcnt(0)8023; SI-NEXT: s_lshl_b64 s[6:7], s[6:7], 38024; SI-NEXT: s_add_u32 s8, s0, s68025; SI-NEXT: s_addc_u32 s9, s1, s78026; SI-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x08027; SI-NEXT: s_mov_b64 s[0:1], 08028; SI-NEXT: s_mov_b32 s11, 0xf0008029; SI-NEXT: v_mov_b32_e32 v8, s58030; SI-NEXT: v_mov_b32_e32 v9, s48031; SI-NEXT: s_waitcnt lgkmcnt(0)8032; SI-NEXT: v_mov_b32_e32 v2, s68033; SI-NEXT: v_mov_b32_e32 v3, s78034; SI-NEXT: s_mov_b32 s10, -18035; SI-NEXT: .LBB104_1: ; %atomicrmw.start8036; SI-NEXT: ; =>This Inner Loop Header: Depth=18037; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]8038; SI-NEXT: v_cndmask_b32_e32 v1, v8, v3, vcc8039; SI-NEXT: v_cndmask_b32_e32 v0, v9, v2, vcc8040; SI-NEXT: s_waitcnt expcnt(0)8041; SI-NEXT: v_mov_b32_e32 v7, v38042; SI-NEXT: v_mov_b32_e32 v6, v28043; SI-NEXT: v_mov_b32_e32 v5, v18044; SI-NEXT: v_mov_b32_e32 v4, v08045; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[8:11], 0 glc8046; SI-NEXT: s_waitcnt vmcnt(0)8047; SI-NEXT: buffer_wbinvl18048; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]8049; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]8050; SI-NEXT: v_mov_b32_e32 v2, v48051; SI-NEXT: v_mov_b32_e32 v3, v58052; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]8053; SI-NEXT: s_cbranch_execnz .LBB104_18054; SI-NEXT: ; %bb.2: ; %atomicrmw.end8055; SI-NEXT: s_or_b64 exec, exec, s[0:1]8056; SI-NEXT: s_mov_b32 s7, 0xf0008057; SI-NEXT: s_mov_b32 s6, -18058; SI-NEXT: s_mov_b32 s4, s28059; SI-NEXT: s_mov_b32 s5, s38060; SI-NEXT: buffer_store_dwordx2 v[4:5], off, s[4:7], 08061; SI-NEXT: s_endpgm8062;8063; VI-LABEL: atomic_umax_i64_ret_addr64:8064; VI: ; %bb.0: ; %entry8065; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x248066; VI-NEXT: s_waitcnt lgkmcnt(0)8067; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 38068; VI-NEXT: s_add_u32 s6, s0, s68069; VI-NEXT: s_addc_u32 s7, s1, s78070; VI-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x08071; VI-NEXT: v_mov_b32_e32 v0, s68072; VI-NEXT: s_mov_b64 s[0:1], 08073; VI-NEXT: v_mov_b32_e32 v4, s58074; VI-NEXT: v_mov_b32_e32 v5, s48075; VI-NEXT: s_waitcnt lgkmcnt(0)8076; VI-NEXT: v_mov_b32_e32 v2, s88077; VI-NEXT: v_mov_b32_e32 v3, s98078; VI-NEXT: v_mov_b32_e32 v1, s78079; VI-NEXT: .LBB104_1: ; %atomicrmw.start8080; VI-NEXT: ; =>This Inner Loop Header: Depth=18081; VI-NEXT: v_mov_b32_e32 v9, v38082; VI-NEXT: v_mov_b32_e32 v8, v28083; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]8084; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc8085; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc8086; VI-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc8087; VI-NEXT: s_waitcnt vmcnt(0)8088; VI-NEXT: buffer_wbinvl1_vol8089; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]8090; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]8091; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]8092; VI-NEXT: s_cbranch_execnz .LBB104_18093; VI-NEXT: ; %bb.2: ; %atomicrmw.end8094; VI-NEXT: s_or_b64 exec, exec, s[0:1]8095; VI-NEXT: v_mov_b32_e32 v0, s28096; VI-NEXT: v_mov_b32_e32 v1, s38097; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]8098; VI-NEXT: s_endpgm8099;8100; GFX9-LABEL: atomic_umax_i64_ret_addr64:8101; GFX9: ; %bb.0: ; %entry8102; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x248103; GFX9-NEXT: s_mov_b64 s[2:3], 08104; GFX9-NEXT: v_mov_b32_e32 v4, 08105; GFX9-NEXT: s_waitcnt lgkmcnt(0)8106; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 38107; GFX9-NEXT: s_add_u32 s0, s8, s08108; GFX9-NEXT: s_addc_u32 s1, s9, s18109; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x08110; GFX9-NEXT: v_mov_b32_e32 v2, s138111; GFX9-NEXT: v_mov_b32_e32 v3, s128112; GFX9-NEXT: s_waitcnt lgkmcnt(0)8113; GFX9-NEXT: v_mov_b32_e32 v0, s48114; GFX9-NEXT: v_mov_b32_e32 v1, s58115; GFX9-NEXT: .LBB104_1: ; %atomicrmw.start8116; GFX9-NEXT: ; =>This Inner Loop Header: Depth=18117; GFX9-NEXT: v_mov_b32_e32 v8, v18118; GFX9-NEXT: v_mov_b32_e32 v7, v08119; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[7:8]8120; GFX9-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc8121; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc8122; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[5:8], s[0:1] glc8123; GFX9-NEXT: s_waitcnt vmcnt(0)8124; GFX9-NEXT: buffer_wbinvl1_vol8125; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]8126; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]8127; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]8128; GFX9-NEXT: s_cbranch_execnz .LBB104_18129; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end8130; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]8131; GFX9-NEXT: v_mov_b32_e32 v2, 08132; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]8133; GFX9-NEXT: s_endpgm8134entry:8135 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index8136 %tmp0 = atomicrmw umax ptr addrspace(1) %ptr, i64 %in seq_cst8137 store i64 %tmp0, ptr addrspace(1) %out28138 ret void8139}8140 8141define void @global_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {8142; SI-LABEL: global_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory:8143; SI: ; %bb.0:8144; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8145; SI-NEXT: s_mov_b32 s6, 08146; SI-NEXT: s_mov_b32 s7, 0xf0008147; SI-NEXT: s_mov_b32 s4, s68148; SI-NEXT: s_mov_b32 s5, s68149; SI-NEXT: buffer_atomic_umax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:328150; SI-NEXT: s_waitcnt vmcnt(0)8151; SI-NEXT: buffer_wbinvl18152; SI-NEXT: s_waitcnt expcnt(0)8153; SI-NEXT: s_setpc_b64 s[30:31]8154;8155; VI-LABEL: global_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory:8156; VI: ; %bb.0:8157; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8158; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v08159; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc8160; VI-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3]8161; VI-NEXT: s_waitcnt vmcnt(0)8162; VI-NEXT: buffer_wbinvl1_vol8163; VI-NEXT: s_setpc_b64 s[30:31]8164;8165; GFX9-LABEL: global_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory:8166; GFX9: ; %bb.0:8167; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8168; GFX9-NEXT: global_atomic_umax_x2 v[0:1], v[2:3], off offset:328169; GFX9-NEXT: s_waitcnt vmcnt(0)8170; GFX9-NEXT: buffer_wbinvl1_vol8171; GFX9-NEXT: s_setpc_b64 s[30:31]8172 %gep = getelementptr i64, ptr addrspace(1) %out, i64 48173 %tmp0 = atomicrmw umax ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !08174 ret void8175}8176 8177define i64 @global_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {8178; SI-LABEL: global_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:8179; SI: ; %bb.0:8180; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8181; SI-NEXT: s_mov_b32 s6, 08182; SI-NEXT: s_mov_b32 s7, 0xf0008183; SI-NEXT: s_mov_b32 s4, s68184; SI-NEXT: s_mov_b32 s5, s68185; SI-NEXT: buffer_atomic_umax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc8186; SI-NEXT: s_waitcnt vmcnt(0)8187; SI-NEXT: buffer_wbinvl18188; SI-NEXT: v_mov_b32_e32 v0, v28189; SI-NEXT: v_mov_b32_e32 v1, v38190; SI-NEXT: s_waitcnt expcnt(0)8191; SI-NEXT: s_setpc_b64 s[30:31]8192;8193; VI-LABEL: global_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:8194; VI: ; %bb.0:8195; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8196; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v08197; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc8198; VI-NEXT: flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc8199; VI-NEXT: s_waitcnt vmcnt(0)8200; VI-NEXT: buffer_wbinvl1_vol8201; VI-NEXT: s_setpc_b64 s[30:31]8202;8203; GFX9-LABEL: global_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:8204; GFX9: ; %bb.0:8205; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8206; GFX9-NEXT: global_atomic_umax_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc8207; GFX9-NEXT: s_waitcnt vmcnt(0)8208; GFX9-NEXT: buffer_wbinvl1_vol8209; GFX9-NEXT: s_setpc_b64 s[30:31]8210 %gep = getelementptr i64, ptr addrspace(1) %out, i64 48211 %result = atomicrmw umax ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !08212 ret i64 %result8213}8214 8215; ---------------------------------------------------------------------8216; atomicrmw umin8217; ---------------------------------------------------------------------8218 8219define void @global_atomic_umin_i64_noret(ptr addrspace(1) %ptr, i64 %in) {8220; SI-LABEL: global_atomic_umin_i64_noret:8221; SI: ; %bb.0:8222; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8223; SI-NEXT: s_mov_b32 s6, 08224; SI-NEXT: s_mov_b32 s7, 0xf0008225; SI-NEXT: s_mov_b32 s4, s68226; SI-NEXT: s_mov_b32 s5, s68227; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr648228; SI-NEXT: s_mov_b64 s[8:9], 08229; SI-NEXT: .LBB107_1: ; %atomicrmw.start8230; SI-NEXT: ; =>This Inner Loop Header: Depth=18231; SI-NEXT: s_waitcnt vmcnt(0)8232; SI-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]8233; SI-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc8234; SI-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc8235; SI-NEXT: s_waitcnt expcnt(0)8236; SI-NEXT: v_mov_b32_e32 v11, v78237; SI-NEXT: v_mov_b32_e32 v10, v68238; SI-NEXT: v_mov_b32_e32 v9, v58239; SI-NEXT: v_mov_b32_e32 v8, v48240; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc8241; SI-NEXT: s_waitcnt vmcnt(0)8242; SI-NEXT: buffer_wbinvl18243; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]8244; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]8245; SI-NEXT: v_mov_b32_e32 v6, v88246; SI-NEXT: v_mov_b32_e32 v7, v98247; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]8248; SI-NEXT: s_cbranch_execnz .LBB107_18249; SI-NEXT: ; %bb.2: ; %atomicrmw.end8250; SI-NEXT: s_or_b64 exec, exec, s[8:9]8251; SI-NEXT: s_waitcnt expcnt(0)8252; SI-NEXT: s_setpc_b64 s[30:31]8253;8254; VI-LABEL: global_atomic_umin_i64_noret:8255; VI: ; %bb.0:8256; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8257; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]8258; VI-NEXT: s_mov_b64 s[4:5], 08259; VI-NEXT: .LBB107_1: ; %atomicrmw.start8260; VI-NEXT: ; =>This Inner Loop Header: Depth=18261; VI-NEXT: s_waitcnt vmcnt(0)8262; VI-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]8263; VI-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc8264; VI-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc8265; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc8266; VI-NEXT: s_waitcnt vmcnt(0)8267; VI-NEXT: buffer_wbinvl1_vol8268; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]8269; VI-NEXT: v_mov_b32_e32 v7, v58270; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8271; VI-NEXT: v_mov_b32_e32 v6, v48272; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]8273; VI-NEXT: s_cbranch_execnz .LBB107_18274; VI-NEXT: ; %bb.2: ; %atomicrmw.end8275; VI-NEXT: s_or_b64 exec, exec, s[4:5]8276; VI-NEXT: s_setpc_b64 s[30:31]8277;8278; GFX9-LABEL: global_atomic_umin_i64_noret:8279; GFX9: ; %bb.0:8280; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8281; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off8282; GFX9-NEXT: s_mov_b64 s[4:5], 08283; GFX9-NEXT: .LBB107_1: ; %atomicrmw.start8284; GFX9-NEXT: ; =>This Inner Loop Header: Depth=18285; GFX9-NEXT: s_waitcnt vmcnt(0)8286; GFX9-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]8287; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc8288; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc8289; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc8290; GFX9-NEXT: s_waitcnt vmcnt(0)8291; GFX9-NEXT: buffer_wbinvl1_vol8292; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]8293; GFX9-NEXT: v_mov_b32_e32 v7, v58294; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8295; GFX9-NEXT: v_mov_b32_e32 v6, v48296; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]8297; GFX9-NEXT: s_cbranch_execnz .LBB107_18298; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end8299; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]8300; GFX9-NEXT: s_setpc_b64 s[30:31]8301 %tmp0 = atomicrmw umin ptr addrspace(1) %ptr, i64 %in seq_cst8302 ret void8303}8304 8305define void @global_atomic_umin_i64_noret_offset(ptr addrspace(1) %out, i64 %in) {8306; SI-LABEL: global_atomic_umin_i64_noret_offset:8307; SI: ; %bb.0:8308; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8309; SI-NEXT: s_mov_b32 s6, 08310; SI-NEXT: s_mov_b32 s7, 0xf0008311; SI-NEXT: s_mov_b32 s4, s68312; SI-NEXT: s_mov_b32 s5, s68313; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:328314; SI-NEXT: s_mov_b64 s[8:9], 08315; SI-NEXT: .LBB108_1: ; %atomicrmw.start8316; SI-NEXT: ; =>This Inner Loop Header: Depth=18317; SI-NEXT: s_waitcnt vmcnt(0)8318; SI-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]8319; SI-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc8320; SI-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc8321; SI-NEXT: s_waitcnt expcnt(0)8322; SI-NEXT: v_mov_b32_e32 v11, v78323; SI-NEXT: v_mov_b32_e32 v10, v68324; SI-NEXT: v_mov_b32_e32 v9, v58325; SI-NEXT: v_mov_b32_e32 v8, v48326; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32 glc8327; SI-NEXT: s_waitcnt vmcnt(0)8328; SI-NEXT: buffer_wbinvl18329; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]8330; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]8331; SI-NEXT: v_mov_b32_e32 v6, v88332; SI-NEXT: v_mov_b32_e32 v7, v98333; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]8334; SI-NEXT: s_cbranch_execnz .LBB108_18335; SI-NEXT: ; %bb.2: ; %atomicrmw.end8336; SI-NEXT: s_or_b64 exec, exec, s[8:9]8337; SI-NEXT: s_waitcnt expcnt(0)8338; SI-NEXT: s_setpc_b64 s[30:31]8339;8340; VI-LABEL: global_atomic_umin_i64_noret_offset:8341; VI: ; %bb.0:8342; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8343; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v08344; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc8345; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]8346; VI-NEXT: s_mov_b64 s[4:5], 08347; VI-NEXT: .LBB108_1: ; %atomicrmw.start8348; VI-NEXT: ; =>This Inner Loop Header: Depth=18349; VI-NEXT: s_waitcnt vmcnt(0)8350; VI-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]8351; VI-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc8352; VI-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc8353; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc8354; VI-NEXT: s_waitcnt vmcnt(0)8355; VI-NEXT: buffer_wbinvl1_vol8356; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]8357; VI-NEXT: v_mov_b32_e32 v7, v58358; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8359; VI-NEXT: v_mov_b32_e32 v6, v48360; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]8361; VI-NEXT: s_cbranch_execnz .LBB108_18362; VI-NEXT: ; %bb.2: ; %atomicrmw.end8363; VI-NEXT: s_or_b64 exec, exec, s[4:5]8364; VI-NEXT: s_setpc_b64 s[30:31]8365;8366; GFX9-LABEL: global_atomic_umin_i64_noret_offset:8367; GFX9: ; %bb.0:8368; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8369; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:328370; GFX9-NEXT: s_mov_b64 s[4:5], 08371; GFX9-NEXT: .LBB108_1: ; %atomicrmw.start8372; GFX9-NEXT: ; =>This Inner Loop Header: Depth=18373; GFX9-NEXT: s_waitcnt vmcnt(0)8374; GFX9-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]8375; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc8376; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc8377; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc8378; GFX9-NEXT: s_waitcnt vmcnt(0)8379; GFX9-NEXT: buffer_wbinvl1_vol8380; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]8381; GFX9-NEXT: v_mov_b32_e32 v7, v58382; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8383; GFX9-NEXT: v_mov_b32_e32 v6, v48384; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]8385; GFX9-NEXT: s_cbranch_execnz .LBB108_18386; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end8387; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]8388; GFX9-NEXT: s_setpc_b64 s[30:31]8389 %gep = getelementptr i64, ptr addrspace(1) %out, i64 48390 %tmp0 = atomicrmw umin ptr addrspace(1) %gep, i64 %in seq_cst8391 ret void8392}8393 8394define i64 @global_atomic_umin_i64_ret(ptr addrspace(1) %ptr, i64 %in) {8395; SI-LABEL: global_atomic_umin_i64_ret:8396; SI: ; %bb.0:8397; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8398; SI-NEXT: v_mov_b32_e32 v5, v38399; SI-NEXT: v_mov_b32_e32 v4, v28400; SI-NEXT: v_mov_b32_e32 v7, v18401; SI-NEXT: v_mov_b32_e32 v6, v08402; SI-NEXT: s_mov_b32 s6, 08403; SI-NEXT: s_mov_b32 s7, 0xf0008404; SI-NEXT: s_mov_b32 s4, s68405; SI-NEXT: s_mov_b32 s5, s68406; SI-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[4:7], 0 addr648407; SI-NEXT: s_mov_b64 s[8:9], 08408; SI-NEXT: .LBB109_1: ; %atomicrmw.start8409; SI-NEXT: ; =>This Inner Loop Header: Depth=18410; SI-NEXT: s_waitcnt vmcnt(0)8411; SI-NEXT: v_cmp_le_u64_e32 vcc, v[10:11], v[4:5]8412; SI-NEXT: v_cndmask_b32_e32 v9, v5, v11, vcc8413; SI-NEXT: v_cndmask_b32_e32 v8, v4, v10, vcc8414; SI-NEXT: s_waitcnt expcnt(0)8415; SI-NEXT: v_mov_b32_e32 v0, v88416; SI-NEXT: v_mov_b32_e32 v1, v98417; SI-NEXT: v_mov_b32_e32 v2, v108418; SI-NEXT: v_mov_b32_e32 v3, v118419; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc8420; SI-NEXT: s_waitcnt vmcnt(0)8421; SI-NEXT: buffer_wbinvl18422; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]8423; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]8424; SI-NEXT: v_mov_b32_e32 v11, v18425; SI-NEXT: v_mov_b32_e32 v10, v08426; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]8427; SI-NEXT: s_cbranch_execnz .LBB109_18428; SI-NEXT: ; %bb.2: ; %atomicrmw.end8429; SI-NEXT: s_or_b64 exec, exec, s[8:9]8430; SI-NEXT: s_waitcnt expcnt(0)8431; SI-NEXT: s_setpc_b64 s[30:31]8432;8433; VI-LABEL: global_atomic_umin_i64_ret:8434; VI: ; %bb.0:8435; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8436; VI-NEXT: flat_load_dwordx2 v[4:5], v[0:1]8437; VI-NEXT: s_mov_b64 s[4:5], 08438; VI-NEXT: .LBB109_1: ; %atomicrmw.start8439; VI-NEXT: ; =>This Inner Loop Header: Depth=18440; VI-NEXT: s_waitcnt vmcnt(0)8441; VI-NEXT: v_mov_b32_e32 v7, v58442; VI-NEXT: v_mov_b32_e32 v6, v48443; VI-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]8444; VI-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc8445; VI-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc8446; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc8447; VI-NEXT: s_waitcnt vmcnt(0)8448; VI-NEXT: buffer_wbinvl1_vol8449; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]8450; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8451; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]8452; VI-NEXT: s_cbranch_execnz .LBB109_18453; VI-NEXT: ; %bb.2: ; %atomicrmw.end8454; VI-NEXT: s_or_b64 exec, exec, s[4:5]8455; VI-NEXT: v_mov_b32_e32 v0, v48456; VI-NEXT: v_mov_b32_e32 v1, v58457; VI-NEXT: s_setpc_b64 s[30:31]8458;8459; GFX9-LABEL: global_atomic_umin_i64_ret:8460; GFX9: ; %bb.0:8461; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8462; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off8463; GFX9-NEXT: s_mov_b64 s[4:5], 08464; GFX9-NEXT: .LBB109_1: ; %atomicrmw.start8465; GFX9-NEXT: ; =>This Inner Loop Header: Depth=18466; GFX9-NEXT: s_waitcnt vmcnt(0)8467; GFX9-NEXT: v_mov_b32_e32 v7, v58468; GFX9-NEXT: v_mov_b32_e32 v6, v48469; GFX9-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]8470; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc8471; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc8472; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc8473; GFX9-NEXT: s_waitcnt vmcnt(0)8474; GFX9-NEXT: buffer_wbinvl1_vol8475; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]8476; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8477; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]8478; GFX9-NEXT: s_cbranch_execnz .LBB109_18479; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end8480; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]8481; GFX9-NEXT: v_mov_b32_e32 v0, v48482; GFX9-NEXT: v_mov_b32_e32 v1, v58483; GFX9-NEXT: s_setpc_b64 s[30:31]8484 %result = atomicrmw umin ptr addrspace(1) %ptr, i64 %in seq_cst8485 ret i64 %result8486}8487 8488define i64 @global_atomic_umin_i64_ret_offset(ptr addrspace(1) %out, i64 %in) {8489; SI-LABEL: global_atomic_umin_i64_ret_offset:8490; SI: ; %bb.0:8491; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8492; SI-NEXT: v_mov_b32_e32 v5, v38493; SI-NEXT: v_mov_b32_e32 v4, v28494; SI-NEXT: v_mov_b32_e32 v7, v18495; SI-NEXT: v_mov_b32_e32 v6, v08496; SI-NEXT: s_mov_b32 s6, 08497; SI-NEXT: s_mov_b32 s7, 0xf0008498; SI-NEXT: s_mov_b32 s4, s68499; SI-NEXT: s_mov_b32 s5, s68500; SI-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[4:7], 0 addr64 offset:328501; SI-NEXT: s_mov_b64 s[8:9], 08502; SI-NEXT: .LBB110_1: ; %atomicrmw.start8503; SI-NEXT: ; =>This Inner Loop Header: Depth=18504; SI-NEXT: s_waitcnt vmcnt(0)8505; SI-NEXT: v_cmp_le_u64_e32 vcc, v[10:11], v[4:5]8506; SI-NEXT: v_cndmask_b32_e32 v9, v5, v11, vcc8507; SI-NEXT: v_cndmask_b32_e32 v8, v4, v10, vcc8508; SI-NEXT: s_waitcnt expcnt(0)8509; SI-NEXT: v_mov_b32_e32 v0, v88510; SI-NEXT: v_mov_b32_e32 v1, v98511; SI-NEXT: v_mov_b32_e32 v2, v108512; SI-NEXT: v_mov_b32_e32 v3, v118513; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 offset:32 glc8514; SI-NEXT: s_waitcnt vmcnt(0)8515; SI-NEXT: buffer_wbinvl18516; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]8517; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]8518; SI-NEXT: v_mov_b32_e32 v11, v18519; SI-NEXT: v_mov_b32_e32 v10, v08520; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]8521; SI-NEXT: s_cbranch_execnz .LBB110_18522; SI-NEXT: ; %bb.2: ; %atomicrmw.end8523; SI-NEXT: s_or_b64 exec, exec, s[8:9]8524; SI-NEXT: s_waitcnt expcnt(0)8525; SI-NEXT: s_setpc_b64 s[30:31]8526;8527; VI-LABEL: global_atomic_umin_i64_ret_offset:8528; VI: ; %bb.0:8529; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8530; VI-NEXT: v_add_u32_e32 v4, vcc, 32, v08531; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc8532; VI-NEXT: flat_load_dwordx2 v[0:1], v[4:5]8533; VI-NEXT: s_mov_b64 s[4:5], 08534; VI-NEXT: .LBB110_1: ; %atomicrmw.start8535; VI-NEXT: ; =>This Inner Loop Header: Depth=18536; VI-NEXT: s_waitcnt vmcnt(0)8537; VI-NEXT: v_mov_b32_e32 v9, v18538; VI-NEXT: v_mov_b32_e32 v8, v08539; VI-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]8540; VI-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc8541; VI-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc8542; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc8543; VI-NEXT: s_waitcnt vmcnt(0)8544; VI-NEXT: buffer_wbinvl1_vol8545; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]8546; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8547; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]8548; VI-NEXT: s_cbranch_execnz .LBB110_18549; VI-NEXT: ; %bb.2: ; %atomicrmw.end8550; VI-NEXT: s_or_b64 exec, exec, s[4:5]8551; VI-NEXT: s_setpc_b64 s[30:31]8552;8553; GFX9-LABEL: global_atomic_umin_i64_ret_offset:8554; GFX9: ; %bb.0:8555; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8556; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:328557; GFX9-NEXT: s_mov_b64 s[4:5], 08558; GFX9-NEXT: .LBB110_1: ; %atomicrmw.start8559; GFX9-NEXT: ; =>This Inner Loop Header: Depth=18560; GFX9-NEXT: s_waitcnt vmcnt(0)8561; GFX9-NEXT: v_mov_b32_e32 v7, v58562; GFX9-NEXT: v_mov_b32_e32 v6, v48563; GFX9-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]8564; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc8565; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc8566; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc8567; GFX9-NEXT: s_waitcnt vmcnt(0)8568; GFX9-NEXT: buffer_wbinvl1_vol8569; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]8570; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8571; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]8572; GFX9-NEXT: s_cbranch_execnz .LBB110_18573; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end8574; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]8575; GFX9-NEXT: v_mov_b32_e32 v0, v48576; GFX9-NEXT: v_mov_b32_e32 v1, v58577; GFX9-NEXT: s_setpc_b64 s[30:31]8578 %gep = getelementptr i64, ptr addrspace(1) %out, i64 48579 %result = atomicrmw umin ptr addrspace(1) %gep, i64 %in seq_cst8580 ret i64 %result8581}8582 8583define amdgpu_gfx void @global_atomic_umin_i64_noret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {8584; SI-LABEL: global_atomic_umin_i64_noret_scalar:8585; SI: ; %bb.0:8586; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8587; SI-NEXT: s_xor_saveexec_b64 s[34:35], -18588; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill8589; SI-NEXT: s_mov_b64 exec, s[34:35]8590; SI-NEXT: s_waitcnt expcnt(0)8591; SI-NEXT: v_writelane_b32 v10, s6, 08592; SI-NEXT: v_writelane_b32 v10, s7, 18593; SI-NEXT: s_mov_b32 s35, s78594; SI-NEXT: s_mov_b32 s34, s68595; SI-NEXT: s_mov_b32 s7, 0xf0008596; SI-NEXT: s_mov_b32 s6, -18597; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 08598; SI-NEXT: s_mov_b64 s[36:37], 08599; SI-NEXT: v_mov_b32_e32 v4, s358600; SI-NEXT: v_mov_b32_e32 v5, s348601; SI-NEXT: .LBB111_1: ; %atomicrmw.start8602; SI-NEXT: ; =>This Inner Loop Header: Depth=18603; SI-NEXT: s_waitcnt vmcnt(0)8604; SI-NEXT: v_cmp_ge_u64_e32 vcc, s[34:35], v[2:3]8605; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc8606; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc8607; SI-NEXT: s_waitcnt expcnt(0)8608; SI-NEXT: v_mov_b32_e32 v9, v38609; SI-NEXT: v_mov_b32_e32 v8, v28610; SI-NEXT: v_mov_b32_e32 v7, v18611; SI-NEXT: v_mov_b32_e32 v6, v08612; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 glc8613; SI-NEXT: s_waitcnt vmcnt(0)8614; SI-NEXT: buffer_wbinvl18615; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]8616; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]8617; SI-NEXT: v_mov_b32_e32 v2, v68618; SI-NEXT: v_mov_b32_e32 v3, v78619; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]8620; SI-NEXT: s_cbranch_execnz .LBB111_18621; SI-NEXT: ; %bb.2: ; %atomicrmw.end8622; SI-NEXT: s_or_b64 exec, exec, s[36:37]8623; SI-NEXT: v_readlane_b32 s7, v10, 18624; SI-NEXT: v_readlane_b32 s6, v10, 08625; SI-NEXT: s_xor_saveexec_b64 s[34:35], -18626; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload8627; SI-NEXT: s_mov_b64 exec, s[34:35]8628; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)8629; SI-NEXT: s_setpc_b64 s[30:31]8630;8631; VI-LABEL: global_atomic_umin_i64_noret_scalar:8632; VI: ; %bb.0:8633; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8634; VI-NEXT: v_mov_b32_e32 v0, s48635; VI-NEXT: v_mov_b32_e32 v1, s58636; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]8637; VI-NEXT: v_mov_b32_e32 v4, s48638; VI-NEXT: s_mov_b64 s[34:35], 08639; VI-NEXT: v_mov_b32_e32 v6, s78640; VI-NEXT: v_mov_b32_e32 v7, s68641; VI-NEXT: v_mov_b32_e32 v5, s58642; VI-NEXT: .LBB111_1: ; %atomicrmw.start8643; VI-NEXT: ; =>This Inner Loop Header: Depth=18644; VI-NEXT: s_waitcnt vmcnt(0)8645; VI-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]8646; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc8647; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc8648; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8649; VI-NEXT: s_waitcnt vmcnt(0)8650; VI-NEXT: buffer_wbinvl1_vol8651; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8652; VI-NEXT: v_mov_b32_e32 v3, v18653; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8654; VI-NEXT: v_mov_b32_e32 v2, v08655; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]8656; VI-NEXT: s_cbranch_execnz .LBB111_18657; VI-NEXT: ; %bb.2: ; %atomicrmw.end8658; VI-NEXT: s_or_b64 exec, exec, s[34:35]8659; VI-NEXT: s_setpc_b64 s[30:31]8660;8661; GFX9-LABEL: global_atomic_umin_i64_noret_scalar:8662; GFX9: ; %bb.0:8663; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8664; GFX9-NEXT: v_mov_b32_e32 v4, 08665; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5]8666; GFX9-NEXT: s_mov_b64 s[34:35], 08667; GFX9-NEXT: v_mov_b32_e32 v5, s78668; GFX9-NEXT: v_mov_b32_e32 v6, s68669; GFX9-NEXT: .LBB111_1: ; %atomicrmw.start8670; GFX9-NEXT: ; =>This Inner Loop Header: Depth=18671; GFX9-NEXT: s_waitcnt vmcnt(0)8672; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]8673; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc8674; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc8675; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc8676; GFX9-NEXT: s_waitcnt vmcnt(0)8677; GFX9-NEXT: buffer_wbinvl1_vol8678; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8679; GFX9-NEXT: v_mov_b32_e32 v3, v18680; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8681; GFX9-NEXT: v_mov_b32_e32 v2, v08682; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]8683; GFX9-NEXT: s_cbranch_execnz .LBB111_18684; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end8685; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]8686; GFX9-NEXT: s_setpc_b64 s[30:31]8687 %tmp0 = atomicrmw umin ptr addrspace(1) %ptr, i64 %in seq_cst8688 ret void8689}8690 8691define amdgpu_gfx void @global_atomic_umin_i64_noret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {8692; SI-LABEL: global_atomic_umin_i64_noret_offset_scalar:8693; SI: ; %bb.0:8694; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8695; SI-NEXT: s_xor_saveexec_b64 s[34:35], -18696; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill8697; SI-NEXT: s_mov_b64 exec, s[34:35]8698; SI-NEXT: s_waitcnt expcnt(0)8699; SI-NEXT: v_writelane_b32 v10, s6, 08700; SI-NEXT: v_writelane_b32 v10, s7, 18701; SI-NEXT: s_mov_b32 s35, s78702; SI-NEXT: s_mov_b32 s34, s68703; SI-NEXT: s_mov_b32 s7, 0xf0008704; SI-NEXT: s_mov_b32 s6, -18705; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:328706; SI-NEXT: s_mov_b64 s[36:37], 08707; SI-NEXT: v_mov_b32_e32 v4, s358708; SI-NEXT: v_mov_b32_e32 v5, s348709; SI-NEXT: .LBB112_1: ; %atomicrmw.start8710; SI-NEXT: ; =>This Inner Loop Header: Depth=18711; SI-NEXT: s_waitcnt vmcnt(0)8712; SI-NEXT: v_cmp_ge_u64_e32 vcc, s[34:35], v[2:3]8713; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc8714; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc8715; SI-NEXT: s_waitcnt expcnt(0)8716; SI-NEXT: v_mov_b32_e32 v9, v38717; SI-NEXT: v_mov_b32_e32 v8, v28718; SI-NEXT: v_mov_b32_e32 v7, v18719; SI-NEXT: v_mov_b32_e32 v6, v08720; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 offset:32 glc8721; SI-NEXT: s_waitcnt vmcnt(0)8722; SI-NEXT: buffer_wbinvl18723; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]8724; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]8725; SI-NEXT: v_mov_b32_e32 v2, v68726; SI-NEXT: v_mov_b32_e32 v3, v78727; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]8728; SI-NEXT: s_cbranch_execnz .LBB112_18729; SI-NEXT: ; %bb.2: ; %atomicrmw.end8730; SI-NEXT: s_or_b64 exec, exec, s[36:37]8731; SI-NEXT: v_readlane_b32 s7, v10, 18732; SI-NEXT: v_readlane_b32 s6, v10, 08733; SI-NEXT: s_xor_saveexec_b64 s[34:35], -18734; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload8735; SI-NEXT: s_mov_b64 exec, s[34:35]8736; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)8737; SI-NEXT: s_setpc_b64 s[30:31]8738;8739; VI-LABEL: global_atomic_umin_i64_noret_offset_scalar:8740; VI: ; %bb.0:8741; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8742; VI-NEXT: s_add_u32 s34, s4, 328743; VI-NEXT: s_addc_u32 s35, s5, 08744; VI-NEXT: v_mov_b32_e32 v4, s348745; VI-NEXT: v_mov_b32_e32 v5, s358746; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]8747; VI-NEXT: s_mov_b64 s[34:35], 08748; VI-NEXT: v_mov_b32_e32 v6, s78749; VI-NEXT: v_mov_b32_e32 v7, s68750; VI-NEXT: .LBB112_1: ; %atomicrmw.start8751; VI-NEXT: ; =>This Inner Loop Header: Depth=18752; VI-NEXT: s_waitcnt vmcnt(0)8753; VI-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]8754; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc8755; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc8756; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8757; VI-NEXT: s_waitcnt vmcnt(0)8758; VI-NEXT: buffer_wbinvl1_vol8759; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8760; VI-NEXT: v_mov_b32_e32 v3, v18761; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8762; VI-NEXT: v_mov_b32_e32 v2, v08763; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]8764; VI-NEXT: s_cbranch_execnz .LBB112_18765; VI-NEXT: ; %bb.2: ; %atomicrmw.end8766; VI-NEXT: s_or_b64 exec, exec, s[34:35]8767; VI-NEXT: s_setpc_b64 s[30:31]8768;8769; GFX9-LABEL: global_atomic_umin_i64_noret_offset_scalar:8770; GFX9: ; %bb.0:8771; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8772; GFX9-NEXT: v_mov_b32_e32 v4, 08773; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5] offset:328774; GFX9-NEXT: s_mov_b64 s[34:35], 08775; GFX9-NEXT: v_mov_b32_e32 v5, s78776; GFX9-NEXT: v_mov_b32_e32 v6, s68777; GFX9-NEXT: .LBB112_1: ; %atomicrmw.start8778; GFX9-NEXT: ; =>This Inner Loop Header: Depth=18779; GFX9-NEXT: s_waitcnt vmcnt(0)8780; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]8781; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc8782; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc8783; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc8784; GFX9-NEXT: s_waitcnt vmcnt(0)8785; GFX9-NEXT: buffer_wbinvl1_vol8786; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8787; GFX9-NEXT: v_mov_b32_e32 v3, v18788; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8789; GFX9-NEXT: v_mov_b32_e32 v2, v08790; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]8791; GFX9-NEXT: s_cbranch_execnz .LBB112_18792; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end8793; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]8794; GFX9-NEXT: s_setpc_b64 s[30:31]8795 %gep = getelementptr i64, ptr addrspace(1) %out, i64 48796 %tmp0 = atomicrmw umin ptr addrspace(1) %gep, i64 %in seq_cst8797 ret void8798}8799 8800define amdgpu_gfx i64 @global_atomic_umin_i64_ret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {8801; SI-LABEL: global_atomic_umin_i64_ret_scalar:8802; SI: ; %bb.0:8803; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8804; SI-NEXT: s_xor_saveexec_b64 s[34:35], -18805; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill8806; SI-NEXT: s_mov_b64 exec, s[34:35]8807; SI-NEXT: s_waitcnt expcnt(0)8808; SI-NEXT: v_writelane_b32 v8, s6, 08809; SI-NEXT: v_writelane_b32 v8, s7, 18810; SI-NEXT: s_mov_b32 s35, s78811; SI-NEXT: s_mov_b32 s34, s68812; SI-NEXT: s_mov_b32 s7, 0xf0008813; SI-NEXT: s_mov_b32 s6, -18814; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 08815; SI-NEXT: s_mov_b64 s[36:37], 08816; SI-NEXT: v_mov_b32_e32 v6, s358817; SI-NEXT: v_mov_b32_e32 v7, s348818; SI-NEXT: .LBB113_1: ; %atomicrmw.start8819; SI-NEXT: ; =>This Inner Loop Header: Depth=18820; SI-NEXT: s_waitcnt vmcnt(0)8821; SI-NEXT: v_cmp_ge_u64_e32 vcc, s[34:35], v[4:5]8822; SI-NEXT: s_waitcnt expcnt(0)8823; SI-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc8824; SI-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc8825; SI-NEXT: v_mov_b32_e32 v0, v28826; SI-NEXT: v_mov_b32_e32 v1, v38827; SI-NEXT: v_mov_b32_e32 v2, v48828; SI-NEXT: v_mov_b32_e32 v3, v58829; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc8830; SI-NEXT: s_waitcnt vmcnt(0)8831; SI-NEXT: buffer_wbinvl18832; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]8833; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]8834; SI-NEXT: v_mov_b32_e32 v5, v18835; SI-NEXT: v_mov_b32_e32 v4, v08836; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]8837; SI-NEXT: s_cbranch_execnz .LBB113_18838; SI-NEXT: ; %bb.2: ; %atomicrmw.end8839; SI-NEXT: s_or_b64 exec, exec, s[36:37]8840; SI-NEXT: v_readlane_b32 s7, v8, 18841; SI-NEXT: v_readlane_b32 s6, v8, 08842; SI-NEXT: s_xor_saveexec_b64 s[34:35], -18843; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload8844; SI-NEXT: s_mov_b64 exec, s[34:35]8845; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)8846; SI-NEXT: s_setpc_b64 s[30:31]8847;8848; VI-LABEL: global_atomic_umin_i64_ret_scalar:8849; VI: ; %bb.0:8850; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8851; VI-NEXT: v_mov_b32_e32 v0, s48852; VI-NEXT: v_mov_b32_e32 v1, s58853; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]8854; VI-NEXT: v_mov_b32_e32 v2, s48855; VI-NEXT: s_mov_b64 s[34:35], 08856; VI-NEXT: v_mov_b32_e32 v4, s78857; VI-NEXT: v_mov_b32_e32 v5, s68858; VI-NEXT: v_mov_b32_e32 v3, s58859; VI-NEXT: .LBB113_1: ; %atomicrmw.start8860; VI-NEXT: ; =>This Inner Loop Header: Depth=18861; VI-NEXT: s_waitcnt vmcnt(0)8862; VI-NEXT: v_mov_b32_e32 v9, v18863; VI-NEXT: v_mov_b32_e32 v8, v08864; VI-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]8865; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc8866; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc8867; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc8868; VI-NEXT: s_waitcnt vmcnt(0)8869; VI-NEXT: buffer_wbinvl1_vol8870; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]8871; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8872; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]8873; VI-NEXT: s_cbranch_execnz .LBB113_18874; VI-NEXT: ; %bb.2: ; %atomicrmw.end8875; VI-NEXT: s_or_b64 exec, exec, s[34:35]8876; VI-NEXT: s_setpc_b64 s[30:31]8877;8878; GFX9-LABEL: global_atomic_umin_i64_ret_scalar:8879; GFX9: ; %bb.0:8880; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8881; GFX9-NEXT: v_mov_b32_e32 v2, 08882; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]8883; GFX9-NEXT: s_mov_b64 s[34:35], 08884; GFX9-NEXT: v_mov_b32_e32 v3, s78885; GFX9-NEXT: v_mov_b32_e32 v4, s68886; GFX9-NEXT: .LBB113_1: ; %atomicrmw.start8887; GFX9-NEXT: ; =>This Inner Loop Header: Depth=18888; GFX9-NEXT: s_waitcnt vmcnt(0)8889; GFX9-NEXT: v_mov_b32_e32 v8, v18890; GFX9-NEXT: v_mov_b32_e32 v7, v08891; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[7:8]8892; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc8893; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc8894; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] glc8895; GFX9-NEXT: s_waitcnt vmcnt(0)8896; GFX9-NEXT: buffer_wbinvl1_vol8897; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]8898; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8899; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]8900; GFX9-NEXT: s_cbranch_execnz .LBB113_18901; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end8902; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]8903; GFX9-NEXT: s_setpc_b64 s[30:31]8904 %result = atomicrmw umin ptr addrspace(1) %ptr, i64 %in seq_cst8905 ret i64 %result8906}8907 8908define amdgpu_gfx i64 @global_atomic_umin_i64_ret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {8909; SI-LABEL: global_atomic_umin_i64_ret_offset_scalar:8910; SI: ; %bb.0:8911; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8912; SI-NEXT: s_xor_saveexec_b64 s[34:35], -18913; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill8914; SI-NEXT: s_mov_b64 exec, s[34:35]8915; SI-NEXT: s_waitcnt expcnt(0)8916; SI-NEXT: v_writelane_b32 v8, s6, 08917; SI-NEXT: v_writelane_b32 v8, s7, 18918; SI-NEXT: s_mov_b32 s35, s78919; SI-NEXT: s_mov_b32 s34, s68920; SI-NEXT: s_mov_b32 s7, 0xf0008921; SI-NEXT: s_mov_b32 s6, -18922; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0 offset:328923; SI-NEXT: s_mov_b64 s[36:37], 08924; SI-NEXT: v_mov_b32_e32 v6, s358925; SI-NEXT: v_mov_b32_e32 v7, s348926; SI-NEXT: .LBB114_1: ; %atomicrmw.start8927; SI-NEXT: ; =>This Inner Loop Header: Depth=18928; SI-NEXT: s_waitcnt vmcnt(0)8929; SI-NEXT: v_cmp_ge_u64_e32 vcc, s[34:35], v[4:5]8930; SI-NEXT: s_waitcnt expcnt(0)8931; SI-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc8932; SI-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc8933; SI-NEXT: v_mov_b32_e32 v0, v28934; SI-NEXT: v_mov_b32_e32 v1, v38935; SI-NEXT: v_mov_b32_e32 v2, v48936; SI-NEXT: v_mov_b32_e32 v3, v58937; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 offset:32 glc8938; SI-NEXT: s_waitcnt vmcnt(0)8939; SI-NEXT: buffer_wbinvl18940; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]8941; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]8942; SI-NEXT: v_mov_b32_e32 v5, v18943; SI-NEXT: v_mov_b32_e32 v4, v08944; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]8945; SI-NEXT: s_cbranch_execnz .LBB114_18946; SI-NEXT: ; %bb.2: ; %atomicrmw.end8947; SI-NEXT: s_or_b64 exec, exec, s[36:37]8948; SI-NEXT: v_readlane_b32 s7, v8, 18949; SI-NEXT: v_readlane_b32 s6, v8, 08950; SI-NEXT: s_xor_saveexec_b64 s[34:35], -18951; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload8952; SI-NEXT: s_mov_b64 exec, s[34:35]8953; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)8954; SI-NEXT: s_setpc_b64 s[30:31]8955;8956; VI-LABEL: global_atomic_umin_i64_ret_offset_scalar:8957; VI: ; %bb.0:8958; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8959; VI-NEXT: s_add_u32 s34, s4, 328960; VI-NEXT: s_addc_u32 s35, s5, 08961; VI-NEXT: v_mov_b32_e32 v2, s348962; VI-NEXT: v_mov_b32_e32 v3, s358963; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]8964; VI-NEXT: s_mov_b64 s[34:35], 08965; VI-NEXT: v_mov_b32_e32 v4, s78966; VI-NEXT: v_mov_b32_e32 v5, s68967; VI-NEXT: .LBB114_1: ; %atomicrmw.start8968; VI-NEXT: ; =>This Inner Loop Header: Depth=18969; VI-NEXT: s_waitcnt vmcnt(0)8970; VI-NEXT: v_mov_b32_e32 v9, v18971; VI-NEXT: v_mov_b32_e32 v8, v08972; VI-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]8973; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc8974; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc8975; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc8976; VI-NEXT: s_waitcnt vmcnt(0)8977; VI-NEXT: buffer_wbinvl1_vol8978; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]8979; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8980; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]8981; VI-NEXT: s_cbranch_execnz .LBB114_18982; VI-NEXT: ; %bb.2: ; %atomicrmw.end8983; VI-NEXT: s_or_b64 exec, exec, s[34:35]8984; VI-NEXT: s_setpc_b64 s[30:31]8985;8986; GFX9-LABEL: global_atomic_umin_i64_ret_offset_scalar:8987; GFX9: ; %bb.0:8988; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8989; GFX9-NEXT: v_mov_b32_e32 v2, 08990; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5] offset:328991; GFX9-NEXT: s_mov_b64 s[34:35], 08992; GFX9-NEXT: v_mov_b32_e32 v3, s78993; GFX9-NEXT: v_mov_b32_e32 v4, s68994; GFX9-NEXT: .LBB114_1: ; %atomicrmw.start8995; GFX9-NEXT: ; =>This Inner Loop Header: Depth=18996; GFX9-NEXT: s_waitcnt vmcnt(0)8997; GFX9-NEXT: v_mov_b32_e32 v8, v18998; GFX9-NEXT: v_mov_b32_e32 v7, v08999; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[7:8]9000; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc9001; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc9002; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] offset:32 glc9003; GFX9-NEXT: s_waitcnt vmcnt(0)9004; GFX9-NEXT: buffer_wbinvl1_vol9005; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]9006; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]9007; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]9008; GFX9-NEXT: s_cbranch_execnz .LBB114_19009; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end9010; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]9011; GFX9-NEXT: s_setpc_b64 s[30:31]9012 %gep = getelementptr i64, ptr addrspace(1) %out, i64 49013 %result = atomicrmw umin ptr addrspace(1) %gep, i64 %in seq_cst9014 ret i64 %result9015}9016 9017define void @global_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {9018; SI-LABEL: global_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory:9019; SI: ; %bb.0:9020; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9021; SI-NEXT: s_mov_b32 s6, 09022; SI-NEXT: s_mov_b32 s7, 0xf0009023; SI-NEXT: s_mov_b32 s4, s69024; SI-NEXT: s_mov_b32 s5, s69025; SI-NEXT: buffer_atomic_umin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:329026; SI-NEXT: s_waitcnt vmcnt(0)9027; SI-NEXT: buffer_wbinvl19028; SI-NEXT: s_waitcnt expcnt(0)9029; SI-NEXT: s_setpc_b64 s[30:31]9030;9031; VI-LABEL: global_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory:9032; VI: ; %bb.0:9033; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9034; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v09035; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc9036; VI-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3]9037; VI-NEXT: s_waitcnt vmcnt(0)9038; VI-NEXT: buffer_wbinvl1_vol9039; VI-NEXT: s_setpc_b64 s[30:31]9040;9041; GFX9-LABEL: global_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory:9042; GFX9: ; %bb.0:9043; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9044; GFX9-NEXT: global_atomic_umin_x2 v[0:1], v[2:3], off offset:329045; GFX9-NEXT: s_waitcnt vmcnt(0)9046; GFX9-NEXT: buffer_wbinvl1_vol9047; GFX9-NEXT: s_setpc_b64 s[30:31]9048 %gep = getelementptr i64, ptr addrspace(1) %out, i64 49049 %tmp0 = atomicrmw umin ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !09050 ret void9051}9052 9053define i64 @global_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {9054; SI-LABEL: global_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:9055; SI: ; %bb.0:9056; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9057; SI-NEXT: s_mov_b32 s6, 09058; SI-NEXT: s_mov_b32 s7, 0xf0009059; SI-NEXT: s_mov_b32 s4, s69060; SI-NEXT: s_mov_b32 s5, s69061; SI-NEXT: buffer_atomic_umin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc9062; SI-NEXT: s_waitcnt vmcnt(0)9063; SI-NEXT: buffer_wbinvl19064; SI-NEXT: v_mov_b32_e32 v0, v29065; SI-NEXT: v_mov_b32_e32 v1, v39066; SI-NEXT: s_waitcnt expcnt(0)9067; SI-NEXT: s_setpc_b64 s[30:31]9068;9069; VI-LABEL: global_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:9070; VI: ; %bb.0:9071; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9072; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v09073; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc9074; VI-NEXT: flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc9075; VI-NEXT: s_waitcnt vmcnt(0)9076; VI-NEXT: buffer_wbinvl1_vol9077; VI-NEXT: s_setpc_b64 s[30:31]9078;9079; GFX9-LABEL: global_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:9080; GFX9: ; %bb.0:9081; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9082; GFX9-NEXT: global_atomic_umin_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc9083; GFX9-NEXT: s_waitcnt vmcnt(0)9084; GFX9-NEXT: buffer_wbinvl1_vol9085; GFX9-NEXT: s_setpc_b64 s[30:31]9086 %gep = getelementptr i64, ptr addrspace(1) %out, i64 49087 %result = atomicrmw umin ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !09088 ret i64 %result9089}9090 9091; ---------------------------------------------------------------------9092; atomicrmw min9093; ---------------------------------------------------------------------9094 9095define void @global_atomic_min_i64_noret(ptr addrspace(1) %ptr, i64 %in) {9096; SI-LABEL: global_atomic_min_i64_noret:9097; SI: ; %bb.0:9098; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9099; SI-NEXT: s_mov_b32 s6, 09100; SI-NEXT: s_mov_b32 s7, 0xf0009101; SI-NEXT: s_mov_b32 s4, s69102; SI-NEXT: s_mov_b32 s5, s69103; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr649104; SI-NEXT: s_mov_b64 s[8:9], 09105; SI-NEXT: .LBB117_1: ; %atomicrmw.start9106; SI-NEXT: ; =>This Inner Loop Header: Depth=19107; SI-NEXT: s_waitcnt vmcnt(0)9108; SI-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]9109; SI-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc9110; SI-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc9111; SI-NEXT: s_waitcnt expcnt(0)9112; SI-NEXT: v_mov_b32_e32 v11, v79113; SI-NEXT: v_mov_b32_e32 v10, v69114; SI-NEXT: v_mov_b32_e32 v9, v59115; SI-NEXT: v_mov_b32_e32 v8, v49116; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc9117; SI-NEXT: s_waitcnt vmcnt(0)9118; SI-NEXT: buffer_wbinvl19119; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]9120; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]9121; SI-NEXT: v_mov_b32_e32 v6, v89122; SI-NEXT: v_mov_b32_e32 v7, v99123; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]9124; SI-NEXT: s_cbranch_execnz .LBB117_19125; SI-NEXT: ; %bb.2: ; %atomicrmw.end9126; SI-NEXT: s_or_b64 exec, exec, s[8:9]9127; SI-NEXT: s_waitcnt expcnt(0)9128; SI-NEXT: s_setpc_b64 s[30:31]9129;9130; VI-LABEL: global_atomic_min_i64_noret:9131; VI: ; %bb.0:9132; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9133; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]9134; VI-NEXT: s_mov_b64 s[4:5], 09135; VI-NEXT: .LBB117_1: ; %atomicrmw.start9136; VI-NEXT: ; =>This Inner Loop Header: Depth=19137; VI-NEXT: s_waitcnt vmcnt(0)9138; VI-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]9139; VI-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc9140; VI-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc9141; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc9142; VI-NEXT: s_waitcnt vmcnt(0)9143; VI-NEXT: buffer_wbinvl1_vol9144; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]9145; VI-NEXT: v_mov_b32_e32 v7, v59146; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9147; VI-NEXT: v_mov_b32_e32 v6, v49148; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]9149; VI-NEXT: s_cbranch_execnz .LBB117_19150; VI-NEXT: ; %bb.2: ; %atomicrmw.end9151; VI-NEXT: s_or_b64 exec, exec, s[4:5]9152; VI-NEXT: s_setpc_b64 s[30:31]9153;9154; GFX9-LABEL: global_atomic_min_i64_noret:9155; GFX9: ; %bb.0:9156; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9157; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off9158; GFX9-NEXT: s_mov_b64 s[4:5], 09159; GFX9-NEXT: .LBB117_1: ; %atomicrmw.start9160; GFX9-NEXT: ; =>This Inner Loop Header: Depth=19161; GFX9-NEXT: s_waitcnt vmcnt(0)9162; GFX9-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]9163; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc9164; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc9165; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc9166; GFX9-NEXT: s_waitcnt vmcnt(0)9167; GFX9-NEXT: buffer_wbinvl1_vol9168; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]9169; GFX9-NEXT: v_mov_b32_e32 v7, v59170; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9171; GFX9-NEXT: v_mov_b32_e32 v6, v49172; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]9173; GFX9-NEXT: s_cbranch_execnz .LBB117_19174; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end9175; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]9176; GFX9-NEXT: s_setpc_b64 s[30:31]9177 %tmp0 = atomicrmw min ptr addrspace(1) %ptr, i64 %in seq_cst9178 ret void9179}9180 9181define void @global_atomic_min_i64_noret_offset(ptr addrspace(1) %out, i64 %in) {9182; SI-LABEL: global_atomic_min_i64_noret_offset:9183; SI: ; %bb.0:9184; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9185; SI-NEXT: s_mov_b32 s6, 09186; SI-NEXT: s_mov_b32 s7, 0xf0009187; SI-NEXT: s_mov_b32 s4, s69188; SI-NEXT: s_mov_b32 s5, s69189; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:329190; SI-NEXT: s_mov_b64 s[8:9], 09191; SI-NEXT: .LBB118_1: ; %atomicrmw.start9192; SI-NEXT: ; =>This Inner Loop Header: Depth=19193; SI-NEXT: s_waitcnt vmcnt(0)9194; SI-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]9195; SI-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc9196; SI-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc9197; SI-NEXT: s_waitcnt expcnt(0)9198; SI-NEXT: v_mov_b32_e32 v11, v79199; SI-NEXT: v_mov_b32_e32 v10, v69200; SI-NEXT: v_mov_b32_e32 v9, v59201; SI-NEXT: v_mov_b32_e32 v8, v49202; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32 glc9203; SI-NEXT: s_waitcnt vmcnt(0)9204; SI-NEXT: buffer_wbinvl19205; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]9206; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]9207; SI-NEXT: v_mov_b32_e32 v6, v89208; SI-NEXT: v_mov_b32_e32 v7, v99209; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]9210; SI-NEXT: s_cbranch_execnz .LBB118_19211; SI-NEXT: ; %bb.2: ; %atomicrmw.end9212; SI-NEXT: s_or_b64 exec, exec, s[8:9]9213; SI-NEXT: s_waitcnt expcnt(0)9214; SI-NEXT: s_setpc_b64 s[30:31]9215;9216; VI-LABEL: global_atomic_min_i64_noret_offset:9217; VI: ; %bb.0:9218; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9219; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v09220; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc9221; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]9222; VI-NEXT: s_mov_b64 s[4:5], 09223; VI-NEXT: .LBB118_1: ; %atomicrmw.start9224; VI-NEXT: ; =>This Inner Loop Header: Depth=19225; VI-NEXT: s_waitcnt vmcnt(0)9226; VI-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]9227; VI-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc9228; VI-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc9229; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc9230; VI-NEXT: s_waitcnt vmcnt(0)9231; VI-NEXT: buffer_wbinvl1_vol9232; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]9233; VI-NEXT: v_mov_b32_e32 v7, v59234; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9235; VI-NEXT: v_mov_b32_e32 v6, v49236; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]9237; VI-NEXT: s_cbranch_execnz .LBB118_19238; VI-NEXT: ; %bb.2: ; %atomicrmw.end9239; VI-NEXT: s_or_b64 exec, exec, s[4:5]9240; VI-NEXT: s_setpc_b64 s[30:31]9241;9242; GFX9-LABEL: global_atomic_min_i64_noret_offset:9243; GFX9: ; %bb.0:9244; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9245; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:329246; GFX9-NEXT: s_mov_b64 s[4:5], 09247; GFX9-NEXT: .LBB118_1: ; %atomicrmw.start9248; GFX9-NEXT: ; =>This Inner Loop Header: Depth=19249; GFX9-NEXT: s_waitcnt vmcnt(0)9250; GFX9-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]9251; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc9252; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc9253; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc9254; GFX9-NEXT: s_waitcnt vmcnt(0)9255; GFX9-NEXT: buffer_wbinvl1_vol9256; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]9257; GFX9-NEXT: v_mov_b32_e32 v7, v59258; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9259; GFX9-NEXT: v_mov_b32_e32 v6, v49260; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]9261; GFX9-NEXT: s_cbranch_execnz .LBB118_19262; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end9263; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]9264; GFX9-NEXT: s_setpc_b64 s[30:31]9265 %gep = getelementptr i64, ptr addrspace(1) %out, i64 49266 %tmp0 = atomicrmw min ptr addrspace(1) %gep, i64 %in seq_cst9267 ret void9268}9269 9270define i64 @global_atomic_min_i64_ret(ptr addrspace(1) %ptr, i64 %in) {9271; SI-LABEL: global_atomic_min_i64_ret:9272; SI: ; %bb.0:9273; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9274; SI-NEXT: v_mov_b32_e32 v5, v39275; SI-NEXT: v_mov_b32_e32 v4, v29276; SI-NEXT: v_mov_b32_e32 v7, v19277; SI-NEXT: v_mov_b32_e32 v6, v09278; SI-NEXT: s_mov_b32 s6, 09279; SI-NEXT: s_mov_b32 s7, 0xf0009280; SI-NEXT: s_mov_b32 s4, s69281; SI-NEXT: s_mov_b32 s5, s69282; SI-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[4:7], 0 addr649283; SI-NEXT: s_mov_b64 s[8:9], 09284; SI-NEXT: .LBB119_1: ; %atomicrmw.start9285; SI-NEXT: ; =>This Inner Loop Header: Depth=19286; SI-NEXT: s_waitcnt vmcnt(0)9287; SI-NEXT: v_cmp_le_i64_e32 vcc, v[10:11], v[4:5]9288; SI-NEXT: v_cndmask_b32_e32 v9, v5, v11, vcc9289; SI-NEXT: v_cndmask_b32_e32 v8, v4, v10, vcc9290; SI-NEXT: s_waitcnt expcnt(0)9291; SI-NEXT: v_mov_b32_e32 v0, v89292; SI-NEXT: v_mov_b32_e32 v1, v99293; SI-NEXT: v_mov_b32_e32 v2, v109294; SI-NEXT: v_mov_b32_e32 v3, v119295; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc9296; SI-NEXT: s_waitcnt vmcnt(0)9297; SI-NEXT: buffer_wbinvl19298; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]9299; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]9300; SI-NEXT: v_mov_b32_e32 v11, v19301; SI-NEXT: v_mov_b32_e32 v10, v09302; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]9303; SI-NEXT: s_cbranch_execnz .LBB119_19304; SI-NEXT: ; %bb.2: ; %atomicrmw.end9305; SI-NEXT: s_or_b64 exec, exec, s[8:9]9306; SI-NEXT: s_waitcnt expcnt(0)9307; SI-NEXT: s_setpc_b64 s[30:31]9308;9309; VI-LABEL: global_atomic_min_i64_ret:9310; VI: ; %bb.0:9311; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9312; VI-NEXT: flat_load_dwordx2 v[4:5], v[0:1]9313; VI-NEXT: s_mov_b64 s[4:5], 09314; VI-NEXT: .LBB119_1: ; %atomicrmw.start9315; VI-NEXT: ; =>This Inner Loop Header: Depth=19316; VI-NEXT: s_waitcnt vmcnt(0)9317; VI-NEXT: v_mov_b32_e32 v7, v59318; VI-NEXT: v_mov_b32_e32 v6, v49319; VI-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]9320; VI-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc9321; VI-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc9322; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc9323; VI-NEXT: s_waitcnt vmcnt(0)9324; VI-NEXT: buffer_wbinvl1_vol9325; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]9326; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9327; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]9328; VI-NEXT: s_cbranch_execnz .LBB119_19329; VI-NEXT: ; %bb.2: ; %atomicrmw.end9330; VI-NEXT: s_or_b64 exec, exec, s[4:5]9331; VI-NEXT: v_mov_b32_e32 v0, v49332; VI-NEXT: v_mov_b32_e32 v1, v59333; VI-NEXT: s_setpc_b64 s[30:31]9334;9335; GFX9-LABEL: global_atomic_min_i64_ret:9336; GFX9: ; %bb.0:9337; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9338; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off9339; GFX9-NEXT: s_mov_b64 s[4:5], 09340; GFX9-NEXT: .LBB119_1: ; %atomicrmw.start9341; GFX9-NEXT: ; =>This Inner Loop Header: Depth=19342; GFX9-NEXT: s_waitcnt vmcnt(0)9343; GFX9-NEXT: v_mov_b32_e32 v7, v59344; GFX9-NEXT: v_mov_b32_e32 v6, v49345; GFX9-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]9346; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc9347; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc9348; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc9349; GFX9-NEXT: s_waitcnt vmcnt(0)9350; GFX9-NEXT: buffer_wbinvl1_vol9351; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]9352; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9353; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]9354; GFX9-NEXT: s_cbranch_execnz .LBB119_19355; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end9356; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]9357; GFX9-NEXT: v_mov_b32_e32 v0, v49358; GFX9-NEXT: v_mov_b32_e32 v1, v59359; GFX9-NEXT: s_setpc_b64 s[30:31]9360 %result = atomicrmw min ptr addrspace(1) %ptr, i64 %in seq_cst9361 ret i64 %result9362}9363 9364define i64 @global_atomic_min_i64_ret_offset(ptr addrspace(1) %out, i64 %in) {9365; SI-LABEL: global_atomic_min_i64_ret_offset:9366; SI: ; %bb.0:9367; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9368; SI-NEXT: v_mov_b32_e32 v5, v39369; SI-NEXT: v_mov_b32_e32 v4, v29370; SI-NEXT: v_mov_b32_e32 v7, v19371; SI-NEXT: v_mov_b32_e32 v6, v09372; SI-NEXT: s_mov_b32 s6, 09373; SI-NEXT: s_mov_b32 s7, 0xf0009374; SI-NEXT: s_mov_b32 s4, s69375; SI-NEXT: s_mov_b32 s5, s69376; SI-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[4:7], 0 addr64 offset:329377; SI-NEXT: s_mov_b64 s[8:9], 09378; SI-NEXT: .LBB120_1: ; %atomicrmw.start9379; SI-NEXT: ; =>This Inner Loop Header: Depth=19380; SI-NEXT: s_waitcnt vmcnt(0)9381; SI-NEXT: v_cmp_le_i64_e32 vcc, v[10:11], v[4:5]9382; SI-NEXT: v_cndmask_b32_e32 v9, v5, v11, vcc9383; SI-NEXT: v_cndmask_b32_e32 v8, v4, v10, vcc9384; SI-NEXT: s_waitcnt expcnt(0)9385; SI-NEXT: v_mov_b32_e32 v0, v89386; SI-NEXT: v_mov_b32_e32 v1, v99387; SI-NEXT: v_mov_b32_e32 v2, v109388; SI-NEXT: v_mov_b32_e32 v3, v119389; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 offset:32 glc9390; SI-NEXT: s_waitcnt vmcnt(0)9391; SI-NEXT: buffer_wbinvl19392; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]9393; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]9394; SI-NEXT: v_mov_b32_e32 v11, v19395; SI-NEXT: v_mov_b32_e32 v10, v09396; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]9397; SI-NEXT: s_cbranch_execnz .LBB120_19398; SI-NEXT: ; %bb.2: ; %atomicrmw.end9399; SI-NEXT: s_or_b64 exec, exec, s[8:9]9400; SI-NEXT: s_waitcnt expcnt(0)9401; SI-NEXT: s_setpc_b64 s[30:31]9402;9403; VI-LABEL: global_atomic_min_i64_ret_offset:9404; VI: ; %bb.0:9405; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9406; VI-NEXT: v_add_u32_e32 v4, vcc, 32, v09407; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc9408; VI-NEXT: flat_load_dwordx2 v[0:1], v[4:5]9409; VI-NEXT: s_mov_b64 s[4:5], 09410; VI-NEXT: .LBB120_1: ; %atomicrmw.start9411; VI-NEXT: ; =>This Inner Loop Header: Depth=19412; VI-NEXT: s_waitcnt vmcnt(0)9413; VI-NEXT: v_mov_b32_e32 v9, v19414; VI-NEXT: v_mov_b32_e32 v8, v09415; VI-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]9416; VI-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc9417; VI-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc9418; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc9419; VI-NEXT: s_waitcnt vmcnt(0)9420; VI-NEXT: buffer_wbinvl1_vol9421; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]9422; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9423; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]9424; VI-NEXT: s_cbranch_execnz .LBB120_19425; VI-NEXT: ; %bb.2: ; %atomicrmw.end9426; VI-NEXT: s_or_b64 exec, exec, s[4:5]9427; VI-NEXT: s_setpc_b64 s[30:31]9428;9429; GFX9-LABEL: global_atomic_min_i64_ret_offset:9430; GFX9: ; %bb.0:9431; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9432; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:329433; GFX9-NEXT: s_mov_b64 s[4:5], 09434; GFX9-NEXT: .LBB120_1: ; %atomicrmw.start9435; GFX9-NEXT: ; =>This Inner Loop Header: Depth=19436; GFX9-NEXT: s_waitcnt vmcnt(0)9437; GFX9-NEXT: v_mov_b32_e32 v7, v59438; GFX9-NEXT: v_mov_b32_e32 v6, v49439; GFX9-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]9440; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc9441; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc9442; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc9443; GFX9-NEXT: s_waitcnt vmcnt(0)9444; GFX9-NEXT: buffer_wbinvl1_vol9445; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]9446; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9447; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]9448; GFX9-NEXT: s_cbranch_execnz .LBB120_19449; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end9450; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]9451; GFX9-NEXT: v_mov_b32_e32 v0, v49452; GFX9-NEXT: v_mov_b32_e32 v1, v59453; GFX9-NEXT: s_setpc_b64 s[30:31]9454 %gep = getelementptr i64, ptr addrspace(1) %out, i64 49455 %result = atomicrmw min ptr addrspace(1) %gep, i64 %in seq_cst9456 ret i64 %result9457}9458 9459define amdgpu_gfx void @global_atomic_min_i64_noret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {9460; SI-LABEL: global_atomic_min_i64_noret_scalar:9461; SI: ; %bb.0:9462; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9463; SI-NEXT: s_xor_saveexec_b64 s[34:35], -19464; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill9465; SI-NEXT: s_mov_b64 exec, s[34:35]9466; SI-NEXT: s_waitcnt expcnt(0)9467; SI-NEXT: v_writelane_b32 v10, s6, 09468; SI-NEXT: v_writelane_b32 v10, s7, 19469; SI-NEXT: s_mov_b32 s35, s79470; SI-NEXT: s_mov_b32 s34, s69471; SI-NEXT: s_mov_b32 s7, 0xf0009472; SI-NEXT: s_mov_b32 s6, -19473; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 09474; SI-NEXT: s_mov_b64 s[36:37], 09475; SI-NEXT: v_mov_b32_e32 v4, s359476; SI-NEXT: v_mov_b32_e32 v5, s349477; SI-NEXT: .LBB121_1: ; %atomicrmw.start9478; SI-NEXT: ; =>This Inner Loop Header: Depth=19479; SI-NEXT: s_waitcnt vmcnt(0)9480; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[34:35], v[2:3]9481; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc9482; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc9483; SI-NEXT: s_waitcnt expcnt(0)9484; SI-NEXT: v_mov_b32_e32 v9, v39485; SI-NEXT: v_mov_b32_e32 v8, v29486; SI-NEXT: v_mov_b32_e32 v7, v19487; SI-NEXT: v_mov_b32_e32 v6, v09488; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 glc9489; SI-NEXT: s_waitcnt vmcnt(0)9490; SI-NEXT: buffer_wbinvl19491; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]9492; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]9493; SI-NEXT: v_mov_b32_e32 v2, v69494; SI-NEXT: v_mov_b32_e32 v3, v79495; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]9496; SI-NEXT: s_cbranch_execnz .LBB121_19497; SI-NEXT: ; %bb.2: ; %atomicrmw.end9498; SI-NEXT: s_or_b64 exec, exec, s[36:37]9499; SI-NEXT: v_readlane_b32 s7, v10, 19500; SI-NEXT: v_readlane_b32 s6, v10, 09501; SI-NEXT: s_xor_saveexec_b64 s[34:35], -19502; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload9503; SI-NEXT: s_mov_b64 exec, s[34:35]9504; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)9505; SI-NEXT: s_setpc_b64 s[30:31]9506;9507; VI-LABEL: global_atomic_min_i64_noret_scalar:9508; VI: ; %bb.0:9509; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9510; VI-NEXT: v_mov_b32_e32 v0, s49511; VI-NEXT: v_mov_b32_e32 v1, s59512; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]9513; VI-NEXT: v_mov_b32_e32 v4, s49514; VI-NEXT: s_mov_b64 s[34:35], 09515; VI-NEXT: v_mov_b32_e32 v6, s79516; VI-NEXT: v_mov_b32_e32 v7, s69517; VI-NEXT: v_mov_b32_e32 v5, s59518; VI-NEXT: .LBB121_1: ; %atomicrmw.start9519; VI-NEXT: ; =>This Inner Loop Header: Depth=19520; VI-NEXT: s_waitcnt vmcnt(0)9521; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]9522; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc9523; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc9524; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc9525; VI-NEXT: s_waitcnt vmcnt(0)9526; VI-NEXT: buffer_wbinvl1_vol9527; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]9528; VI-NEXT: v_mov_b32_e32 v3, v19529; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]9530; VI-NEXT: v_mov_b32_e32 v2, v09531; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]9532; VI-NEXT: s_cbranch_execnz .LBB121_19533; VI-NEXT: ; %bb.2: ; %atomicrmw.end9534; VI-NEXT: s_or_b64 exec, exec, s[34:35]9535; VI-NEXT: s_setpc_b64 s[30:31]9536;9537; GFX9-LABEL: global_atomic_min_i64_noret_scalar:9538; GFX9: ; %bb.0:9539; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9540; GFX9-NEXT: v_mov_b32_e32 v4, 09541; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5]9542; GFX9-NEXT: s_mov_b64 s[34:35], 09543; GFX9-NEXT: v_mov_b32_e32 v5, s79544; GFX9-NEXT: v_mov_b32_e32 v6, s69545; GFX9-NEXT: .LBB121_1: ; %atomicrmw.start9546; GFX9-NEXT: ; =>This Inner Loop Header: Depth=19547; GFX9-NEXT: s_waitcnt vmcnt(0)9548; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]9549; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc9550; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc9551; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc9552; GFX9-NEXT: s_waitcnt vmcnt(0)9553; GFX9-NEXT: buffer_wbinvl1_vol9554; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]9555; GFX9-NEXT: v_mov_b32_e32 v3, v19556; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]9557; GFX9-NEXT: v_mov_b32_e32 v2, v09558; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]9559; GFX9-NEXT: s_cbranch_execnz .LBB121_19560; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end9561; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]9562; GFX9-NEXT: s_setpc_b64 s[30:31]9563 %tmp0 = atomicrmw min ptr addrspace(1) %ptr, i64 %in seq_cst9564 ret void9565}9566 9567define amdgpu_gfx void @global_atomic_min_i64_noret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {9568; SI-LABEL: global_atomic_min_i64_noret_offset_scalar:9569; SI: ; %bb.0:9570; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9571; SI-NEXT: s_xor_saveexec_b64 s[34:35], -19572; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill9573; SI-NEXT: s_mov_b64 exec, s[34:35]9574; SI-NEXT: s_waitcnt expcnt(0)9575; SI-NEXT: v_writelane_b32 v10, s6, 09576; SI-NEXT: v_writelane_b32 v10, s7, 19577; SI-NEXT: s_mov_b32 s35, s79578; SI-NEXT: s_mov_b32 s34, s69579; SI-NEXT: s_mov_b32 s7, 0xf0009580; SI-NEXT: s_mov_b32 s6, -19581; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:329582; SI-NEXT: s_mov_b64 s[36:37], 09583; SI-NEXT: v_mov_b32_e32 v4, s359584; SI-NEXT: v_mov_b32_e32 v5, s349585; SI-NEXT: .LBB122_1: ; %atomicrmw.start9586; SI-NEXT: ; =>This Inner Loop Header: Depth=19587; SI-NEXT: s_waitcnt vmcnt(0)9588; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[34:35], v[2:3]9589; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc9590; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc9591; SI-NEXT: s_waitcnt expcnt(0)9592; SI-NEXT: v_mov_b32_e32 v9, v39593; SI-NEXT: v_mov_b32_e32 v8, v29594; SI-NEXT: v_mov_b32_e32 v7, v19595; SI-NEXT: v_mov_b32_e32 v6, v09596; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 offset:32 glc9597; SI-NEXT: s_waitcnt vmcnt(0)9598; SI-NEXT: buffer_wbinvl19599; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]9600; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]9601; SI-NEXT: v_mov_b32_e32 v2, v69602; SI-NEXT: v_mov_b32_e32 v3, v79603; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]9604; SI-NEXT: s_cbranch_execnz .LBB122_19605; SI-NEXT: ; %bb.2: ; %atomicrmw.end9606; SI-NEXT: s_or_b64 exec, exec, s[36:37]9607; SI-NEXT: v_readlane_b32 s7, v10, 19608; SI-NEXT: v_readlane_b32 s6, v10, 09609; SI-NEXT: s_xor_saveexec_b64 s[34:35], -19610; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload9611; SI-NEXT: s_mov_b64 exec, s[34:35]9612; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)9613; SI-NEXT: s_setpc_b64 s[30:31]9614;9615; VI-LABEL: global_atomic_min_i64_noret_offset_scalar:9616; VI: ; %bb.0:9617; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9618; VI-NEXT: s_add_u32 s34, s4, 329619; VI-NEXT: s_addc_u32 s35, s5, 09620; VI-NEXT: v_mov_b32_e32 v4, s349621; VI-NEXT: v_mov_b32_e32 v5, s359622; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]9623; VI-NEXT: s_mov_b64 s[34:35], 09624; VI-NEXT: v_mov_b32_e32 v6, s79625; VI-NEXT: v_mov_b32_e32 v7, s69626; VI-NEXT: .LBB122_1: ; %atomicrmw.start9627; VI-NEXT: ; =>This Inner Loop Header: Depth=19628; VI-NEXT: s_waitcnt vmcnt(0)9629; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]9630; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc9631; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc9632; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc9633; VI-NEXT: s_waitcnt vmcnt(0)9634; VI-NEXT: buffer_wbinvl1_vol9635; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]9636; VI-NEXT: v_mov_b32_e32 v3, v19637; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]9638; VI-NEXT: v_mov_b32_e32 v2, v09639; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]9640; VI-NEXT: s_cbranch_execnz .LBB122_19641; VI-NEXT: ; %bb.2: ; %atomicrmw.end9642; VI-NEXT: s_or_b64 exec, exec, s[34:35]9643; VI-NEXT: s_setpc_b64 s[30:31]9644;9645; GFX9-LABEL: global_atomic_min_i64_noret_offset_scalar:9646; GFX9: ; %bb.0:9647; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9648; GFX9-NEXT: v_mov_b32_e32 v4, 09649; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5] offset:329650; GFX9-NEXT: s_mov_b64 s[34:35], 09651; GFX9-NEXT: v_mov_b32_e32 v5, s79652; GFX9-NEXT: v_mov_b32_e32 v6, s69653; GFX9-NEXT: .LBB122_1: ; %atomicrmw.start9654; GFX9-NEXT: ; =>This Inner Loop Header: Depth=19655; GFX9-NEXT: s_waitcnt vmcnt(0)9656; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]9657; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc9658; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc9659; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc9660; GFX9-NEXT: s_waitcnt vmcnt(0)9661; GFX9-NEXT: buffer_wbinvl1_vol9662; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]9663; GFX9-NEXT: v_mov_b32_e32 v3, v19664; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]9665; GFX9-NEXT: v_mov_b32_e32 v2, v09666; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]9667; GFX9-NEXT: s_cbranch_execnz .LBB122_19668; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end9669; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]9670; GFX9-NEXT: s_setpc_b64 s[30:31]9671 %gep = getelementptr i64, ptr addrspace(1) %out, i64 49672 %tmp0 = atomicrmw min ptr addrspace(1) %gep, i64 %in seq_cst9673 ret void9674}9675 9676define amdgpu_gfx i64 @global_atomic_min_i64_ret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {9677; SI-LABEL: global_atomic_min_i64_ret_scalar:9678; SI: ; %bb.0:9679; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9680; SI-NEXT: s_xor_saveexec_b64 s[34:35], -19681; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill9682; SI-NEXT: s_mov_b64 exec, s[34:35]9683; SI-NEXT: s_waitcnt expcnt(0)9684; SI-NEXT: v_writelane_b32 v8, s6, 09685; SI-NEXT: v_writelane_b32 v8, s7, 19686; SI-NEXT: s_mov_b32 s35, s79687; SI-NEXT: s_mov_b32 s34, s69688; SI-NEXT: s_mov_b32 s7, 0xf0009689; SI-NEXT: s_mov_b32 s6, -19690; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 09691; SI-NEXT: s_mov_b64 s[36:37], 09692; SI-NEXT: v_mov_b32_e32 v6, s359693; SI-NEXT: v_mov_b32_e32 v7, s349694; SI-NEXT: .LBB123_1: ; %atomicrmw.start9695; SI-NEXT: ; =>This Inner Loop Header: Depth=19696; SI-NEXT: s_waitcnt vmcnt(0)9697; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[34:35], v[4:5]9698; SI-NEXT: s_waitcnt expcnt(0)9699; SI-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc9700; SI-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc9701; SI-NEXT: v_mov_b32_e32 v0, v29702; SI-NEXT: v_mov_b32_e32 v1, v39703; SI-NEXT: v_mov_b32_e32 v2, v49704; SI-NEXT: v_mov_b32_e32 v3, v59705; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc9706; SI-NEXT: s_waitcnt vmcnt(0)9707; SI-NEXT: buffer_wbinvl19708; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]9709; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]9710; SI-NEXT: v_mov_b32_e32 v5, v19711; SI-NEXT: v_mov_b32_e32 v4, v09712; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]9713; SI-NEXT: s_cbranch_execnz .LBB123_19714; SI-NEXT: ; %bb.2: ; %atomicrmw.end9715; SI-NEXT: s_or_b64 exec, exec, s[36:37]9716; SI-NEXT: v_readlane_b32 s7, v8, 19717; SI-NEXT: v_readlane_b32 s6, v8, 09718; SI-NEXT: s_xor_saveexec_b64 s[34:35], -19719; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload9720; SI-NEXT: s_mov_b64 exec, s[34:35]9721; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)9722; SI-NEXT: s_setpc_b64 s[30:31]9723;9724; VI-LABEL: global_atomic_min_i64_ret_scalar:9725; VI: ; %bb.0:9726; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9727; VI-NEXT: v_mov_b32_e32 v0, s49728; VI-NEXT: v_mov_b32_e32 v1, s59729; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]9730; VI-NEXT: v_mov_b32_e32 v2, s49731; VI-NEXT: s_mov_b64 s[34:35], 09732; VI-NEXT: v_mov_b32_e32 v4, s79733; VI-NEXT: v_mov_b32_e32 v5, s69734; VI-NEXT: v_mov_b32_e32 v3, s59735; VI-NEXT: .LBB123_1: ; %atomicrmw.start9736; VI-NEXT: ; =>This Inner Loop Header: Depth=19737; VI-NEXT: s_waitcnt vmcnt(0)9738; VI-NEXT: v_mov_b32_e32 v9, v19739; VI-NEXT: v_mov_b32_e32 v8, v09740; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]9741; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc9742; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc9743; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc9744; VI-NEXT: s_waitcnt vmcnt(0)9745; VI-NEXT: buffer_wbinvl1_vol9746; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]9747; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]9748; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]9749; VI-NEXT: s_cbranch_execnz .LBB123_19750; VI-NEXT: ; %bb.2: ; %atomicrmw.end9751; VI-NEXT: s_or_b64 exec, exec, s[34:35]9752; VI-NEXT: s_setpc_b64 s[30:31]9753;9754; GFX9-LABEL: global_atomic_min_i64_ret_scalar:9755; GFX9: ; %bb.0:9756; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9757; GFX9-NEXT: v_mov_b32_e32 v2, 09758; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]9759; GFX9-NEXT: s_mov_b64 s[34:35], 09760; GFX9-NEXT: v_mov_b32_e32 v3, s79761; GFX9-NEXT: v_mov_b32_e32 v4, s69762; GFX9-NEXT: .LBB123_1: ; %atomicrmw.start9763; GFX9-NEXT: ; =>This Inner Loop Header: Depth=19764; GFX9-NEXT: s_waitcnt vmcnt(0)9765; GFX9-NEXT: v_mov_b32_e32 v8, v19766; GFX9-NEXT: v_mov_b32_e32 v7, v09767; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[7:8]9768; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc9769; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc9770; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] glc9771; GFX9-NEXT: s_waitcnt vmcnt(0)9772; GFX9-NEXT: buffer_wbinvl1_vol9773; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]9774; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]9775; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]9776; GFX9-NEXT: s_cbranch_execnz .LBB123_19777; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end9778; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]9779; GFX9-NEXT: s_setpc_b64 s[30:31]9780 %result = atomicrmw min ptr addrspace(1) %ptr, i64 %in seq_cst9781 ret i64 %result9782}9783 9784define amdgpu_gfx i64 @global_atomic_min_i64_ret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {9785; SI-LABEL: global_atomic_min_i64_ret_offset_scalar:9786; SI: ; %bb.0:9787; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9788; SI-NEXT: s_xor_saveexec_b64 s[34:35], -19789; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill9790; SI-NEXT: s_mov_b64 exec, s[34:35]9791; SI-NEXT: s_waitcnt expcnt(0)9792; SI-NEXT: v_writelane_b32 v8, s6, 09793; SI-NEXT: v_writelane_b32 v8, s7, 19794; SI-NEXT: s_mov_b32 s35, s79795; SI-NEXT: s_mov_b32 s34, s69796; SI-NEXT: s_mov_b32 s7, 0xf0009797; SI-NEXT: s_mov_b32 s6, -19798; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0 offset:329799; SI-NEXT: s_mov_b64 s[36:37], 09800; SI-NEXT: v_mov_b32_e32 v6, s359801; SI-NEXT: v_mov_b32_e32 v7, s349802; SI-NEXT: .LBB124_1: ; %atomicrmw.start9803; SI-NEXT: ; =>This Inner Loop Header: Depth=19804; SI-NEXT: s_waitcnt vmcnt(0)9805; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[34:35], v[4:5]9806; SI-NEXT: s_waitcnt expcnt(0)9807; SI-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc9808; SI-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc9809; SI-NEXT: v_mov_b32_e32 v0, v29810; SI-NEXT: v_mov_b32_e32 v1, v39811; SI-NEXT: v_mov_b32_e32 v2, v49812; SI-NEXT: v_mov_b32_e32 v3, v59813; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 offset:32 glc9814; SI-NEXT: s_waitcnt vmcnt(0)9815; SI-NEXT: buffer_wbinvl19816; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]9817; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]9818; SI-NEXT: v_mov_b32_e32 v5, v19819; SI-NEXT: v_mov_b32_e32 v4, v09820; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]9821; SI-NEXT: s_cbranch_execnz .LBB124_19822; SI-NEXT: ; %bb.2: ; %atomicrmw.end9823; SI-NEXT: s_or_b64 exec, exec, s[36:37]9824; SI-NEXT: v_readlane_b32 s7, v8, 19825; SI-NEXT: v_readlane_b32 s6, v8, 09826; SI-NEXT: s_xor_saveexec_b64 s[34:35], -19827; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload9828; SI-NEXT: s_mov_b64 exec, s[34:35]9829; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)9830; SI-NEXT: s_setpc_b64 s[30:31]9831;9832; VI-LABEL: global_atomic_min_i64_ret_offset_scalar:9833; VI: ; %bb.0:9834; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9835; VI-NEXT: s_add_u32 s34, s4, 329836; VI-NEXT: s_addc_u32 s35, s5, 09837; VI-NEXT: v_mov_b32_e32 v2, s349838; VI-NEXT: v_mov_b32_e32 v3, s359839; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]9840; VI-NEXT: s_mov_b64 s[34:35], 09841; VI-NEXT: v_mov_b32_e32 v4, s79842; VI-NEXT: v_mov_b32_e32 v5, s69843; VI-NEXT: .LBB124_1: ; %atomicrmw.start9844; VI-NEXT: ; =>This Inner Loop Header: Depth=19845; VI-NEXT: s_waitcnt vmcnt(0)9846; VI-NEXT: v_mov_b32_e32 v9, v19847; VI-NEXT: v_mov_b32_e32 v8, v09848; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]9849; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc9850; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc9851; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc9852; VI-NEXT: s_waitcnt vmcnt(0)9853; VI-NEXT: buffer_wbinvl1_vol9854; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]9855; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]9856; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]9857; VI-NEXT: s_cbranch_execnz .LBB124_19858; VI-NEXT: ; %bb.2: ; %atomicrmw.end9859; VI-NEXT: s_or_b64 exec, exec, s[34:35]9860; VI-NEXT: s_setpc_b64 s[30:31]9861;9862; GFX9-LABEL: global_atomic_min_i64_ret_offset_scalar:9863; GFX9: ; %bb.0:9864; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9865; GFX9-NEXT: v_mov_b32_e32 v2, 09866; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5] offset:329867; GFX9-NEXT: s_mov_b64 s[34:35], 09868; GFX9-NEXT: v_mov_b32_e32 v3, s79869; GFX9-NEXT: v_mov_b32_e32 v4, s69870; GFX9-NEXT: .LBB124_1: ; %atomicrmw.start9871; GFX9-NEXT: ; =>This Inner Loop Header: Depth=19872; GFX9-NEXT: s_waitcnt vmcnt(0)9873; GFX9-NEXT: v_mov_b32_e32 v8, v19874; GFX9-NEXT: v_mov_b32_e32 v7, v09875; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[7:8]9876; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc9877; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc9878; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] offset:32 glc9879; GFX9-NEXT: s_waitcnt vmcnt(0)9880; GFX9-NEXT: buffer_wbinvl1_vol9881; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]9882; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]9883; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]9884; GFX9-NEXT: s_cbranch_execnz .LBB124_19885; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end9886; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]9887; GFX9-NEXT: s_setpc_b64 s[30:31]9888 %gep = getelementptr i64, ptr addrspace(1) %out, i64 49889 %result = atomicrmw min ptr addrspace(1) %gep, i64 %in seq_cst9890 ret i64 %result9891}9892 9893define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr addrspace(1) %out, i64 %in, i64 %index) {9894; SI-LABEL: atomic_min_i64_addr64_offset:9895; SI: ; %bb.0: ; %entry9896; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd9897; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x99898; SI-NEXT: s_waitcnt lgkmcnt(0)9899; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 39900; SI-NEXT: s_add_u32 s4, s0, s49901; SI-NEXT: s_addc_u32 s5, s1, s59902; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x89903; SI-NEXT: s_mov_b64 s[0:1], 09904; SI-NEXT: s_mov_b32 s7, 0xf0009905; SI-NEXT: v_mov_b32_e32 v4, s39906; SI-NEXT: v_mov_b32_e32 v5, s29907; SI-NEXT: s_waitcnt lgkmcnt(0)9908; SI-NEXT: v_mov_b32_e32 v2, s89909; SI-NEXT: v_mov_b32_e32 v3, s99910; SI-NEXT: s_mov_b32 s6, -19911; SI-NEXT: .LBB125_1: ; %atomicrmw.start9912; SI-NEXT: ; =>This Inner Loop Header: Depth=19913; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]9914; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc9915; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc9916; SI-NEXT: s_waitcnt expcnt(0)9917; SI-NEXT: v_mov_b32_e32 v9, v39918; SI-NEXT: v_mov_b32_e32 v8, v29919; SI-NEXT: v_mov_b32_e32 v7, v19920; SI-NEXT: v_mov_b32_e32 v6, v09921; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 offset:32 glc9922; SI-NEXT: s_waitcnt vmcnt(0)9923; SI-NEXT: buffer_wbinvl19924; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]9925; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]9926; SI-NEXT: v_mov_b32_e32 v2, v69927; SI-NEXT: v_mov_b32_e32 v3, v79928; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]9929; SI-NEXT: s_cbranch_execnz .LBB125_19930; SI-NEXT: ; %bb.2: ; %atomicrmw.end9931; SI-NEXT: s_endpgm9932;9933; VI-LABEL: atomic_min_i64_addr64_offset:9934; VI: ; %bb.0: ; %entry9935; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x349936; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249937; VI-NEXT: s_mov_b64 s[4:5], 09938; VI-NEXT: s_waitcnt lgkmcnt(0)9939; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 39940; VI-NEXT: s_add_u32 s0, s0, s69941; VI-NEXT: s_addc_u32 s1, s1, s79942; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x209943; VI-NEXT: s_add_u32 s0, s0, 329944; VI-NEXT: s_addc_u32 s1, s1, 09945; VI-NEXT: v_mov_b32_e32 v5, s19946; VI-NEXT: v_mov_b32_e32 v6, s39947; VI-NEXT: s_waitcnt lgkmcnt(0)9948; VI-NEXT: v_mov_b32_e32 v2, s69949; VI-NEXT: v_mov_b32_e32 v7, s29950; VI-NEXT: v_mov_b32_e32 v3, s79951; VI-NEXT: v_mov_b32_e32 v4, s09952; VI-NEXT: .LBB125_1: ; %atomicrmw.start9953; VI-NEXT: ; =>This Inner Loop Header: Depth=19954; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]9955; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc9956; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc9957; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc9958; VI-NEXT: s_waitcnt vmcnt(0)9959; VI-NEXT: buffer_wbinvl1_vol9960; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]9961; VI-NEXT: v_mov_b32_e32 v3, v19962; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9963; VI-NEXT: v_mov_b32_e32 v2, v09964; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]9965; VI-NEXT: s_cbranch_execnz .LBB125_19966; VI-NEXT: ; %bb.2: ; %atomicrmw.end9967; VI-NEXT: s_endpgm9968;9969; GFX9-LABEL: atomic_min_i64_addr64_offset:9970; GFX9: ; %bb.0: ; %entry9971; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x349972; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249973; GFX9-NEXT: v_mov_b32_e32 v6, 09974; GFX9-NEXT: s_waitcnt lgkmcnt(0)9975; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 39976; GFX9-NEXT: s_add_u32 s0, s0, s49977; GFX9-NEXT: s_addc_u32 s1, s1, s59978; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x209979; GFX9-NEXT: s_mov_b64 s[4:5], 09980; GFX9-NEXT: v_mov_b32_e32 v4, s39981; GFX9-NEXT: v_mov_b32_e32 v5, s29982; GFX9-NEXT: s_waitcnt lgkmcnt(0)9983; GFX9-NEXT: v_mov_b32_e32 v2, s69984; GFX9-NEXT: v_mov_b32_e32 v3, s79985; GFX9-NEXT: .LBB125_1: ; %atomicrmw.start9986; GFX9-NEXT: ; =>This Inner Loop Header: Depth=19987; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]9988; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc9989; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc9990; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] offset:32 glc9991; GFX9-NEXT: s_waitcnt vmcnt(0)9992; GFX9-NEXT: buffer_wbinvl1_vol9993; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]9994; GFX9-NEXT: v_mov_b32_e32 v3, v19995; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9996; GFX9-NEXT: v_mov_b32_e32 v2, v09997; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]9998; GFX9-NEXT: s_cbranch_execnz .LBB125_19999; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end10000; GFX9-NEXT: s_endpgm10001entry:10002 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index10003 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 410004 %tmp0 = atomicrmw min ptr addrspace(1) %gep, i64 %in seq_cst10005 ret void10006}10007 10008define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {10009; SI-LABEL: atomic_min_i64_ret_addr64_offset:10010; SI: ; %bb.0: ; %entry10011; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x910012; SI-NEXT: s_waitcnt lgkmcnt(0)10013; SI-NEXT: s_lshl_b64 s[6:7], s[6:7], 310014; SI-NEXT: s_add_u32 s8, s0, s610015; SI-NEXT: s_addc_u32 s9, s1, s710016; SI-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x810017; SI-NEXT: s_mov_b64 s[0:1], 010018; SI-NEXT: s_mov_b32 s11, 0xf00010019; SI-NEXT: v_mov_b32_e32 v8, s510020; SI-NEXT: v_mov_b32_e32 v9, s410021; SI-NEXT: s_waitcnt lgkmcnt(0)10022; SI-NEXT: v_mov_b32_e32 v2, s610023; SI-NEXT: v_mov_b32_e32 v3, s710024; SI-NEXT: s_mov_b32 s10, -110025; SI-NEXT: .LBB126_1: ; %atomicrmw.start10026; SI-NEXT: ; =>This Inner Loop Header: Depth=110027; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]10028; SI-NEXT: v_cndmask_b32_e32 v1, v8, v3, vcc10029; SI-NEXT: v_cndmask_b32_e32 v0, v9, v2, vcc10030; SI-NEXT: s_waitcnt expcnt(0)10031; SI-NEXT: v_mov_b32_e32 v7, v310032; SI-NEXT: v_mov_b32_e32 v6, v210033; SI-NEXT: v_mov_b32_e32 v5, v110034; SI-NEXT: v_mov_b32_e32 v4, v010035; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[8:11], 0 offset:32 glc10036; SI-NEXT: s_waitcnt vmcnt(0)10037; SI-NEXT: buffer_wbinvl110038; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]10039; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]10040; SI-NEXT: v_mov_b32_e32 v2, v410041; SI-NEXT: v_mov_b32_e32 v3, v510042; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]10043; SI-NEXT: s_cbranch_execnz .LBB126_110044; SI-NEXT: ; %bb.2: ; %atomicrmw.end10045; SI-NEXT: s_or_b64 exec, exec, s[0:1]10046; SI-NEXT: s_mov_b32 s7, 0xf00010047; SI-NEXT: s_mov_b32 s6, -110048; SI-NEXT: s_mov_b32 s4, s210049; SI-NEXT: s_mov_b32 s5, s310050; SI-NEXT: buffer_store_dwordx2 v[4:5], off, s[4:7], 010051; SI-NEXT: s_endpgm10052;10053; VI-LABEL: atomic_min_i64_ret_addr64_offset:10054; VI: ; %bb.0: ; %entry10055; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x2410056; VI-NEXT: s_mov_b64 s[8:9], 010057; VI-NEXT: s_waitcnt lgkmcnt(0)10058; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 310059; VI-NEXT: s_add_u32 s0, s0, s610060; VI-NEXT: s_addc_u32 s1, s1, s710061; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x2010062; VI-NEXT: s_add_u32 s0, s0, 3210063; VI-NEXT: s_addc_u32 s1, s1, 010064; VI-NEXT: v_mov_b32_e32 v0, s010065; VI-NEXT: v_mov_b32_e32 v4, s510066; VI-NEXT: s_waitcnt lgkmcnt(0)10067; VI-NEXT: v_mov_b32_e32 v2, s610068; VI-NEXT: v_mov_b32_e32 v5, s410069; VI-NEXT: v_mov_b32_e32 v3, s710070; VI-NEXT: v_mov_b32_e32 v1, s110071; VI-NEXT: .LBB126_1: ; %atomicrmw.start10072; VI-NEXT: ; =>This Inner Loop Header: Depth=110073; VI-NEXT: v_mov_b32_e32 v9, v310074; VI-NEXT: v_mov_b32_e32 v8, v210075; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]10076; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc10077; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc10078; VI-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc10079; VI-NEXT: s_waitcnt vmcnt(0)10080; VI-NEXT: buffer_wbinvl1_vol10081; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]10082; VI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]10083; VI-NEXT: s_andn2_b64 exec, exec, s[8:9]10084; VI-NEXT: s_cbranch_execnz .LBB126_110085; VI-NEXT: ; %bb.2: ; %atomicrmw.end10086; VI-NEXT: s_or_b64 exec, exec, s[8:9]10087; VI-NEXT: v_mov_b32_e32 v0, s210088; VI-NEXT: v_mov_b32_e32 v1, s310089; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]10090; VI-NEXT: s_endpgm10091;10092; GFX9-LABEL: atomic_min_i64_ret_addr64_offset:10093; GFX9: ; %bb.0: ; %entry10094; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2410095; GFX9-NEXT: s_mov_b64 s[2:3], 010096; GFX9-NEXT: v_mov_b32_e32 v4, 010097; GFX9-NEXT: s_waitcnt lgkmcnt(0)10098; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 310099; GFX9-NEXT: s_add_u32 s0, s8, s010100; GFX9-NEXT: s_addc_u32 s1, s9, s110101; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x2010102; GFX9-NEXT: v_mov_b32_e32 v2, s1310103; GFX9-NEXT: v_mov_b32_e32 v3, s1210104; GFX9-NEXT: s_waitcnt lgkmcnt(0)10105; GFX9-NEXT: v_mov_b32_e32 v0, s410106; GFX9-NEXT: v_mov_b32_e32 v1, s510107; GFX9-NEXT: .LBB126_1: ; %atomicrmw.start10108; GFX9-NEXT: ; =>This Inner Loop Header: Depth=110109; GFX9-NEXT: v_mov_b32_e32 v8, v110110; GFX9-NEXT: v_mov_b32_e32 v7, v010111; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[7:8]10112; GFX9-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc10113; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc10114; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[5:8], s[0:1] offset:32 glc10115; GFX9-NEXT: s_waitcnt vmcnt(0)10116; GFX9-NEXT: buffer_wbinvl1_vol10117; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]10118; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]10119; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]10120; GFX9-NEXT: s_cbranch_execnz .LBB126_110121; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end10122; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]10123; GFX9-NEXT: v_mov_b32_e32 v2, 010124; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]10125; GFX9-NEXT: s_endpgm10126entry:10127 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index10128 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 410129 %tmp0 = atomicrmw min ptr addrspace(1) %gep, i64 %in seq_cst10130 store i64 %tmp0, ptr addrspace(1) %out210131 ret void10132}10133 10134define amdgpu_kernel void @atomic_min_i64(ptr addrspace(1) %out, i64 %in) {10135; SI-LABEL: atomic_min_i64:10136; SI: ; %bb.0: ; %entry10137; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x910138; SI-NEXT: s_waitcnt lgkmcnt(0)10139; SI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x010140; SI-NEXT: s_mov_b64 s[8:9], 010141; SI-NEXT: s_mov_b32 s7, 0xf00010142; SI-NEXT: v_mov_b32_e32 v4, s310143; SI-NEXT: v_mov_b32_e32 v5, s210144; SI-NEXT: s_waitcnt lgkmcnt(0)10145; SI-NEXT: v_mov_b32_e32 v2, s410146; SI-NEXT: v_mov_b32_e32 v3, s510147; SI-NEXT: s_mov_b32 s6, -110148; SI-NEXT: s_mov_b32 s4, s010149; SI-NEXT: s_mov_b32 s5, s110150; SI-NEXT: .LBB127_1: ; %atomicrmw.start10151; SI-NEXT: ; =>This Inner Loop Header: Depth=110152; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]10153; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc10154; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc10155; SI-NEXT: s_waitcnt expcnt(0)10156; SI-NEXT: v_mov_b32_e32 v9, v310157; SI-NEXT: v_mov_b32_e32 v8, v210158; SI-NEXT: v_mov_b32_e32 v7, v110159; SI-NEXT: v_mov_b32_e32 v6, v010160; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 glc10161; SI-NEXT: s_waitcnt vmcnt(0)10162; SI-NEXT: buffer_wbinvl110163; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]10164; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]10165; SI-NEXT: v_mov_b32_e32 v2, v610166; SI-NEXT: v_mov_b32_e32 v3, v710167; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]10168; SI-NEXT: s_cbranch_execnz .LBB127_110169; SI-NEXT: ; %bb.2: ; %atomicrmw.end10170; SI-NEXT: s_endpgm10171;10172; VI-LABEL: atomic_min_i64:10173; VI: ; %bb.0: ; %entry10174; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410175; VI-NEXT: s_mov_b64 s[4:5], 010176; VI-NEXT: s_waitcnt lgkmcnt(0)10177; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x010178; VI-NEXT: v_mov_b32_e32 v5, s110179; VI-NEXT: v_mov_b32_e32 v6, s310180; VI-NEXT: v_mov_b32_e32 v7, s210181; VI-NEXT: v_mov_b32_e32 v4, s010182; VI-NEXT: s_waitcnt lgkmcnt(0)10183; VI-NEXT: v_mov_b32_e32 v2, s610184; VI-NEXT: v_mov_b32_e32 v3, s710185; VI-NEXT: .LBB127_1: ; %atomicrmw.start10186; VI-NEXT: ; =>This Inner Loop Header: Depth=110187; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]10188; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc10189; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc10190; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc10191; VI-NEXT: s_waitcnt vmcnt(0)10192; VI-NEXT: buffer_wbinvl1_vol10193; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]10194; VI-NEXT: v_mov_b32_e32 v3, v110195; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]10196; VI-NEXT: v_mov_b32_e32 v2, v010197; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]10198; VI-NEXT: s_cbranch_execnz .LBB127_110199; VI-NEXT: ; %bb.2: ; %atomicrmw.end10200; VI-NEXT: s_endpgm10201;10202; GFX9-LABEL: atomic_min_i64:10203; GFX9: ; %bb.0: ; %entry10204; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410205; GFX9-NEXT: s_mov_b64 s[4:5], 010206; GFX9-NEXT: v_mov_b32_e32 v6, 010207; GFX9-NEXT: s_waitcnt lgkmcnt(0)10208; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x010209; GFX9-NEXT: v_mov_b32_e32 v4, s310210; GFX9-NEXT: v_mov_b32_e32 v5, s210211; GFX9-NEXT: s_waitcnt lgkmcnt(0)10212; GFX9-NEXT: v_mov_b32_e32 v2, s610213; GFX9-NEXT: v_mov_b32_e32 v3, s710214; GFX9-NEXT: .LBB127_1: ; %atomicrmw.start10215; GFX9-NEXT: ; =>This Inner Loop Header: Depth=110216; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]10217; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc10218; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc10219; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc10220; GFX9-NEXT: s_waitcnt vmcnt(0)10221; GFX9-NEXT: buffer_wbinvl1_vol10222; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]10223; GFX9-NEXT: v_mov_b32_e32 v3, v110224; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]10225; GFX9-NEXT: v_mov_b32_e32 v2, v010226; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]10227; GFX9-NEXT: s_cbranch_execnz .LBB127_110228; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end10229; GFX9-NEXT: s_endpgm10230entry:10231 %tmp0 = atomicrmw min ptr addrspace(1) %out, i64 %in seq_cst10232 ret void10233}10234 10235define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {10236; SI-LABEL: atomic_min_i64_ret_addr64:10237; SI: ; %bb.0: ; %entry10238; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x910239; SI-NEXT: s_waitcnt lgkmcnt(0)10240; SI-NEXT: s_lshl_b64 s[6:7], s[6:7], 310241; SI-NEXT: s_add_u32 s8, s0, s610242; SI-NEXT: s_addc_u32 s9, s1, s710243; SI-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x010244; SI-NEXT: s_mov_b64 s[0:1], 010245; SI-NEXT: s_mov_b32 s11, 0xf00010246; SI-NEXT: v_mov_b32_e32 v8, s510247; SI-NEXT: v_mov_b32_e32 v9, s410248; SI-NEXT: s_waitcnt lgkmcnt(0)10249; SI-NEXT: v_mov_b32_e32 v2, s610250; SI-NEXT: v_mov_b32_e32 v3, s710251; SI-NEXT: s_mov_b32 s10, -110252; SI-NEXT: .LBB128_1: ; %atomicrmw.start10253; SI-NEXT: ; =>This Inner Loop Header: Depth=110254; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]10255; SI-NEXT: v_cndmask_b32_e32 v1, v8, v3, vcc10256; SI-NEXT: v_cndmask_b32_e32 v0, v9, v2, vcc10257; SI-NEXT: s_waitcnt expcnt(0)10258; SI-NEXT: v_mov_b32_e32 v7, v310259; SI-NEXT: v_mov_b32_e32 v6, v210260; SI-NEXT: v_mov_b32_e32 v5, v110261; SI-NEXT: v_mov_b32_e32 v4, v010262; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[8:11], 0 glc10263; SI-NEXT: s_waitcnt vmcnt(0)10264; SI-NEXT: buffer_wbinvl110265; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]10266; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]10267; SI-NEXT: v_mov_b32_e32 v2, v410268; SI-NEXT: v_mov_b32_e32 v3, v510269; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]10270; SI-NEXT: s_cbranch_execnz .LBB128_110271; SI-NEXT: ; %bb.2: ; %atomicrmw.end10272; SI-NEXT: s_or_b64 exec, exec, s[0:1]10273; SI-NEXT: s_mov_b32 s7, 0xf00010274; SI-NEXT: s_mov_b32 s6, -110275; SI-NEXT: s_mov_b32 s4, s210276; SI-NEXT: s_mov_b32 s5, s310277; SI-NEXT: buffer_store_dwordx2 v[4:5], off, s[4:7], 010278; SI-NEXT: s_endpgm10279;10280; VI-LABEL: atomic_min_i64_ret_addr64:10281; VI: ; %bb.0: ; %entry10282; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x2410283; VI-NEXT: s_waitcnt lgkmcnt(0)10284; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 310285; VI-NEXT: s_add_u32 s6, s0, s610286; VI-NEXT: s_addc_u32 s7, s1, s710287; VI-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x010288; VI-NEXT: v_mov_b32_e32 v0, s610289; VI-NEXT: s_mov_b64 s[0:1], 010290; VI-NEXT: v_mov_b32_e32 v4, s510291; VI-NEXT: v_mov_b32_e32 v5, s410292; VI-NEXT: s_waitcnt lgkmcnt(0)10293; VI-NEXT: v_mov_b32_e32 v2, s810294; VI-NEXT: v_mov_b32_e32 v3, s910295; VI-NEXT: v_mov_b32_e32 v1, s710296; VI-NEXT: .LBB128_1: ; %atomicrmw.start10297; VI-NEXT: ; =>This Inner Loop Header: Depth=110298; VI-NEXT: v_mov_b32_e32 v9, v310299; VI-NEXT: v_mov_b32_e32 v8, v210300; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]10301; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc10302; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc10303; VI-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc10304; VI-NEXT: s_waitcnt vmcnt(0)10305; VI-NEXT: buffer_wbinvl1_vol10306; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]10307; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]10308; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]10309; VI-NEXT: s_cbranch_execnz .LBB128_110310; VI-NEXT: ; %bb.2: ; %atomicrmw.end10311; VI-NEXT: s_or_b64 exec, exec, s[0:1]10312; VI-NEXT: v_mov_b32_e32 v0, s210313; VI-NEXT: v_mov_b32_e32 v1, s310314; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]10315; VI-NEXT: s_endpgm10316;10317; GFX9-LABEL: atomic_min_i64_ret_addr64:10318; GFX9: ; %bb.0: ; %entry10319; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2410320; GFX9-NEXT: s_mov_b64 s[2:3], 010321; GFX9-NEXT: v_mov_b32_e32 v4, 010322; GFX9-NEXT: s_waitcnt lgkmcnt(0)10323; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 310324; GFX9-NEXT: s_add_u32 s0, s8, s010325; GFX9-NEXT: s_addc_u32 s1, s9, s110326; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x010327; GFX9-NEXT: v_mov_b32_e32 v2, s1310328; GFX9-NEXT: v_mov_b32_e32 v3, s1210329; GFX9-NEXT: s_waitcnt lgkmcnt(0)10330; GFX9-NEXT: v_mov_b32_e32 v0, s410331; GFX9-NEXT: v_mov_b32_e32 v1, s510332; GFX9-NEXT: .LBB128_1: ; %atomicrmw.start10333; GFX9-NEXT: ; =>This Inner Loop Header: Depth=110334; GFX9-NEXT: v_mov_b32_e32 v8, v110335; GFX9-NEXT: v_mov_b32_e32 v7, v010336; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[7:8]10337; GFX9-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc10338; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc10339; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[5:8], s[0:1] glc10340; GFX9-NEXT: s_waitcnt vmcnt(0)10341; GFX9-NEXT: buffer_wbinvl1_vol10342; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]10343; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]10344; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]10345; GFX9-NEXT: s_cbranch_execnz .LBB128_110346; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end10347; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]10348; GFX9-NEXT: v_mov_b32_e32 v2, 010349; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]10350; GFX9-NEXT: s_endpgm10351entry:10352 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index10353 %tmp0 = atomicrmw min ptr addrspace(1) %ptr, i64 %in seq_cst10354 store i64 %tmp0, ptr addrspace(1) %out210355 ret void10356}10357 10358define void @global_atomic_min_i64_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {10359; SI-LABEL: global_atomic_min_i64_noret_offset__amdgpu_no_remote_memory:10360; SI: ; %bb.0:10361; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10362; SI-NEXT: s_mov_b32 s6, 010363; SI-NEXT: s_mov_b32 s7, 0xf00010364; SI-NEXT: s_mov_b32 s4, s610365; SI-NEXT: s_mov_b32 s5, s610366; SI-NEXT: buffer_atomic_smin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:3210367; SI-NEXT: s_waitcnt vmcnt(0)10368; SI-NEXT: buffer_wbinvl110369; SI-NEXT: s_waitcnt expcnt(0)10370; SI-NEXT: s_setpc_b64 s[30:31]10371;10372; VI-LABEL: global_atomic_min_i64_noret_offset__amdgpu_no_remote_memory:10373; VI: ; %bb.0:10374; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10375; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v010376; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc10377; VI-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3]10378; VI-NEXT: s_waitcnt vmcnt(0)10379; VI-NEXT: buffer_wbinvl1_vol10380; VI-NEXT: s_setpc_b64 s[30:31]10381;10382; GFX9-LABEL: global_atomic_min_i64_noret_offset__amdgpu_no_remote_memory:10383; GFX9: ; %bb.0:10384; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10385; GFX9-NEXT: global_atomic_smin_x2 v[0:1], v[2:3], off offset:3210386; GFX9-NEXT: s_waitcnt vmcnt(0)10387; GFX9-NEXT: buffer_wbinvl1_vol10388; GFX9-NEXT: s_setpc_b64 s[30:31]10389 %gep = getelementptr i64, ptr addrspace(1) %out, i64 410390 %tmp0 = atomicrmw min ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !010391 ret void10392}10393 10394define i64 @global_atomic_min_i64_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {10395; SI-LABEL: global_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:10396; SI: ; %bb.0:10397; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10398; SI-NEXT: s_mov_b32 s6, 010399; SI-NEXT: s_mov_b32 s7, 0xf00010400; SI-NEXT: s_mov_b32 s4, s610401; SI-NEXT: s_mov_b32 s5, s610402; SI-NEXT: buffer_atomic_smin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc10403; SI-NEXT: s_waitcnt vmcnt(0)10404; SI-NEXT: buffer_wbinvl110405; SI-NEXT: v_mov_b32_e32 v0, v210406; SI-NEXT: v_mov_b32_e32 v1, v310407; SI-NEXT: s_waitcnt expcnt(0)10408; SI-NEXT: s_setpc_b64 s[30:31]10409;10410; VI-LABEL: global_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:10411; VI: ; %bb.0:10412; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10413; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v010414; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc10415; VI-NEXT: flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc10416; VI-NEXT: s_waitcnt vmcnt(0)10417; VI-NEXT: buffer_wbinvl1_vol10418; VI-NEXT: s_setpc_b64 s[30:31]10419;10420; GFX9-LABEL: global_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:10421; GFX9: ; %bb.0:10422; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10423; GFX9-NEXT: global_atomic_smin_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc10424; GFX9-NEXT: s_waitcnt vmcnt(0)10425; GFX9-NEXT: buffer_wbinvl1_vol10426; GFX9-NEXT: s_setpc_b64 s[30:31]10427 %gep = getelementptr i64, ptr addrspace(1) %out, i64 410428 %result = atomicrmw min ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !010429 ret i64 %result10430}10431 10432; ---------------------------------------------------------------------10433; atomicrmw uinc_wrap10434; ---------------------------------------------------------------------10435 10436define void @global_atomic_uinc_wrap_i64_noret(ptr addrspace(1) %ptr, i64 %in) {10437; SI-LABEL: global_atomic_uinc_wrap_i64_noret:10438; SI: ; %bb.0:10439; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10440; SI-NEXT: s_mov_b32 s6, 010441; SI-NEXT: s_mov_b32 s7, 0xf00010442; SI-NEXT: s_mov_b32 s4, s610443; SI-NEXT: s_mov_b32 s5, s610444; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr6410445; SI-NEXT: s_mov_b64 s[8:9], 010446; SI-NEXT: .LBB131_1: ; %atomicrmw.start10447; SI-NEXT: ; =>This Inner Loop Header: Depth=110448; SI-NEXT: s_waitcnt vmcnt(0)10449; SI-NEXT: v_add_i32_e32 v4, vcc, 1, v610450; SI-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc10451; SI-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]10452; SI-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc10453; SI-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc10454; SI-NEXT: s_waitcnt expcnt(0)10455; SI-NEXT: v_mov_b32_e32 v11, v710456; SI-NEXT: v_mov_b32_e32 v10, v610457; SI-NEXT: v_mov_b32_e32 v9, v510458; SI-NEXT: v_mov_b32_e32 v8, v410459; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc10460; SI-NEXT: s_waitcnt vmcnt(0)10461; SI-NEXT: buffer_wbinvl110462; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]10463; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]10464; SI-NEXT: v_mov_b32_e32 v6, v810465; SI-NEXT: v_mov_b32_e32 v7, v910466; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]10467; SI-NEXT: s_cbranch_execnz .LBB131_110468; SI-NEXT: ; %bb.2: ; %atomicrmw.end10469; SI-NEXT: s_or_b64 exec, exec, s[8:9]10470; SI-NEXT: s_waitcnt expcnt(0)10471; SI-NEXT: s_setpc_b64 s[30:31]10472;10473; VI-LABEL: global_atomic_uinc_wrap_i64_noret:10474; VI: ; %bb.0:10475; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10476; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]10477; VI-NEXT: s_mov_b64 s[4:5], 010478; VI-NEXT: .LBB131_1: ; %atomicrmw.start10479; VI-NEXT: ; =>This Inner Loop Header: Depth=110480; VI-NEXT: s_waitcnt vmcnt(0)10481; VI-NEXT: v_add_u32_e32 v4, vcc, 1, v610482; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc10483; VI-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]10484; VI-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc10485; VI-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc10486; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc10487; VI-NEXT: s_waitcnt vmcnt(0)10488; VI-NEXT: buffer_wbinvl1_vol10489; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]10490; VI-NEXT: v_mov_b32_e32 v7, v510491; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]10492; VI-NEXT: v_mov_b32_e32 v6, v410493; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]10494; VI-NEXT: s_cbranch_execnz .LBB131_110495; VI-NEXT: ; %bb.2: ; %atomicrmw.end10496; VI-NEXT: s_or_b64 exec, exec, s[4:5]10497; VI-NEXT: s_setpc_b64 s[30:31]10498;10499; GFX9-LABEL: global_atomic_uinc_wrap_i64_noret:10500; GFX9: ; %bb.0:10501; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10502; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off10503; GFX9-NEXT: s_mov_b64 s[4:5], 010504; GFX9-NEXT: .LBB131_1: ; %atomicrmw.start10505; GFX9-NEXT: ; =>This Inner Loop Header: Depth=110506; GFX9-NEXT: s_waitcnt vmcnt(0)10507; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, 1, v610508; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v7, vcc10509; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]10510; GFX9-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc10511; GFX9-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc10512; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc10513; GFX9-NEXT: s_waitcnt vmcnt(0)10514; GFX9-NEXT: buffer_wbinvl1_vol10515; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]10516; GFX9-NEXT: v_mov_b32_e32 v7, v510517; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]10518; GFX9-NEXT: v_mov_b32_e32 v6, v410519; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]10520; GFX9-NEXT: s_cbranch_execnz .LBB131_110521; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end10522; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]10523; GFX9-NEXT: s_setpc_b64 s[30:31]10524 %tmp0 = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 %in seq_cst10525 ret void10526}10527 10528define void @global_atomic_uinc_wrap_i64_noret_offset(ptr addrspace(1) %out, i64 %in) {10529; SI-LABEL: global_atomic_uinc_wrap_i64_noret_offset:10530; SI: ; %bb.0:10531; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10532; SI-NEXT: s_mov_b32 s6, 010533; SI-NEXT: s_mov_b32 s7, 0xf00010534; SI-NEXT: s_mov_b32 s4, s610535; SI-NEXT: s_mov_b32 s5, s610536; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:3210537; SI-NEXT: s_mov_b64 s[8:9], 010538; SI-NEXT: .LBB132_1: ; %atomicrmw.start10539; SI-NEXT: ; =>This Inner Loop Header: Depth=110540; SI-NEXT: s_waitcnt vmcnt(0)10541; SI-NEXT: v_add_i32_e32 v4, vcc, 1, v610542; SI-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc10543; SI-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]10544; SI-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc10545; SI-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc10546; SI-NEXT: s_waitcnt expcnt(0)10547; SI-NEXT: v_mov_b32_e32 v11, v710548; SI-NEXT: v_mov_b32_e32 v10, v610549; SI-NEXT: v_mov_b32_e32 v9, v510550; SI-NEXT: v_mov_b32_e32 v8, v410551; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32 glc10552; SI-NEXT: s_waitcnt vmcnt(0)10553; SI-NEXT: buffer_wbinvl110554; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]10555; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]10556; SI-NEXT: v_mov_b32_e32 v6, v810557; SI-NEXT: v_mov_b32_e32 v7, v910558; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]10559; SI-NEXT: s_cbranch_execnz .LBB132_110560; SI-NEXT: ; %bb.2: ; %atomicrmw.end10561; SI-NEXT: s_or_b64 exec, exec, s[8:9]10562; SI-NEXT: s_waitcnt expcnt(0)10563; SI-NEXT: s_setpc_b64 s[30:31]10564;10565; VI-LABEL: global_atomic_uinc_wrap_i64_noret_offset:10566; VI: ; %bb.0:10567; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10568; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v010569; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc10570; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]10571; VI-NEXT: s_mov_b64 s[4:5], 010572; VI-NEXT: .LBB132_1: ; %atomicrmw.start10573; VI-NEXT: ; =>This Inner Loop Header: Depth=110574; VI-NEXT: s_waitcnt vmcnt(0)10575; VI-NEXT: v_add_u32_e32 v4, vcc, 1, v610576; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc10577; VI-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]10578; VI-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc10579; VI-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc10580; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc10581; VI-NEXT: s_waitcnt vmcnt(0)10582; VI-NEXT: buffer_wbinvl1_vol10583; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]10584; VI-NEXT: v_mov_b32_e32 v7, v510585; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]10586; VI-NEXT: v_mov_b32_e32 v6, v410587; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]10588; VI-NEXT: s_cbranch_execnz .LBB132_110589; VI-NEXT: ; %bb.2: ; %atomicrmw.end10590; VI-NEXT: s_or_b64 exec, exec, s[4:5]10591; VI-NEXT: s_setpc_b64 s[30:31]10592;10593; GFX9-LABEL: global_atomic_uinc_wrap_i64_noret_offset:10594; GFX9: ; %bb.0:10595; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10596; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:3210597; GFX9-NEXT: s_mov_b64 s[4:5], 010598; GFX9-NEXT: .LBB132_1: ; %atomicrmw.start10599; GFX9-NEXT: ; =>This Inner Loop Header: Depth=110600; GFX9-NEXT: s_waitcnt vmcnt(0)10601; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, 1, v610602; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v7, vcc10603; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]10604; GFX9-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc10605; GFX9-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc10606; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc10607; GFX9-NEXT: s_waitcnt vmcnt(0)10608; GFX9-NEXT: buffer_wbinvl1_vol10609; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]10610; GFX9-NEXT: v_mov_b32_e32 v7, v510611; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]10612; GFX9-NEXT: v_mov_b32_e32 v6, v410613; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]10614; GFX9-NEXT: s_cbranch_execnz .LBB132_110615; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end10616; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]10617; GFX9-NEXT: s_setpc_b64 s[30:31]10618 %gep = getelementptr i64, ptr addrspace(1) %out, i64 410619 %tmp0 = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 %in seq_cst10620 ret void10621}10622 10623define i64 @global_atomic_uinc_wrap_i64_ret(ptr addrspace(1) %ptr, i64 %in) {10624; SI-LABEL: global_atomic_uinc_wrap_i64_ret:10625; SI: ; %bb.0:10626; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10627; SI-NEXT: v_mov_b32_e32 v5, v310628; SI-NEXT: v_mov_b32_e32 v4, v210629; SI-NEXT: v_mov_b32_e32 v7, v110630; SI-NEXT: v_mov_b32_e32 v6, v010631; SI-NEXT: s_mov_b32 s6, 010632; SI-NEXT: s_mov_b32 s7, 0xf00010633; SI-NEXT: s_mov_b32 s4, s610634; SI-NEXT: s_mov_b32 s5, s610635; SI-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[4:7], 0 addr6410636; SI-NEXT: s_mov_b64 s[8:9], 010637; SI-NEXT: .LBB133_1: ; %atomicrmw.start10638; SI-NEXT: ; =>This Inner Loop Header: Depth=110639; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)10640; SI-NEXT: v_add_i32_e32 v0, vcc, 1, v1010641; SI-NEXT: v_addc_u32_e32 v1, vcc, 0, v11, vcc10642; SI-NEXT: v_cmp_lt_u64_e32 vcc, v[10:11], v[4:5]10643; SI-NEXT: v_cndmask_b32_e32 v9, 0, v1, vcc10644; SI-NEXT: v_cndmask_b32_e32 v8, 0, v0, vcc10645; SI-NEXT: v_mov_b32_e32 v0, v810646; SI-NEXT: v_mov_b32_e32 v1, v910647; SI-NEXT: v_mov_b32_e32 v2, v1010648; SI-NEXT: v_mov_b32_e32 v3, v1110649; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc10650; SI-NEXT: s_waitcnt vmcnt(0)10651; SI-NEXT: buffer_wbinvl110652; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]10653; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]10654; SI-NEXT: v_mov_b32_e32 v11, v110655; SI-NEXT: v_mov_b32_e32 v10, v010656; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]10657; SI-NEXT: s_cbranch_execnz .LBB133_110658; SI-NEXT: ; %bb.2: ; %atomicrmw.end10659; SI-NEXT: s_or_b64 exec, exec, s[8:9]10660; SI-NEXT: s_waitcnt expcnt(0)10661; SI-NEXT: s_setpc_b64 s[30:31]10662;10663; VI-LABEL: global_atomic_uinc_wrap_i64_ret:10664; VI: ; %bb.0:10665; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10666; VI-NEXT: flat_load_dwordx2 v[4:5], v[0:1]10667; VI-NEXT: s_mov_b64 s[4:5], 010668; VI-NEXT: .LBB133_1: ; %atomicrmw.start10669; VI-NEXT: ; =>This Inner Loop Header: Depth=110670; VI-NEXT: s_waitcnt vmcnt(0)10671; VI-NEXT: v_mov_b32_e32 v7, v510672; VI-NEXT: v_mov_b32_e32 v6, v410673; VI-NEXT: v_add_u32_e32 v4, vcc, 1, v610674; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc10675; VI-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]10676; VI-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc10677; VI-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc10678; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc10679; VI-NEXT: s_waitcnt vmcnt(0)10680; VI-NEXT: buffer_wbinvl1_vol10681; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]10682; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]10683; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]10684; VI-NEXT: s_cbranch_execnz .LBB133_110685; VI-NEXT: ; %bb.2: ; %atomicrmw.end10686; VI-NEXT: s_or_b64 exec, exec, s[4:5]10687; VI-NEXT: v_mov_b32_e32 v0, v410688; VI-NEXT: v_mov_b32_e32 v1, v510689; VI-NEXT: s_setpc_b64 s[30:31]10690;10691; GFX9-LABEL: global_atomic_uinc_wrap_i64_ret:10692; GFX9: ; %bb.0:10693; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10694; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off10695; GFX9-NEXT: s_mov_b64 s[4:5], 010696; GFX9-NEXT: .LBB133_1: ; %atomicrmw.start10697; GFX9-NEXT: ; =>This Inner Loop Header: Depth=110698; GFX9-NEXT: s_waitcnt vmcnt(0)10699; GFX9-NEXT: v_mov_b32_e32 v7, v510700; GFX9-NEXT: v_mov_b32_e32 v6, v410701; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, 1, v610702; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v7, vcc10703; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]10704; GFX9-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc10705; GFX9-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc10706; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc10707; GFX9-NEXT: s_waitcnt vmcnt(0)10708; GFX9-NEXT: buffer_wbinvl1_vol10709; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]10710; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]10711; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]10712; GFX9-NEXT: s_cbranch_execnz .LBB133_110713; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end10714; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]10715; GFX9-NEXT: v_mov_b32_e32 v0, v410716; GFX9-NEXT: v_mov_b32_e32 v1, v510717; GFX9-NEXT: s_setpc_b64 s[30:31]10718 %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 %in seq_cst10719 ret i64 %result10720}10721 10722define i64 @global_atomic_uinc_wrap_i64_ret_offset(ptr addrspace(1) %out, i64 %in) {10723; SI-LABEL: global_atomic_uinc_wrap_i64_ret_offset:10724; SI: ; %bb.0:10725; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10726; SI-NEXT: v_mov_b32_e32 v5, v310727; SI-NEXT: v_mov_b32_e32 v4, v210728; SI-NEXT: v_mov_b32_e32 v7, v110729; SI-NEXT: v_mov_b32_e32 v6, v010730; SI-NEXT: s_mov_b32 s6, 010731; SI-NEXT: s_mov_b32 s7, 0xf00010732; SI-NEXT: s_mov_b32 s4, s610733; SI-NEXT: s_mov_b32 s5, s610734; SI-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[4:7], 0 addr64 offset:3210735; SI-NEXT: s_mov_b64 s[8:9], 010736; SI-NEXT: .LBB134_1: ; %atomicrmw.start10737; SI-NEXT: ; =>This Inner Loop Header: Depth=110738; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)10739; SI-NEXT: v_add_i32_e32 v0, vcc, 1, v1010740; SI-NEXT: v_addc_u32_e32 v1, vcc, 0, v11, vcc10741; SI-NEXT: v_cmp_lt_u64_e32 vcc, v[10:11], v[4:5]10742; SI-NEXT: v_cndmask_b32_e32 v9, 0, v1, vcc10743; SI-NEXT: v_cndmask_b32_e32 v8, 0, v0, vcc10744; SI-NEXT: v_mov_b32_e32 v0, v810745; SI-NEXT: v_mov_b32_e32 v1, v910746; SI-NEXT: v_mov_b32_e32 v2, v1010747; SI-NEXT: v_mov_b32_e32 v3, v1110748; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 offset:32 glc10749; SI-NEXT: s_waitcnt vmcnt(0)10750; SI-NEXT: buffer_wbinvl110751; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]10752; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]10753; SI-NEXT: v_mov_b32_e32 v11, v110754; SI-NEXT: v_mov_b32_e32 v10, v010755; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]10756; SI-NEXT: s_cbranch_execnz .LBB134_110757; SI-NEXT: ; %bb.2: ; %atomicrmw.end10758; SI-NEXT: s_or_b64 exec, exec, s[8:9]10759; SI-NEXT: s_waitcnt expcnt(0)10760; SI-NEXT: s_setpc_b64 s[30:31]10761;10762; VI-LABEL: global_atomic_uinc_wrap_i64_ret_offset:10763; VI: ; %bb.0:10764; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10765; VI-NEXT: v_add_u32_e32 v4, vcc, 32, v010766; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc10767; VI-NEXT: flat_load_dwordx2 v[0:1], v[4:5]10768; VI-NEXT: s_mov_b64 s[4:5], 010769; VI-NEXT: .LBB134_1: ; %atomicrmw.start10770; VI-NEXT: ; =>This Inner Loop Header: Depth=110771; VI-NEXT: s_waitcnt vmcnt(0)10772; VI-NEXT: v_mov_b32_e32 v9, v110773; VI-NEXT: v_mov_b32_e32 v8, v010774; VI-NEXT: v_add_u32_e32 v0, vcc, 1, v810775; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v9, vcc10776; VI-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[2:3]10777; VI-NEXT: v_cndmask_b32_e32 v7, 0, v1, vcc10778; VI-NEXT: v_cndmask_b32_e32 v6, 0, v0, vcc10779; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc10780; VI-NEXT: s_waitcnt vmcnt(0)10781; VI-NEXT: buffer_wbinvl1_vol10782; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]10783; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]10784; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]10785; VI-NEXT: s_cbranch_execnz .LBB134_110786; VI-NEXT: ; %bb.2: ; %atomicrmw.end10787; VI-NEXT: s_or_b64 exec, exec, s[4:5]10788; VI-NEXT: s_setpc_b64 s[30:31]10789;10790; GFX9-LABEL: global_atomic_uinc_wrap_i64_ret_offset:10791; GFX9: ; %bb.0:10792; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10793; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:3210794; GFX9-NEXT: s_mov_b64 s[4:5], 010795; GFX9-NEXT: .LBB134_1: ; %atomicrmw.start10796; GFX9-NEXT: ; =>This Inner Loop Header: Depth=110797; GFX9-NEXT: s_waitcnt vmcnt(0)10798; GFX9-NEXT: v_mov_b32_e32 v7, v510799; GFX9-NEXT: v_mov_b32_e32 v6, v410800; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, 1, v610801; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v7, vcc10802; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]10803; GFX9-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc10804; GFX9-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc10805; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc10806; GFX9-NEXT: s_waitcnt vmcnt(0)10807; GFX9-NEXT: buffer_wbinvl1_vol10808; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]10809; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]10810; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]10811; GFX9-NEXT: s_cbranch_execnz .LBB134_110812; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end10813; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]10814; GFX9-NEXT: v_mov_b32_e32 v0, v410815; GFX9-NEXT: v_mov_b32_e32 v1, v510816; GFX9-NEXT: s_setpc_b64 s[30:31]10817 %gep = getelementptr i64, ptr addrspace(1) %out, i64 410818 %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 %in seq_cst10819 ret i64 %result10820}10821 10822define amdgpu_gfx void @global_atomic_uinc_wrap_i64_noret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {10823; SI-LABEL: global_atomic_uinc_wrap_i64_noret_scalar:10824; SI: ; %bb.0:10825; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10826; SI-NEXT: s_xor_saveexec_b64 s[34:35], -110827; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill10828; SI-NEXT: s_mov_b64 exec, s[34:35]10829; SI-NEXT: s_waitcnt expcnt(0)10830; SI-NEXT: v_writelane_b32 v8, s6, 010831; SI-NEXT: v_writelane_b32 v8, s7, 110832; SI-NEXT: s_mov_b32 s35, s710833; SI-NEXT: s_mov_b32 s34, s610834; SI-NEXT: s_mov_b32 s7, 0xf00010835; SI-NEXT: s_mov_b32 s6, -110836; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 010837; SI-NEXT: s_mov_b64 s[36:37], 010838; SI-NEXT: .LBB135_1: ; %atomicrmw.start10839; SI-NEXT: ; =>This Inner Loop Header: Depth=110840; SI-NEXT: s_waitcnt vmcnt(0)10841; SI-NEXT: v_add_i32_e32 v0, vcc, 1, v210842; SI-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc10843; SI-NEXT: v_cmp_gt_u64_e32 vcc, s[34:35], v[2:3]10844; SI-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc10845; SI-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc10846; SI-NEXT: s_waitcnt expcnt(0)10847; SI-NEXT: v_mov_b32_e32 v7, v310848; SI-NEXT: v_mov_b32_e32 v6, v210849; SI-NEXT: v_mov_b32_e32 v5, v110850; SI-NEXT: v_mov_b32_e32 v4, v010851; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 glc10852; SI-NEXT: s_waitcnt vmcnt(0)10853; SI-NEXT: buffer_wbinvl110854; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]10855; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]10856; SI-NEXT: v_mov_b32_e32 v2, v410857; SI-NEXT: v_mov_b32_e32 v3, v510858; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]10859; SI-NEXT: s_cbranch_execnz .LBB135_110860; SI-NEXT: ; %bb.2: ; %atomicrmw.end10861; SI-NEXT: s_or_b64 exec, exec, s[36:37]10862; SI-NEXT: v_readlane_b32 s7, v8, 110863; SI-NEXT: v_readlane_b32 s6, v8, 010864; SI-NEXT: s_xor_saveexec_b64 s[34:35], -110865; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload10866; SI-NEXT: s_mov_b64 exec, s[34:35]10867; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)10868; SI-NEXT: s_setpc_b64 s[30:31]10869;10870; VI-LABEL: global_atomic_uinc_wrap_i64_noret_scalar:10871; VI: ; %bb.0:10872; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10873; VI-NEXT: v_mov_b32_e32 v0, s410874; VI-NEXT: v_mov_b32_e32 v1, s510875; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]10876; VI-NEXT: v_mov_b32_e32 v4, s410877; VI-NEXT: s_mov_b64 s[34:35], 010878; VI-NEXT: v_mov_b32_e32 v5, s510879; VI-NEXT: .LBB135_1: ; %atomicrmw.start10880; VI-NEXT: ; =>This Inner Loop Header: Depth=110881; VI-NEXT: s_waitcnt vmcnt(0)10882; VI-NEXT: v_add_u32_e32 v0, vcc, 1, v210883; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc10884; VI-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]10885; VI-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc10886; VI-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc10887; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc10888; VI-NEXT: s_waitcnt vmcnt(0)10889; VI-NEXT: buffer_wbinvl1_vol10890; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]10891; VI-NEXT: v_mov_b32_e32 v3, v110892; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]10893; VI-NEXT: v_mov_b32_e32 v2, v010894; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]10895; VI-NEXT: s_cbranch_execnz .LBB135_110896; VI-NEXT: ; %bb.2: ; %atomicrmw.end10897; VI-NEXT: s_or_b64 exec, exec, s[34:35]10898; VI-NEXT: s_setpc_b64 s[30:31]10899;10900; GFX9-LABEL: global_atomic_uinc_wrap_i64_noret_scalar:10901; GFX9: ; %bb.0:10902; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10903; GFX9-NEXT: v_mov_b32_e32 v4, 010904; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5]10905; GFX9-NEXT: s_mov_b64 s[34:35], 010906; GFX9-NEXT: .LBB135_1: ; %atomicrmw.start10907; GFX9-NEXT: ; =>This Inner Loop Header: Depth=110908; GFX9-NEXT: s_waitcnt vmcnt(0)10909; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 1, v210910; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc10911; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]10912; GFX9-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc10913; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc10914; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc10915; GFX9-NEXT: s_waitcnt vmcnt(0)10916; GFX9-NEXT: buffer_wbinvl1_vol10917; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]10918; GFX9-NEXT: v_mov_b32_e32 v3, v110919; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]10920; GFX9-NEXT: v_mov_b32_e32 v2, v010921; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]10922; GFX9-NEXT: s_cbranch_execnz .LBB135_110923; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end10924; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]10925; GFX9-NEXT: s_setpc_b64 s[30:31]10926 %tmp0 = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 %in seq_cst10927 ret void10928}10929 10930define amdgpu_gfx void @global_atomic_uinc_wrap_i64_noret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {10931; SI-LABEL: global_atomic_uinc_wrap_i64_noret_offset_scalar:10932; SI: ; %bb.0:10933; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10934; SI-NEXT: s_xor_saveexec_b64 s[34:35], -110935; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill10936; SI-NEXT: s_mov_b64 exec, s[34:35]10937; SI-NEXT: s_waitcnt expcnt(0)10938; SI-NEXT: v_writelane_b32 v8, s6, 010939; SI-NEXT: v_writelane_b32 v8, s7, 110940; SI-NEXT: s_mov_b32 s35, s710941; SI-NEXT: s_mov_b32 s34, s610942; SI-NEXT: s_mov_b32 s7, 0xf00010943; SI-NEXT: s_mov_b32 s6, -110944; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:3210945; SI-NEXT: s_mov_b64 s[36:37], 010946; SI-NEXT: .LBB136_1: ; %atomicrmw.start10947; SI-NEXT: ; =>This Inner Loop Header: Depth=110948; SI-NEXT: s_waitcnt vmcnt(0)10949; SI-NEXT: v_add_i32_e32 v0, vcc, 1, v210950; SI-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc10951; SI-NEXT: v_cmp_gt_u64_e32 vcc, s[34:35], v[2:3]10952; SI-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc10953; SI-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc10954; SI-NEXT: s_waitcnt expcnt(0)10955; SI-NEXT: v_mov_b32_e32 v7, v310956; SI-NEXT: v_mov_b32_e32 v6, v210957; SI-NEXT: v_mov_b32_e32 v5, v110958; SI-NEXT: v_mov_b32_e32 v4, v010959; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 offset:32 glc10960; SI-NEXT: s_waitcnt vmcnt(0)10961; SI-NEXT: buffer_wbinvl110962; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]10963; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]10964; SI-NEXT: v_mov_b32_e32 v2, v410965; SI-NEXT: v_mov_b32_e32 v3, v510966; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]10967; SI-NEXT: s_cbranch_execnz .LBB136_110968; SI-NEXT: ; %bb.2: ; %atomicrmw.end10969; SI-NEXT: s_or_b64 exec, exec, s[36:37]10970; SI-NEXT: v_readlane_b32 s7, v8, 110971; SI-NEXT: v_readlane_b32 s6, v8, 010972; SI-NEXT: s_xor_saveexec_b64 s[34:35], -110973; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload10974; SI-NEXT: s_mov_b64 exec, s[34:35]10975; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)10976; SI-NEXT: s_setpc_b64 s[30:31]10977;10978; VI-LABEL: global_atomic_uinc_wrap_i64_noret_offset_scalar:10979; VI: ; %bb.0:10980; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10981; VI-NEXT: s_add_u32 s34, s4, 3210982; VI-NEXT: s_addc_u32 s35, s5, 010983; VI-NEXT: v_mov_b32_e32 v4, s3410984; VI-NEXT: v_mov_b32_e32 v5, s3510985; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]10986; VI-NEXT: s_mov_b64 s[34:35], 010987; VI-NEXT: .LBB136_1: ; %atomicrmw.start10988; VI-NEXT: ; =>This Inner Loop Header: Depth=110989; VI-NEXT: s_waitcnt vmcnt(0)10990; VI-NEXT: v_add_u32_e32 v0, vcc, 1, v210991; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc10992; VI-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]10993; VI-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc10994; VI-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc10995; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc10996; VI-NEXT: s_waitcnt vmcnt(0)10997; VI-NEXT: buffer_wbinvl1_vol10998; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]10999; VI-NEXT: v_mov_b32_e32 v3, v111000; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]11001; VI-NEXT: v_mov_b32_e32 v2, v011002; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]11003; VI-NEXT: s_cbranch_execnz .LBB136_111004; VI-NEXT: ; %bb.2: ; %atomicrmw.end11005; VI-NEXT: s_or_b64 exec, exec, s[34:35]11006; VI-NEXT: s_setpc_b64 s[30:31]11007;11008; GFX9-LABEL: global_atomic_uinc_wrap_i64_noret_offset_scalar:11009; GFX9: ; %bb.0:11010; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11011; GFX9-NEXT: v_mov_b32_e32 v4, 011012; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5] offset:3211013; GFX9-NEXT: s_mov_b64 s[34:35], 011014; GFX9-NEXT: .LBB136_1: ; %atomicrmw.start11015; GFX9-NEXT: ; =>This Inner Loop Header: Depth=111016; GFX9-NEXT: s_waitcnt vmcnt(0)11017; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 1, v211018; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc11019; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]11020; GFX9-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc11021; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc11022; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc11023; GFX9-NEXT: s_waitcnt vmcnt(0)11024; GFX9-NEXT: buffer_wbinvl1_vol11025; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]11026; GFX9-NEXT: v_mov_b32_e32 v3, v111027; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]11028; GFX9-NEXT: v_mov_b32_e32 v2, v011029; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]11030; GFX9-NEXT: s_cbranch_execnz .LBB136_111031; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end11032; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]11033; GFX9-NEXT: s_setpc_b64 s[30:31]11034 %gep = getelementptr i64, ptr addrspace(1) %out, i64 411035 %tmp0 = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 %in seq_cst11036 ret void11037}11038 11039define amdgpu_gfx i64 @global_atomic_uinc_wrap_i64_ret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {11040; SI-LABEL: global_atomic_uinc_wrap_i64_ret_scalar:11041; SI: ; %bb.0:11042; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11043; SI-NEXT: s_xor_saveexec_b64 s[34:35], -111044; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill11045; SI-NEXT: s_mov_b64 exec, s[34:35]11046; SI-NEXT: s_waitcnt expcnt(0)11047; SI-NEXT: v_writelane_b32 v6, s6, 011048; SI-NEXT: v_writelane_b32 v6, s7, 111049; SI-NEXT: s_mov_b32 s35, s711050; SI-NEXT: s_mov_b32 s34, s611051; SI-NEXT: s_mov_b32 s7, 0xf00011052; SI-NEXT: s_mov_b32 s6, -111053; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 011054; SI-NEXT: s_mov_b64 s[36:37], 011055; SI-NEXT: .LBB137_1: ; %atomicrmw.start11056; SI-NEXT: ; =>This Inner Loop Header: Depth=111057; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)11058; SI-NEXT: v_add_i32_e32 v0, vcc, 1, v411059; SI-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc11060; SI-NEXT: v_cmp_gt_u64_e32 vcc, s[34:35], v[4:5]11061; SI-NEXT: v_cndmask_b32_e32 v3, 0, v1, vcc11062; SI-NEXT: v_cndmask_b32_e32 v2, 0, v0, vcc11063; SI-NEXT: v_mov_b32_e32 v0, v211064; SI-NEXT: v_mov_b32_e32 v1, v311065; SI-NEXT: v_mov_b32_e32 v2, v411066; SI-NEXT: v_mov_b32_e32 v3, v511067; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc11068; SI-NEXT: s_waitcnt vmcnt(0)11069; SI-NEXT: buffer_wbinvl111070; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]11071; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]11072; SI-NEXT: v_mov_b32_e32 v5, v111073; SI-NEXT: v_mov_b32_e32 v4, v011074; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]11075; SI-NEXT: s_cbranch_execnz .LBB137_111076; SI-NEXT: ; %bb.2: ; %atomicrmw.end11077; SI-NEXT: s_or_b64 exec, exec, s[36:37]11078; SI-NEXT: v_readlane_b32 s7, v6, 111079; SI-NEXT: v_readlane_b32 s6, v6, 011080; SI-NEXT: s_xor_saveexec_b64 s[34:35], -111081; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload11082; SI-NEXT: s_mov_b64 exec, s[34:35]11083; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)11084; SI-NEXT: s_setpc_b64 s[30:31]11085;11086; VI-LABEL: global_atomic_uinc_wrap_i64_ret_scalar:11087; VI: ; %bb.0:11088; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11089; VI-NEXT: v_mov_b32_e32 v0, s411090; VI-NEXT: v_mov_b32_e32 v1, s511091; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]11092; VI-NEXT: v_mov_b32_e32 v2, s411093; VI-NEXT: s_mov_b64 s[34:35], 011094; VI-NEXT: v_mov_b32_e32 v3, s511095; VI-NEXT: .LBB137_1: ; %atomicrmw.start11096; VI-NEXT: ; =>This Inner Loop Header: Depth=111097; VI-NEXT: s_waitcnt vmcnt(0)11098; VI-NEXT: v_mov_b32_e32 v7, v111099; VI-NEXT: v_mov_b32_e32 v6, v011100; VI-NEXT: v_add_u32_e32 v0, vcc, 1, v611101; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc11102; VI-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]11103; VI-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc11104; VI-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc11105; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc11106; VI-NEXT: s_waitcnt vmcnt(0)11107; VI-NEXT: buffer_wbinvl1_vol11108; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]11109; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]11110; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]11111; VI-NEXT: s_cbranch_execnz .LBB137_111112; VI-NEXT: ; %bb.2: ; %atomicrmw.end11113; VI-NEXT: s_or_b64 exec, exec, s[34:35]11114; VI-NEXT: s_setpc_b64 s[30:31]11115;11116; GFX9-LABEL: global_atomic_uinc_wrap_i64_ret_scalar:11117; GFX9: ; %bb.0:11118; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11119; GFX9-NEXT: v_mov_b32_e32 v2, 011120; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]11121; GFX9-NEXT: s_mov_b64 s[34:35], 011122; GFX9-NEXT: .LBB137_1: ; %atomicrmw.start11123; GFX9-NEXT: ; =>This Inner Loop Header: Depth=111124; GFX9-NEXT: s_waitcnt vmcnt(0)11125; GFX9-NEXT: v_mov_b32_e32 v6, v111126; GFX9-NEXT: v_mov_b32_e32 v5, v011127; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 1, v511128; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v6, vcc11129; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[5:6]11130; GFX9-NEXT: v_cndmask_b32_e32 v4, 0, v1, vcc11131; GFX9-NEXT: v_cndmask_b32_e32 v3, 0, v0, vcc11132; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] glc11133; GFX9-NEXT: s_waitcnt vmcnt(0)11134; GFX9-NEXT: buffer_wbinvl1_vol11135; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]11136; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]11137; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]11138; GFX9-NEXT: s_cbranch_execnz .LBB137_111139; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end11140; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]11141; GFX9-NEXT: s_setpc_b64 s[30:31]11142 %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 %in seq_cst11143 ret i64 %result11144}11145 11146define amdgpu_gfx i64 @global_atomic_uinc_wrap_i64_ret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {11147; SI-LABEL: global_atomic_uinc_wrap_i64_ret_offset_scalar:11148; SI: ; %bb.0:11149; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11150; SI-NEXT: s_xor_saveexec_b64 s[34:35], -111151; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill11152; SI-NEXT: s_mov_b64 exec, s[34:35]11153; SI-NEXT: s_waitcnt expcnt(0)11154; SI-NEXT: v_writelane_b32 v6, s6, 011155; SI-NEXT: v_writelane_b32 v6, s7, 111156; SI-NEXT: s_mov_b32 s35, s711157; SI-NEXT: s_mov_b32 s34, s611158; SI-NEXT: s_mov_b32 s7, 0xf00011159; SI-NEXT: s_mov_b32 s6, -111160; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0 offset:3211161; SI-NEXT: s_mov_b64 s[36:37], 011162; SI-NEXT: .LBB138_1: ; %atomicrmw.start11163; SI-NEXT: ; =>This Inner Loop Header: Depth=111164; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)11165; SI-NEXT: v_add_i32_e32 v0, vcc, 1, v411166; SI-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc11167; SI-NEXT: v_cmp_gt_u64_e32 vcc, s[34:35], v[4:5]11168; SI-NEXT: v_cndmask_b32_e32 v3, 0, v1, vcc11169; SI-NEXT: v_cndmask_b32_e32 v2, 0, v0, vcc11170; SI-NEXT: v_mov_b32_e32 v0, v211171; SI-NEXT: v_mov_b32_e32 v1, v311172; SI-NEXT: v_mov_b32_e32 v2, v411173; SI-NEXT: v_mov_b32_e32 v3, v511174; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 offset:32 glc11175; SI-NEXT: s_waitcnt vmcnt(0)11176; SI-NEXT: buffer_wbinvl111177; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]11178; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]11179; SI-NEXT: v_mov_b32_e32 v5, v111180; SI-NEXT: v_mov_b32_e32 v4, v011181; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]11182; SI-NEXT: s_cbranch_execnz .LBB138_111183; SI-NEXT: ; %bb.2: ; %atomicrmw.end11184; SI-NEXT: s_or_b64 exec, exec, s[36:37]11185; SI-NEXT: v_readlane_b32 s7, v6, 111186; SI-NEXT: v_readlane_b32 s6, v6, 011187; SI-NEXT: s_xor_saveexec_b64 s[34:35], -111188; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload11189; SI-NEXT: s_mov_b64 exec, s[34:35]11190; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)11191; SI-NEXT: s_setpc_b64 s[30:31]11192;11193; VI-LABEL: global_atomic_uinc_wrap_i64_ret_offset_scalar:11194; VI: ; %bb.0:11195; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11196; VI-NEXT: s_add_u32 s34, s4, 3211197; VI-NEXT: s_addc_u32 s35, s5, 011198; VI-NEXT: v_mov_b32_e32 v2, s3411199; VI-NEXT: v_mov_b32_e32 v3, s3511200; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]11201; VI-NEXT: s_mov_b64 s[34:35], 011202; VI-NEXT: .LBB138_1: ; %atomicrmw.start11203; VI-NEXT: ; =>This Inner Loop Header: Depth=111204; VI-NEXT: s_waitcnt vmcnt(0)11205; VI-NEXT: v_mov_b32_e32 v7, v111206; VI-NEXT: v_mov_b32_e32 v6, v011207; VI-NEXT: v_add_u32_e32 v0, vcc, 1, v611208; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc11209; VI-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]11210; VI-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc11211; VI-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc11212; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc11213; VI-NEXT: s_waitcnt vmcnt(0)11214; VI-NEXT: buffer_wbinvl1_vol11215; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]11216; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]11217; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]11218; VI-NEXT: s_cbranch_execnz .LBB138_111219; VI-NEXT: ; %bb.2: ; %atomicrmw.end11220; VI-NEXT: s_or_b64 exec, exec, s[34:35]11221; VI-NEXT: s_setpc_b64 s[30:31]11222;11223; GFX9-LABEL: global_atomic_uinc_wrap_i64_ret_offset_scalar:11224; GFX9: ; %bb.0:11225; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11226; GFX9-NEXT: v_mov_b32_e32 v2, 011227; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5] offset:3211228; GFX9-NEXT: s_mov_b64 s[34:35], 011229; GFX9-NEXT: .LBB138_1: ; %atomicrmw.start11230; GFX9-NEXT: ; =>This Inner Loop Header: Depth=111231; GFX9-NEXT: s_waitcnt vmcnt(0)11232; GFX9-NEXT: v_mov_b32_e32 v6, v111233; GFX9-NEXT: v_mov_b32_e32 v5, v011234; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 1, v511235; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v6, vcc11236; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[5:6]11237; GFX9-NEXT: v_cndmask_b32_e32 v4, 0, v1, vcc11238; GFX9-NEXT: v_cndmask_b32_e32 v3, 0, v0, vcc11239; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] offset:32 glc11240; GFX9-NEXT: s_waitcnt vmcnt(0)11241; GFX9-NEXT: buffer_wbinvl1_vol11242; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]11243; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]11244; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]11245; GFX9-NEXT: s_cbranch_execnz .LBB138_111246; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end11247; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]11248; GFX9-NEXT: s_setpc_b64 s[30:31]11249 %gep = getelementptr i64, ptr addrspace(1) %out, i64 411250 %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 %in seq_cst11251 ret i64 %result11252}11253 11254define void @global_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {11255; SI-LABEL: global_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory:11256; SI: ; %bb.0:11257; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11258; SI-NEXT: s_mov_b32 s6, 011259; SI-NEXT: s_mov_b32 s7, 0xf00011260; SI-NEXT: s_mov_b32 s4, s611261; SI-NEXT: s_mov_b32 s5, s611262; SI-NEXT: buffer_atomic_inc_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:3211263; SI-NEXT: s_waitcnt vmcnt(0)11264; SI-NEXT: buffer_wbinvl111265; SI-NEXT: s_waitcnt expcnt(0)11266; SI-NEXT: s_setpc_b64 s[30:31]11267;11268; VI-LABEL: global_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory:11269; VI: ; %bb.0:11270; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11271; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v011272; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc11273; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]11274; VI-NEXT: s_waitcnt vmcnt(0)11275; VI-NEXT: buffer_wbinvl1_vol11276; VI-NEXT: s_setpc_b64 s[30:31]11277;11278; GFX9-LABEL: global_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory:11279; GFX9: ; %bb.0:11280; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11281; GFX9-NEXT: global_atomic_inc_x2 v[0:1], v[2:3], off offset:3211282; GFX9-NEXT: s_waitcnt vmcnt(0)11283; GFX9-NEXT: buffer_wbinvl1_vol11284; GFX9-NEXT: s_setpc_b64 s[30:31]11285 %gep = getelementptr i64, ptr addrspace(1) %out, i64 411286 %tmp0 = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !011287 ret void11288}11289 11290define i64 @global_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {11291; SI-LABEL: global_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory:11292; SI: ; %bb.0:11293; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11294; SI-NEXT: s_mov_b32 s6, 011295; SI-NEXT: s_mov_b32 s7, 0xf00011296; SI-NEXT: s_mov_b32 s4, s611297; SI-NEXT: s_mov_b32 s5, s611298; SI-NEXT: buffer_atomic_inc_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc11299; SI-NEXT: s_waitcnt vmcnt(0)11300; SI-NEXT: buffer_wbinvl111301; SI-NEXT: v_mov_b32_e32 v0, v211302; SI-NEXT: v_mov_b32_e32 v1, v311303; SI-NEXT: s_waitcnt expcnt(0)11304; SI-NEXT: s_setpc_b64 s[30:31]11305;11306; VI-LABEL: global_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory:11307; VI: ; %bb.0:11308; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11309; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v011310; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc11311; VI-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc11312; VI-NEXT: s_waitcnt vmcnt(0)11313; VI-NEXT: buffer_wbinvl1_vol11314; VI-NEXT: s_setpc_b64 s[30:31]11315;11316; GFX9-LABEL: global_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory:11317; GFX9: ; %bb.0:11318; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11319; GFX9-NEXT: global_atomic_inc_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc11320; GFX9-NEXT: s_waitcnt vmcnt(0)11321; GFX9-NEXT: buffer_wbinvl1_vol11322; GFX9-NEXT: s_setpc_b64 s[30:31]11323 %gep = getelementptr i64, ptr addrspace(1) %out, i64 411324 %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !011325 ret i64 %result11326}11327 11328; ---------------------------------------------------------------------11329; atomicrmw udec_wrap11330; ---------------------------------------------------------------------11331 11332define void @global_atomic_udec_wrap_i64_noret(ptr addrspace(1) %ptr, i64 %in) {11333; SI-LABEL: global_atomic_udec_wrap_i64_noret:11334; SI: ; %bb.0:11335; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11336; SI-NEXT: s_mov_b32 s10, 011337; SI-NEXT: s_mov_b32 s11, 0xf00011338; SI-NEXT: s_mov_b32 s8, s1011339; SI-NEXT: s_mov_b32 s9, s1011340; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[8:11], 0 addr6411341; SI-NEXT: s_mov_b64 s[6:7], 011342; SI-NEXT: .LBB141_1: ; %atomicrmw.start11343; SI-NEXT: ; =>This Inner Loop Header: Depth=111344; SI-NEXT: s_waitcnt vmcnt(0)11345; SI-NEXT: v_add_i32_e32 v4, vcc, -1, v611346; SI-NEXT: v_addc_u32_e32 v5, vcc, -1, v7, vcc11347; SI-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]11348; SI-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]11349; SI-NEXT: s_or_b64 vcc, vcc, s[4:5]11350; SI-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc11351; SI-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc11352; SI-NEXT: s_waitcnt expcnt(0)11353; SI-NEXT: v_mov_b32_e32 v11, v711354; SI-NEXT: v_mov_b32_e32 v10, v611355; SI-NEXT: v_mov_b32_e32 v9, v511356; SI-NEXT: v_mov_b32_e32 v8, v411357; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[8:11], 0 addr64 glc11358; SI-NEXT: s_waitcnt vmcnt(0)11359; SI-NEXT: buffer_wbinvl111360; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]11361; SI-NEXT: s_or_b64 s[6:7], vcc, s[6:7]11362; SI-NEXT: v_mov_b32_e32 v6, v811363; SI-NEXT: v_mov_b32_e32 v7, v911364; SI-NEXT: s_andn2_b64 exec, exec, s[6:7]11365; SI-NEXT: s_cbranch_execnz .LBB141_111366; SI-NEXT: ; %bb.2: ; %atomicrmw.end11367; SI-NEXT: s_or_b64 exec, exec, s[6:7]11368; SI-NEXT: s_waitcnt expcnt(0)11369; SI-NEXT: s_setpc_b64 s[30:31]11370;11371; VI-LABEL: global_atomic_udec_wrap_i64_noret:11372; VI: ; %bb.0:11373; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11374; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]11375; VI-NEXT: s_mov_b64 s[8:9], 011376; VI-NEXT: .LBB141_1: ; %atomicrmw.start11377; VI-NEXT: ; =>This Inner Loop Header: Depth=111378; VI-NEXT: s_waitcnt vmcnt(0)11379; VI-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]11380; VI-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]11381; VI-NEXT: v_add_u32_e64 v4, s[6:7], -1, v611382; VI-NEXT: v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]11383; VI-NEXT: s_or_b64 vcc, vcc, s[4:5]11384; VI-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc11385; VI-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc11386; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc11387; VI-NEXT: s_waitcnt vmcnt(0)11388; VI-NEXT: buffer_wbinvl1_vol11389; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]11390; VI-NEXT: v_mov_b32_e32 v7, v511391; VI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]11392; VI-NEXT: v_mov_b32_e32 v6, v411393; VI-NEXT: s_andn2_b64 exec, exec, s[8:9]11394; VI-NEXT: s_cbranch_execnz .LBB141_111395; VI-NEXT: ; %bb.2: ; %atomicrmw.end11396; VI-NEXT: s_or_b64 exec, exec, s[8:9]11397; VI-NEXT: s_setpc_b64 s[30:31]11398;11399; GFX9-LABEL: global_atomic_udec_wrap_i64_noret:11400; GFX9: ; %bb.0:11401; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11402; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off11403; GFX9-NEXT: s_mov_b64 s[8:9], 011404; GFX9-NEXT: .LBB141_1: ; %atomicrmw.start11405; GFX9-NEXT: ; =>This Inner Loop Header: Depth=111406; GFX9-NEXT: s_waitcnt vmcnt(0)11407; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]11408; GFX9-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]11409; GFX9-NEXT: v_add_co_u32_e64 v4, s[6:7], -1, v611410; GFX9-NEXT: v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7]11411; GFX9-NEXT: s_or_b64 vcc, vcc, s[4:5]11412; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc11413; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc11414; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc11415; GFX9-NEXT: s_waitcnt vmcnt(0)11416; GFX9-NEXT: buffer_wbinvl1_vol11417; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]11418; GFX9-NEXT: v_mov_b32_e32 v7, v511419; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]11420; GFX9-NEXT: v_mov_b32_e32 v6, v411421; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]11422; GFX9-NEXT: s_cbranch_execnz .LBB141_111423; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end11424; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]11425; GFX9-NEXT: s_setpc_b64 s[30:31]11426 %tmp0 = atomicrmw udec_wrap ptr addrspace(1) %ptr, i64 %in seq_cst11427 ret void11428}11429 11430define void @global_atomic_udec_wrap_i64_noret_offset(ptr addrspace(1) %out, i64 %in) {11431; SI-LABEL: global_atomic_udec_wrap_i64_noret_offset:11432; SI: ; %bb.0:11433; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11434; SI-NEXT: s_mov_b32 s10, 011435; SI-NEXT: s_mov_b32 s11, 0xf00011436; SI-NEXT: s_mov_b32 s8, s1011437; SI-NEXT: s_mov_b32 s9, s1011438; SI-NEXT: buffer_load_dwordx2 v[6:7], v[0:1], s[8:11], 0 addr64 offset:3211439; SI-NEXT: s_mov_b64 s[6:7], 011440; SI-NEXT: .LBB142_1: ; %atomicrmw.start11441; SI-NEXT: ; =>This Inner Loop Header: Depth=111442; SI-NEXT: s_waitcnt vmcnt(0)11443; SI-NEXT: v_add_i32_e32 v4, vcc, -1, v611444; SI-NEXT: v_addc_u32_e32 v5, vcc, -1, v7, vcc11445; SI-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]11446; SI-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]11447; SI-NEXT: s_or_b64 vcc, vcc, s[4:5]11448; SI-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc11449; SI-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc11450; SI-NEXT: s_waitcnt expcnt(0)11451; SI-NEXT: v_mov_b32_e32 v11, v711452; SI-NEXT: v_mov_b32_e32 v10, v611453; SI-NEXT: v_mov_b32_e32 v9, v511454; SI-NEXT: v_mov_b32_e32 v8, v411455; SI-NEXT: buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[8:11], 0 addr64 offset:32 glc11456; SI-NEXT: s_waitcnt vmcnt(0)11457; SI-NEXT: buffer_wbinvl111458; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]11459; SI-NEXT: s_or_b64 s[6:7], vcc, s[6:7]11460; SI-NEXT: v_mov_b32_e32 v6, v811461; SI-NEXT: v_mov_b32_e32 v7, v911462; SI-NEXT: s_andn2_b64 exec, exec, s[6:7]11463; SI-NEXT: s_cbranch_execnz .LBB142_111464; SI-NEXT: ; %bb.2: ; %atomicrmw.end11465; SI-NEXT: s_or_b64 exec, exec, s[6:7]11466; SI-NEXT: s_waitcnt expcnt(0)11467; SI-NEXT: s_setpc_b64 s[30:31]11468;11469; VI-LABEL: global_atomic_udec_wrap_i64_noret_offset:11470; VI: ; %bb.0:11471; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11472; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v011473; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc11474; VI-NEXT: flat_load_dwordx2 v[6:7], v[0:1]11475; VI-NEXT: s_mov_b64 s[8:9], 011476; VI-NEXT: .LBB142_1: ; %atomicrmw.start11477; VI-NEXT: ; =>This Inner Loop Header: Depth=111478; VI-NEXT: s_waitcnt vmcnt(0)11479; VI-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]11480; VI-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]11481; VI-NEXT: v_add_u32_e64 v4, s[6:7], -1, v611482; VI-NEXT: v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]11483; VI-NEXT: s_or_b64 vcc, vcc, s[4:5]11484; VI-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc11485; VI-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc11486; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc11487; VI-NEXT: s_waitcnt vmcnt(0)11488; VI-NEXT: buffer_wbinvl1_vol11489; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]11490; VI-NEXT: v_mov_b32_e32 v7, v511491; VI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]11492; VI-NEXT: v_mov_b32_e32 v6, v411493; VI-NEXT: s_andn2_b64 exec, exec, s[8:9]11494; VI-NEXT: s_cbranch_execnz .LBB142_111495; VI-NEXT: ; %bb.2: ; %atomicrmw.end11496; VI-NEXT: s_or_b64 exec, exec, s[8:9]11497; VI-NEXT: s_setpc_b64 s[30:31]11498;11499; GFX9-LABEL: global_atomic_udec_wrap_i64_noret_offset:11500; GFX9: ; %bb.0:11501; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11502; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:3211503; GFX9-NEXT: s_mov_b64 s[8:9], 011504; GFX9-NEXT: .LBB142_1: ; %atomicrmw.start11505; GFX9-NEXT: ; =>This Inner Loop Header: Depth=111506; GFX9-NEXT: s_waitcnt vmcnt(0)11507; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]11508; GFX9-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]11509; GFX9-NEXT: v_add_co_u32_e64 v4, s[6:7], -1, v611510; GFX9-NEXT: v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7]11511; GFX9-NEXT: s_or_b64 vcc, vcc, s[4:5]11512; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc11513; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc11514; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc11515; GFX9-NEXT: s_waitcnt vmcnt(0)11516; GFX9-NEXT: buffer_wbinvl1_vol11517; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]11518; GFX9-NEXT: v_mov_b32_e32 v7, v511519; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]11520; GFX9-NEXT: v_mov_b32_e32 v6, v411521; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]11522; GFX9-NEXT: s_cbranch_execnz .LBB142_111523; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end11524; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]11525; GFX9-NEXT: s_setpc_b64 s[30:31]11526 %gep = getelementptr i64, ptr addrspace(1) %out, i64 411527 %tmp0 = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 %in seq_cst11528 ret void11529}11530 11531define i64 @global_atomic_udec_wrap_i64_ret(ptr addrspace(1) %ptr, i64 %in) {11532; SI-LABEL: global_atomic_udec_wrap_i64_ret:11533; SI: ; %bb.0:11534; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11535; SI-NEXT: v_mov_b32_e32 v5, v311536; SI-NEXT: v_mov_b32_e32 v4, v211537; SI-NEXT: v_mov_b32_e32 v7, v111538; SI-NEXT: v_mov_b32_e32 v6, v011539; SI-NEXT: s_mov_b32 s10, 011540; SI-NEXT: s_mov_b32 s11, 0xf00011541; SI-NEXT: s_mov_b32 s8, s1011542; SI-NEXT: s_mov_b32 s9, s1011543; SI-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[8:11], 0 addr6411544; SI-NEXT: s_mov_b64 s[6:7], 011545; SI-NEXT: .LBB143_1: ; %atomicrmw.start11546; SI-NEXT: ; =>This Inner Loop Header: Depth=111547; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)11548; SI-NEXT: v_add_i32_e32 v0, vcc, -1, v1011549; SI-NEXT: v_addc_u32_e32 v1, vcc, -1, v11, vcc11550; SI-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11]11551; SI-NEXT: v_cmp_gt_u64_e64 s[4:5], v[10:11], v[4:5]11552; SI-NEXT: s_or_b64 vcc, vcc, s[4:5]11553; SI-NEXT: v_cndmask_b32_e32 v9, v1, v5, vcc11554; SI-NEXT: v_cndmask_b32_e32 v8, v0, v4, vcc11555; SI-NEXT: v_mov_b32_e32 v0, v811556; SI-NEXT: v_mov_b32_e32 v1, v911557; SI-NEXT: v_mov_b32_e32 v2, v1011558; SI-NEXT: v_mov_b32_e32 v3, v1111559; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[8:11], 0 addr64 glc11560; SI-NEXT: s_waitcnt vmcnt(0)11561; SI-NEXT: buffer_wbinvl111562; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]11563; SI-NEXT: s_or_b64 s[6:7], vcc, s[6:7]11564; SI-NEXT: v_mov_b32_e32 v11, v111565; SI-NEXT: v_mov_b32_e32 v10, v011566; SI-NEXT: s_andn2_b64 exec, exec, s[6:7]11567; SI-NEXT: s_cbranch_execnz .LBB143_111568; SI-NEXT: ; %bb.2: ; %atomicrmw.end11569; SI-NEXT: s_or_b64 exec, exec, s[6:7]11570; SI-NEXT: s_waitcnt expcnt(0)11571; SI-NEXT: s_setpc_b64 s[30:31]11572;11573; VI-LABEL: global_atomic_udec_wrap_i64_ret:11574; VI: ; %bb.0:11575; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11576; VI-NEXT: flat_load_dwordx2 v[4:5], v[0:1]11577; VI-NEXT: s_mov_b64 s[8:9], 011578; VI-NEXT: .LBB143_1: ; %atomicrmw.start11579; VI-NEXT: ; =>This Inner Loop Header: Depth=111580; VI-NEXT: s_waitcnt vmcnt(0)11581; VI-NEXT: v_mov_b32_e32 v7, v511582; VI-NEXT: v_mov_b32_e32 v6, v411583; VI-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]11584; VI-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]11585; VI-NEXT: v_add_u32_e64 v4, s[6:7], -1, v611586; VI-NEXT: v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]11587; VI-NEXT: s_or_b64 vcc, vcc, s[4:5]11588; VI-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc11589; VI-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc11590; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc11591; VI-NEXT: s_waitcnt vmcnt(0)11592; VI-NEXT: buffer_wbinvl1_vol11593; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]11594; VI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]11595; VI-NEXT: s_andn2_b64 exec, exec, s[8:9]11596; VI-NEXT: s_cbranch_execnz .LBB143_111597; VI-NEXT: ; %bb.2: ; %atomicrmw.end11598; VI-NEXT: s_or_b64 exec, exec, s[8:9]11599; VI-NEXT: v_mov_b32_e32 v0, v411600; VI-NEXT: v_mov_b32_e32 v1, v511601; VI-NEXT: s_setpc_b64 s[30:31]11602;11603; GFX9-LABEL: global_atomic_udec_wrap_i64_ret:11604; GFX9: ; %bb.0:11605; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11606; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off11607; GFX9-NEXT: s_mov_b64 s[8:9], 011608; GFX9-NEXT: .LBB143_1: ; %atomicrmw.start11609; GFX9-NEXT: ; =>This Inner Loop Header: Depth=111610; GFX9-NEXT: s_waitcnt vmcnt(0)11611; GFX9-NEXT: v_mov_b32_e32 v7, v511612; GFX9-NEXT: v_mov_b32_e32 v6, v411613; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]11614; GFX9-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]11615; GFX9-NEXT: v_add_co_u32_e64 v4, s[6:7], -1, v611616; GFX9-NEXT: v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7]11617; GFX9-NEXT: s_or_b64 vcc, vcc, s[4:5]11618; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc11619; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc11620; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc11621; GFX9-NEXT: s_waitcnt vmcnt(0)11622; GFX9-NEXT: buffer_wbinvl1_vol11623; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]11624; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]11625; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]11626; GFX9-NEXT: s_cbranch_execnz .LBB143_111627; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end11628; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]11629; GFX9-NEXT: v_mov_b32_e32 v0, v411630; GFX9-NEXT: v_mov_b32_e32 v1, v511631; GFX9-NEXT: s_setpc_b64 s[30:31]11632 %result = atomicrmw udec_wrap ptr addrspace(1) %ptr, i64 %in seq_cst11633 ret i64 %result11634}11635 11636define i64 @global_atomic_udec_wrap_i64_ret_offset(ptr addrspace(1) %out, i64 %in) {11637; SI-LABEL: global_atomic_udec_wrap_i64_ret_offset:11638; SI: ; %bb.0:11639; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11640; SI-NEXT: v_mov_b32_e32 v5, v311641; SI-NEXT: v_mov_b32_e32 v4, v211642; SI-NEXT: v_mov_b32_e32 v7, v111643; SI-NEXT: v_mov_b32_e32 v6, v011644; SI-NEXT: s_mov_b32 s10, 011645; SI-NEXT: s_mov_b32 s11, 0xf00011646; SI-NEXT: s_mov_b32 s8, s1011647; SI-NEXT: s_mov_b32 s9, s1011648; SI-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[8:11], 0 addr64 offset:3211649; SI-NEXT: s_mov_b64 s[6:7], 011650; SI-NEXT: .LBB144_1: ; %atomicrmw.start11651; SI-NEXT: ; =>This Inner Loop Header: Depth=111652; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)11653; SI-NEXT: v_add_i32_e32 v0, vcc, -1, v1011654; SI-NEXT: v_addc_u32_e32 v1, vcc, -1, v11, vcc11655; SI-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11]11656; SI-NEXT: v_cmp_gt_u64_e64 s[4:5], v[10:11], v[4:5]11657; SI-NEXT: s_or_b64 vcc, vcc, s[4:5]11658; SI-NEXT: v_cndmask_b32_e32 v9, v1, v5, vcc11659; SI-NEXT: v_cndmask_b32_e32 v8, v0, v4, vcc11660; SI-NEXT: v_mov_b32_e32 v0, v811661; SI-NEXT: v_mov_b32_e32 v1, v911662; SI-NEXT: v_mov_b32_e32 v2, v1011663; SI-NEXT: v_mov_b32_e32 v3, v1111664; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[8:11], 0 addr64 offset:32 glc11665; SI-NEXT: s_waitcnt vmcnt(0)11666; SI-NEXT: buffer_wbinvl111667; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]11668; SI-NEXT: s_or_b64 s[6:7], vcc, s[6:7]11669; SI-NEXT: v_mov_b32_e32 v11, v111670; SI-NEXT: v_mov_b32_e32 v10, v011671; SI-NEXT: s_andn2_b64 exec, exec, s[6:7]11672; SI-NEXT: s_cbranch_execnz .LBB144_111673; SI-NEXT: ; %bb.2: ; %atomicrmw.end11674; SI-NEXT: s_or_b64 exec, exec, s[6:7]11675; SI-NEXT: s_waitcnt expcnt(0)11676; SI-NEXT: s_setpc_b64 s[30:31]11677;11678; VI-LABEL: global_atomic_udec_wrap_i64_ret_offset:11679; VI: ; %bb.0:11680; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11681; VI-NEXT: v_add_u32_e32 v4, vcc, 32, v011682; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc11683; VI-NEXT: flat_load_dwordx2 v[0:1], v[4:5]11684; VI-NEXT: s_mov_b64 s[8:9], 011685; VI-NEXT: .LBB144_1: ; %atomicrmw.start11686; VI-NEXT: ; =>This Inner Loop Header: Depth=111687; VI-NEXT: s_waitcnt vmcnt(0)11688; VI-NEXT: v_mov_b32_e32 v9, v111689; VI-NEXT: v_mov_b32_e32 v8, v011690; VI-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]11691; VI-NEXT: v_cmp_gt_u64_e64 s[4:5], v[8:9], v[2:3]11692; VI-NEXT: v_add_u32_e64 v0, s[6:7], -1, v811693; VI-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v9, s[6:7]11694; VI-NEXT: s_or_b64 vcc, vcc, s[4:5]11695; VI-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc11696; VI-NEXT: v_cndmask_b32_e32 v6, v0, v2, vcc11697; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc11698; VI-NEXT: s_waitcnt vmcnt(0)11699; VI-NEXT: buffer_wbinvl1_vol11700; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]11701; VI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]11702; VI-NEXT: s_andn2_b64 exec, exec, s[8:9]11703; VI-NEXT: s_cbranch_execnz .LBB144_111704; VI-NEXT: ; %bb.2: ; %atomicrmw.end11705; VI-NEXT: s_or_b64 exec, exec, s[8:9]11706; VI-NEXT: s_setpc_b64 s[30:31]11707;11708; GFX9-LABEL: global_atomic_udec_wrap_i64_ret_offset:11709; GFX9: ; %bb.0:11710; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11711; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:3211712; GFX9-NEXT: s_mov_b64 s[8:9], 011713; GFX9-NEXT: .LBB144_1: ; %atomicrmw.start11714; GFX9-NEXT: ; =>This Inner Loop Header: Depth=111715; GFX9-NEXT: s_waitcnt vmcnt(0)11716; GFX9-NEXT: v_mov_b32_e32 v7, v511717; GFX9-NEXT: v_mov_b32_e32 v6, v411718; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]11719; GFX9-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]11720; GFX9-NEXT: v_add_co_u32_e64 v4, s[6:7], -1, v611721; GFX9-NEXT: v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7]11722; GFX9-NEXT: s_or_b64 vcc, vcc, s[4:5]11723; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc11724; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc11725; GFX9-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc11726; GFX9-NEXT: s_waitcnt vmcnt(0)11727; GFX9-NEXT: buffer_wbinvl1_vol11728; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]11729; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]11730; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]11731; GFX9-NEXT: s_cbranch_execnz .LBB144_111732; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end11733; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]11734; GFX9-NEXT: v_mov_b32_e32 v0, v411735; GFX9-NEXT: v_mov_b32_e32 v1, v511736; GFX9-NEXT: s_setpc_b64 s[30:31]11737 %gep = getelementptr i64, ptr addrspace(1) %out, i64 411738 %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 %in seq_cst11739 ret i64 %result11740}11741 11742define amdgpu_gfx void @global_atomic_udec_wrap_i64_noret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {11743; SI-LABEL: global_atomic_udec_wrap_i64_noret_scalar:11744; SI: ; %bb.0:11745; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11746; SI-NEXT: s_xor_saveexec_b64 s[34:35], -111747; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill11748; SI-NEXT: s_mov_b64 exec, s[34:35]11749; SI-NEXT: s_waitcnt expcnt(0)11750; SI-NEXT: v_writelane_b32 v10, s6, 011751; SI-NEXT: v_writelane_b32 v10, s7, 111752; SI-NEXT: s_mov_b32 s35, s711753; SI-NEXT: s_mov_b32 s34, s611754; SI-NEXT: s_mov_b32 s7, 0xf00011755; SI-NEXT: s_mov_b32 s6, -111756; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 011757; SI-NEXT: s_mov_b64 s[38:39], 011758; SI-NEXT: v_mov_b32_e32 v4, s3511759; SI-NEXT: v_mov_b32_e32 v5, s3411760; SI-NEXT: .LBB145_1: ; %atomicrmw.start11761; SI-NEXT: ; =>This Inner Loop Header: Depth=111762; SI-NEXT: s_waitcnt vmcnt(0)11763; SI-NEXT: v_add_i32_e32 v0, vcc, -1, v211764; SI-NEXT: v_addc_u32_e32 v1, vcc, -1, v3, vcc11765; SI-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]11766; SI-NEXT: v_cmp_lt_u64_e64 s[36:37], s[34:35], v[2:3]11767; SI-NEXT: s_or_b64 vcc, vcc, s[36:37]11768; SI-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc11769; SI-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc11770; SI-NEXT: s_waitcnt expcnt(0)11771; SI-NEXT: v_mov_b32_e32 v9, v311772; SI-NEXT: v_mov_b32_e32 v8, v211773; SI-NEXT: v_mov_b32_e32 v7, v111774; SI-NEXT: v_mov_b32_e32 v6, v011775; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 glc11776; SI-NEXT: s_waitcnt vmcnt(0)11777; SI-NEXT: buffer_wbinvl111778; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]11779; SI-NEXT: s_or_b64 s[38:39], vcc, s[38:39]11780; SI-NEXT: v_mov_b32_e32 v2, v611781; SI-NEXT: v_mov_b32_e32 v3, v711782; SI-NEXT: s_andn2_b64 exec, exec, s[38:39]11783; SI-NEXT: s_cbranch_execnz .LBB145_111784; SI-NEXT: ; %bb.2: ; %atomicrmw.end11785; SI-NEXT: s_or_b64 exec, exec, s[38:39]11786; SI-NEXT: v_readlane_b32 s7, v10, 111787; SI-NEXT: v_readlane_b32 s6, v10, 011788; SI-NEXT: s_xor_saveexec_b64 s[34:35], -111789; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload11790; SI-NEXT: s_mov_b64 exec, s[34:35]11791; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)11792; SI-NEXT: s_setpc_b64 s[30:31]11793;11794; VI-LABEL: global_atomic_udec_wrap_i64_noret_scalar:11795; VI: ; %bb.0:11796; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11797; VI-NEXT: v_mov_b32_e32 v0, s411798; VI-NEXT: v_mov_b32_e32 v1, s511799; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]11800; VI-NEXT: v_mov_b32_e32 v4, s411801; VI-NEXT: s_mov_b64 s[38:39], 011802; VI-NEXT: v_mov_b32_e32 v6, s711803; VI-NEXT: v_mov_b32_e32 v7, s611804; VI-NEXT: v_mov_b32_e32 v5, s511805; VI-NEXT: .LBB145_1: ; %atomicrmw.start11806; VI-NEXT: ; =>This Inner Loop Header: Depth=111807; VI-NEXT: s_waitcnt vmcnt(0)11808; VI-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]11809; VI-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]11810; VI-NEXT: v_add_u32_e64 v0, s[36:37], -1, v211811; VI-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37]11812; VI-NEXT: s_or_b64 vcc, vcc, s[34:35]11813; VI-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc11814; VI-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc11815; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc11816; VI-NEXT: s_waitcnt vmcnt(0)11817; VI-NEXT: buffer_wbinvl1_vol11818; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]11819; VI-NEXT: v_mov_b32_e32 v3, v111820; VI-NEXT: s_or_b64 s[38:39], vcc, s[38:39]11821; VI-NEXT: v_mov_b32_e32 v2, v011822; VI-NEXT: s_andn2_b64 exec, exec, s[38:39]11823; VI-NEXT: s_cbranch_execnz .LBB145_111824; VI-NEXT: ; %bb.2: ; %atomicrmw.end11825; VI-NEXT: s_or_b64 exec, exec, s[38:39]11826; VI-NEXT: s_setpc_b64 s[30:31]11827;11828; GFX9-LABEL: global_atomic_udec_wrap_i64_noret_scalar:11829; GFX9: ; %bb.0:11830; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11831; GFX9-NEXT: v_mov_b32_e32 v4, 011832; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5]11833; GFX9-NEXT: s_mov_b64 s[38:39], 011834; GFX9-NEXT: v_mov_b32_e32 v5, s711835; GFX9-NEXT: v_mov_b32_e32 v6, s611836; GFX9-NEXT: .LBB145_1: ; %atomicrmw.start11837; GFX9-NEXT: ; =>This Inner Loop Header: Depth=111838; GFX9-NEXT: s_waitcnt vmcnt(0)11839; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]11840; GFX9-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]11841; GFX9-NEXT: v_add_co_u32_e64 v0, s[36:37], -1, v211842; GFX9-NEXT: v_addc_co_u32_e64 v1, s[36:37], -1, v3, s[36:37]11843; GFX9-NEXT: s_or_b64 vcc, vcc, s[34:35]11844; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc11845; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc11846; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc11847; GFX9-NEXT: s_waitcnt vmcnt(0)11848; GFX9-NEXT: buffer_wbinvl1_vol11849; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]11850; GFX9-NEXT: v_mov_b32_e32 v3, v111851; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39]11852; GFX9-NEXT: v_mov_b32_e32 v2, v011853; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39]11854; GFX9-NEXT: s_cbranch_execnz .LBB145_111855; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end11856; GFX9-NEXT: s_or_b64 exec, exec, s[38:39]11857; GFX9-NEXT: s_setpc_b64 s[30:31]11858 %tmp0 = atomicrmw udec_wrap ptr addrspace(1) %ptr, i64 %in seq_cst11859 ret void11860}11861 11862define amdgpu_gfx void @global_atomic_udec_wrap_i64_noret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {11863; SI-LABEL: global_atomic_udec_wrap_i64_noret_offset_scalar:11864; SI: ; %bb.0:11865; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11866; SI-NEXT: s_xor_saveexec_b64 s[34:35], -111867; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill11868; SI-NEXT: s_mov_b64 exec, s[34:35]11869; SI-NEXT: s_waitcnt expcnt(0)11870; SI-NEXT: v_writelane_b32 v10, s6, 011871; SI-NEXT: v_writelane_b32 v10, s7, 111872; SI-NEXT: s_mov_b32 s35, s711873; SI-NEXT: s_mov_b32 s34, s611874; SI-NEXT: s_mov_b32 s7, 0xf00011875; SI-NEXT: s_mov_b32 s6, -111876; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:3211877; SI-NEXT: s_mov_b64 s[38:39], 011878; SI-NEXT: v_mov_b32_e32 v4, s3511879; SI-NEXT: v_mov_b32_e32 v5, s3411880; SI-NEXT: .LBB146_1: ; %atomicrmw.start11881; SI-NEXT: ; =>This Inner Loop Header: Depth=111882; SI-NEXT: s_waitcnt vmcnt(0)11883; SI-NEXT: v_add_i32_e32 v0, vcc, -1, v211884; SI-NEXT: v_addc_u32_e32 v1, vcc, -1, v3, vcc11885; SI-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]11886; SI-NEXT: v_cmp_lt_u64_e64 s[36:37], s[34:35], v[2:3]11887; SI-NEXT: s_or_b64 vcc, vcc, s[36:37]11888; SI-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc11889; SI-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc11890; SI-NEXT: s_waitcnt expcnt(0)11891; SI-NEXT: v_mov_b32_e32 v9, v311892; SI-NEXT: v_mov_b32_e32 v8, v211893; SI-NEXT: v_mov_b32_e32 v7, v111894; SI-NEXT: v_mov_b32_e32 v6, v011895; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 offset:32 glc11896; SI-NEXT: s_waitcnt vmcnt(0)11897; SI-NEXT: buffer_wbinvl111898; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]11899; SI-NEXT: s_or_b64 s[38:39], vcc, s[38:39]11900; SI-NEXT: v_mov_b32_e32 v2, v611901; SI-NEXT: v_mov_b32_e32 v3, v711902; SI-NEXT: s_andn2_b64 exec, exec, s[38:39]11903; SI-NEXT: s_cbranch_execnz .LBB146_111904; SI-NEXT: ; %bb.2: ; %atomicrmw.end11905; SI-NEXT: s_or_b64 exec, exec, s[38:39]11906; SI-NEXT: v_readlane_b32 s7, v10, 111907; SI-NEXT: v_readlane_b32 s6, v10, 011908; SI-NEXT: s_xor_saveexec_b64 s[34:35], -111909; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload11910; SI-NEXT: s_mov_b64 exec, s[34:35]11911; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)11912; SI-NEXT: s_setpc_b64 s[30:31]11913;11914; VI-LABEL: global_atomic_udec_wrap_i64_noret_offset_scalar:11915; VI: ; %bb.0:11916; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11917; VI-NEXT: s_add_u32 s34, s4, 3211918; VI-NEXT: s_addc_u32 s35, s5, 011919; VI-NEXT: v_mov_b32_e32 v4, s3411920; VI-NEXT: v_mov_b32_e32 v5, s3511921; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]11922; VI-NEXT: s_mov_b64 s[38:39], 011923; VI-NEXT: v_mov_b32_e32 v6, s711924; VI-NEXT: v_mov_b32_e32 v7, s611925; VI-NEXT: .LBB146_1: ; %atomicrmw.start11926; VI-NEXT: ; =>This Inner Loop Header: Depth=111927; VI-NEXT: s_waitcnt vmcnt(0)11928; VI-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]11929; VI-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]11930; VI-NEXT: v_add_u32_e64 v0, s[36:37], -1, v211931; VI-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37]11932; VI-NEXT: s_or_b64 vcc, vcc, s[34:35]11933; VI-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc11934; VI-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc11935; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc11936; VI-NEXT: s_waitcnt vmcnt(0)11937; VI-NEXT: buffer_wbinvl1_vol11938; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]11939; VI-NEXT: v_mov_b32_e32 v3, v111940; VI-NEXT: s_or_b64 s[38:39], vcc, s[38:39]11941; VI-NEXT: v_mov_b32_e32 v2, v011942; VI-NEXT: s_andn2_b64 exec, exec, s[38:39]11943; VI-NEXT: s_cbranch_execnz .LBB146_111944; VI-NEXT: ; %bb.2: ; %atomicrmw.end11945; VI-NEXT: s_or_b64 exec, exec, s[38:39]11946; VI-NEXT: s_setpc_b64 s[30:31]11947;11948; GFX9-LABEL: global_atomic_udec_wrap_i64_noret_offset_scalar:11949; GFX9: ; %bb.0:11950; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11951; GFX9-NEXT: v_mov_b32_e32 v4, 011952; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5] offset:3211953; GFX9-NEXT: s_mov_b64 s[38:39], 011954; GFX9-NEXT: v_mov_b32_e32 v5, s711955; GFX9-NEXT: v_mov_b32_e32 v6, s611956; GFX9-NEXT: .LBB146_1: ; %atomicrmw.start11957; GFX9-NEXT: ; =>This Inner Loop Header: Depth=111958; GFX9-NEXT: s_waitcnt vmcnt(0)11959; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]11960; GFX9-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]11961; GFX9-NEXT: v_add_co_u32_e64 v0, s[36:37], -1, v211962; GFX9-NEXT: v_addc_co_u32_e64 v1, s[36:37], -1, v3, s[36:37]11963; GFX9-NEXT: s_or_b64 vcc, vcc, s[34:35]11964; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc11965; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc11966; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc11967; GFX9-NEXT: s_waitcnt vmcnt(0)11968; GFX9-NEXT: buffer_wbinvl1_vol11969; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]11970; GFX9-NEXT: v_mov_b32_e32 v3, v111971; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39]11972; GFX9-NEXT: v_mov_b32_e32 v2, v011973; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39]11974; GFX9-NEXT: s_cbranch_execnz .LBB146_111975; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end11976; GFX9-NEXT: s_or_b64 exec, exec, s[38:39]11977; GFX9-NEXT: s_setpc_b64 s[30:31]11978 %gep = getelementptr i64, ptr addrspace(1) %out, i64 411979 %tmp0 = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 %in seq_cst11980 ret void11981}11982 11983define amdgpu_gfx i64 @global_atomic_udec_wrap_i64_ret_scalar(ptr addrspace(1) inreg %ptr, i64 inreg %in) {11984; SI-LABEL: global_atomic_udec_wrap_i64_ret_scalar:11985; SI: ; %bb.0:11986; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11987; SI-NEXT: s_xor_saveexec_b64 s[34:35], -111988; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill11989; SI-NEXT: s_mov_b64 exec, s[34:35]11990; SI-NEXT: s_waitcnt expcnt(0)11991; SI-NEXT: v_writelane_b32 v8, s6, 011992; SI-NEXT: v_writelane_b32 v8, s7, 111993; SI-NEXT: s_mov_b32 s35, s711994; SI-NEXT: s_mov_b32 s34, s611995; SI-NEXT: s_mov_b32 s7, 0xf00011996; SI-NEXT: s_mov_b32 s6, -111997; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 011998; SI-NEXT: s_mov_b64 s[38:39], 011999; SI-NEXT: v_mov_b32_e32 v6, s3512000; SI-NEXT: v_mov_b32_e32 v7, s3412001; SI-NEXT: .LBB147_1: ; %atomicrmw.start12002; SI-NEXT: ; =>This Inner Loop Header: Depth=112003; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)12004; SI-NEXT: v_add_i32_e32 v0, vcc, -1, v412005; SI-NEXT: v_addc_u32_e32 v1, vcc, -1, v5, vcc12006; SI-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]12007; SI-NEXT: v_cmp_lt_u64_e64 s[36:37], s[34:35], v[4:5]12008; SI-NEXT: s_or_b64 vcc, vcc, s[36:37]12009; SI-NEXT: v_cndmask_b32_e32 v3, v1, v6, vcc12010; SI-NEXT: v_cndmask_b32_e32 v2, v0, v7, vcc12011; SI-NEXT: v_mov_b32_e32 v0, v212012; SI-NEXT: v_mov_b32_e32 v1, v312013; SI-NEXT: v_mov_b32_e32 v2, v412014; SI-NEXT: v_mov_b32_e32 v3, v512015; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc12016; SI-NEXT: s_waitcnt vmcnt(0)12017; SI-NEXT: buffer_wbinvl112018; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]12019; SI-NEXT: s_or_b64 s[38:39], vcc, s[38:39]12020; SI-NEXT: v_mov_b32_e32 v5, v112021; SI-NEXT: v_mov_b32_e32 v4, v012022; SI-NEXT: s_andn2_b64 exec, exec, s[38:39]12023; SI-NEXT: s_cbranch_execnz .LBB147_112024; SI-NEXT: ; %bb.2: ; %atomicrmw.end12025; SI-NEXT: s_or_b64 exec, exec, s[38:39]12026; SI-NEXT: v_readlane_b32 s7, v8, 112027; SI-NEXT: v_readlane_b32 s6, v8, 012028; SI-NEXT: s_xor_saveexec_b64 s[34:35], -112029; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload12030; SI-NEXT: s_mov_b64 exec, s[34:35]12031; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)12032; SI-NEXT: s_setpc_b64 s[30:31]12033;12034; VI-LABEL: global_atomic_udec_wrap_i64_ret_scalar:12035; VI: ; %bb.0:12036; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12037; VI-NEXT: v_mov_b32_e32 v0, s412038; VI-NEXT: v_mov_b32_e32 v1, s512039; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]12040; VI-NEXT: v_mov_b32_e32 v2, s412041; VI-NEXT: s_mov_b64 s[38:39], 012042; VI-NEXT: v_mov_b32_e32 v4, s712043; VI-NEXT: v_mov_b32_e32 v5, s612044; VI-NEXT: v_mov_b32_e32 v3, s512045; VI-NEXT: .LBB147_1: ; %atomicrmw.start12046; VI-NEXT: ; =>This Inner Loop Header: Depth=112047; VI-NEXT: s_waitcnt vmcnt(0)12048; VI-NEXT: v_mov_b32_e32 v9, v112049; VI-NEXT: v_mov_b32_e32 v8, v012050; VI-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]12051; VI-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]12052; VI-NEXT: v_add_u32_e64 v0, s[36:37], -1, v812053; VI-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37]12054; VI-NEXT: s_or_b64 vcc, vcc, s[34:35]12055; VI-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc12056; VI-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc12057; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc12058; VI-NEXT: s_waitcnt vmcnt(0)12059; VI-NEXT: buffer_wbinvl1_vol12060; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]12061; VI-NEXT: s_or_b64 s[38:39], vcc, s[38:39]12062; VI-NEXT: s_andn2_b64 exec, exec, s[38:39]12063; VI-NEXT: s_cbranch_execnz .LBB147_112064; VI-NEXT: ; %bb.2: ; %atomicrmw.end12065; VI-NEXT: s_or_b64 exec, exec, s[38:39]12066; VI-NEXT: s_setpc_b64 s[30:31]12067;12068; GFX9-LABEL: global_atomic_udec_wrap_i64_ret_scalar:12069; GFX9: ; %bb.0:12070; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12071; GFX9-NEXT: v_mov_b32_e32 v2, 012072; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]12073; GFX9-NEXT: s_mov_b64 s[38:39], 012074; GFX9-NEXT: v_mov_b32_e32 v3, s712075; GFX9-NEXT: v_mov_b32_e32 v4, s612076; GFX9-NEXT: .LBB147_1: ; %atomicrmw.start12077; GFX9-NEXT: ; =>This Inner Loop Header: Depth=112078; GFX9-NEXT: s_waitcnt vmcnt(0)12079; GFX9-NEXT: v_mov_b32_e32 v8, v112080; GFX9-NEXT: v_mov_b32_e32 v7, v012081; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[7:8]12082; GFX9-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[7:8]12083; GFX9-NEXT: v_add_co_u32_e64 v0, s[36:37], -1, v712084; GFX9-NEXT: v_addc_co_u32_e64 v1, s[36:37], -1, v8, s[36:37]12085; GFX9-NEXT: s_or_b64 vcc, vcc, s[34:35]12086; GFX9-NEXT: v_cndmask_b32_e32 v6, v1, v3, vcc12087; GFX9-NEXT: v_cndmask_b32_e32 v5, v0, v4, vcc12088; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] glc12089; GFX9-NEXT: s_waitcnt vmcnt(0)12090; GFX9-NEXT: buffer_wbinvl1_vol12091; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]12092; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39]12093; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39]12094; GFX9-NEXT: s_cbranch_execnz .LBB147_112095; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end12096; GFX9-NEXT: s_or_b64 exec, exec, s[38:39]12097; GFX9-NEXT: s_setpc_b64 s[30:31]12098 %result = atomicrmw udec_wrap ptr addrspace(1) %ptr, i64 %in seq_cst12099 ret i64 %result12100}12101 12102define amdgpu_gfx i64 @global_atomic_udec_wrap_i64_ret_offset_scalar(ptr addrspace(1) inreg %out, i64 inreg %in) {12103; SI-LABEL: global_atomic_udec_wrap_i64_ret_offset_scalar:12104; SI: ; %bb.0:12105; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12106; SI-NEXT: s_xor_saveexec_b64 s[34:35], -112107; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill12108; SI-NEXT: s_mov_b64 exec, s[34:35]12109; SI-NEXT: s_waitcnt expcnt(0)12110; SI-NEXT: v_writelane_b32 v8, s6, 012111; SI-NEXT: v_writelane_b32 v8, s7, 112112; SI-NEXT: s_mov_b32 s35, s712113; SI-NEXT: s_mov_b32 s34, s612114; SI-NEXT: s_mov_b32 s7, 0xf00012115; SI-NEXT: s_mov_b32 s6, -112116; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0 offset:3212117; SI-NEXT: s_mov_b64 s[38:39], 012118; SI-NEXT: v_mov_b32_e32 v6, s3512119; SI-NEXT: v_mov_b32_e32 v7, s3412120; SI-NEXT: .LBB148_1: ; %atomicrmw.start12121; SI-NEXT: ; =>This Inner Loop Header: Depth=112122; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)12123; SI-NEXT: v_add_i32_e32 v0, vcc, -1, v412124; SI-NEXT: v_addc_u32_e32 v1, vcc, -1, v5, vcc12125; SI-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]12126; SI-NEXT: v_cmp_lt_u64_e64 s[36:37], s[34:35], v[4:5]12127; SI-NEXT: s_or_b64 vcc, vcc, s[36:37]12128; SI-NEXT: v_cndmask_b32_e32 v3, v1, v6, vcc12129; SI-NEXT: v_cndmask_b32_e32 v2, v0, v7, vcc12130; SI-NEXT: v_mov_b32_e32 v0, v212131; SI-NEXT: v_mov_b32_e32 v1, v312132; SI-NEXT: v_mov_b32_e32 v2, v412133; SI-NEXT: v_mov_b32_e32 v3, v512134; SI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 offset:32 glc12135; SI-NEXT: s_waitcnt vmcnt(0)12136; SI-NEXT: buffer_wbinvl112137; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]12138; SI-NEXT: s_or_b64 s[38:39], vcc, s[38:39]12139; SI-NEXT: v_mov_b32_e32 v5, v112140; SI-NEXT: v_mov_b32_e32 v4, v012141; SI-NEXT: s_andn2_b64 exec, exec, s[38:39]12142; SI-NEXT: s_cbranch_execnz .LBB148_112143; SI-NEXT: ; %bb.2: ; %atomicrmw.end12144; SI-NEXT: s_or_b64 exec, exec, s[38:39]12145; SI-NEXT: v_readlane_b32 s7, v8, 112146; SI-NEXT: v_readlane_b32 s6, v8, 012147; SI-NEXT: s_xor_saveexec_b64 s[34:35], -112148; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload12149; SI-NEXT: s_mov_b64 exec, s[34:35]12150; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)12151; SI-NEXT: s_setpc_b64 s[30:31]12152;12153; VI-LABEL: global_atomic_udec_wrap_i64_ret_offset_scalar:12154; VI: ; %bb.0:12155; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12156; VI-NEXT: s_add_u32 s34, s4, 3212157; VI-NEXT: s_addc_u32 s35, s5, 012158; VI-NEXT: v_mov_b32_e32 v2, s3412159; VI-NEXT: v_mov_b32_e32 v3, s3512160; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]12161; VI-NEXT: s_mov_b64 s[38:39], 012162; VI-NEXT: v_mov_b32_e32 v4, s712163; VI-NEXT: v_mov_b32_e32 v5, s612164; VI-NEXT: .LBB148_1: ; %atomicrmw.start12165; VI-NEXT: ; =>This Inner Loop Header: Depth=112166; VI-NEXT: s_waitcnt vmcnt(0)12167; VI-NEXT: v_mov_b32_e32 v9, v112168; VI-NEXT: v_mov_b32_e32 v8, v012169; VI-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]12170; VI-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]12171; VI-NEXT: v_add_u32_e64 v0, s[36:37], -1, v812172; VI-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37]12173; VI-NEXT: s_or_b64 vcc, vcc, s[34:35]12174; VI-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc12175; VI-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc12176; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc12177; VI-NEXT: s_waitcnt vmcnt(0)12178; VI-NEXT: buffer_wbinvl1_vol12179; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]12180; VI-NEXT: s_or_b64 s[38:39], vcc, s[38:39]12181; VI-NEXT: s_andn2_b64 exec, exec, s[38:39]12182; VI-NEXT: s_cbranch_execnz .LBB148_112183; VI-NEXT: ; %bb.2: ; %atomicrmw.end12184; VI-NEXT: s_or_b64 exec, exec, s[38:39]12185; VI-NEXT: s_setpc_b64 s[30:31]12186;12187; GFX9-LABEL: global_atomic_udec_wrap_i64_ret_offset_scalar:12188; GFX9: ; %bb.0:12189; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12190; GFX9-NEXT: v_mov_b32_e32 v2, 012191; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5] offset:3212192; GFX9-NEXT: s_mov_b64 s[38:39], 012193; GFX9-NEXT: v_mov_b32_e32 v3, s712194; GFX9-NEXT: v_mov_b32_e32 v4, s612195; GFX9-NEXT: .LBB148_1: ; %atomicrmw.start12196; GFX9-NEXT: ; =>This Inner Loop Header: Depth=112197; GFX9-NEXT: s_waitcnt vmcnt(0)12198; GFX9-NEXT: v_mov_b32_e32 v8, v112199; GFX9-NEXT: v_mov_b32_e32 v7, v012200; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[7:8]12201; GFX9-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[7:8]12202; GFX9-NEXT: v_add_co_u32_e64 v0, s[36:37], -1, v712203; GFX9-NEXT: v_addc_co_u32_e64 v1, s[36:37], -1, v8, s[36:37]12204; GFX9-NEXT: s_or_b64 vcc, vcc, s[34:35]12205; GFX9-NEXT: v_cndmask_b32_e32 v6, v1, v3, vcc12206; GFX9-NEXT: v_cndmask_b32_e32 v5, v0, v4, vcc12207; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] offset:32 glc12208; GFX9-NEXT: s_waitcnt vmcnt(0)12209; GFX9-NEXT: buffer_wbinvl1_vol12210; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]12211; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39]12212; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39]12213; GFX9-NEXT: s_cbranch_execnz .LBB148_112214; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end12215; GFX9-NEXT: s_or_b64 exec, exec, s[38:39]12216; GFX9-NEXT: s_setpc_b64 s[30:31]12217 %gep = getelementptr i64, ptr addrspace(1) %out, i64 412218 %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 %in seq_cst12219 ret i64 %result12220}12221 12222define void @global_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {12223; SI-LABEL: global_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory:12224; SI: ; %bb.0:12225; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12226; SI-NEXT: s_mov_b32 s6, 012227; SI-NEXT: s_mov_b32 s7, 0xf00012228; SI-NEXT: s_mov_b32 s4, s612229; SI-NEXT: s_mov_b32 s5, s612230; SI-NEXT: buffer_atomic_dec_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:3212231; SI-NEXT: s_waitcnt vmcnt(0)12232; SI-NEXT: buffer_wbinvl112233; SI-NEXT: s_waitcnt expcnt(0)12234; SI-NEXT: s_setpc_b64 s[30:31]12235;12236; VI-LABEL: global_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory:12237; VI: ; %bb.0:12238; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12239; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v012240; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc12241; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]12242; VI-NEXT: s_waitcnt vmcnt(0)12243; VI-NEXT: buffer_wbinvl1_vol12244; VI-NEXT: s_setpc_b64 s[30:31]12245;12246; GFX9-LABEL: global_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory:12247; GFX9: ; %bb.0:12248; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12249; GFX9-NEXT: global_atomic_dec_x2 v[0:1], v[2:3], off offset:3212250; GFX9-NEXT: s_waitcnt vmcnt(0)12251; GFX9-NEXT: buffer_wbinvl1_vol12252; GFX9-NEXT: s_setpc_b64 s[30:31]12253 %gep = getelementptr i64, ptr addrspace(1) %out, i64 412254 %tmp0 = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !012255 ret void12256}12257 12258define i64 @global_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i64 %in) {12259; SI-LABEL: global_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory:12260; SI: ; %bb.0:12261; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12262; SI-NEXT: s_mov_b32 s6, 012263; SI-NEXT: s_mov_b32 s7, 0xf00012264; SI-NEXT: s_mov_b32 s4, s612265; SI-NEXT: s_mov_b32 s5, s612266; SI-NEXT: buffer_atomic_dec_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc12267; SI-NEXT: s_waitcnt vmcnt(0)12268; SI-NEXT: buffer_wbinvl112269; SI-NEXT: v_mov_b32_e32 v0, v212270; SI-NEXT: v_mov_b32_e32 v1, v312271; SI-NEXT: s_waitcnt expcnt(0)12272; SI-NEXT: s_setpc_b64 s[30:31]12273;12274; VI-LABEL: global_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory:12275; VI: ; %bb.0:12276; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12277; VI-NEXT: v_add_u32_e32 v0, vcc, 32, v012278; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc12279; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc12280; VI-NEXT: s_waitcnt vmcnt(0)12281; VI-NEXT: buffer_wbinvl1_vol12282; VI-NEXT: s_setpc_b64 s[30:31]12283;12284; GFX9-LABEL: global_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory:12285; GFX9: ; %bb.0:12286; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12287; GFX9-NEXT: global_atomic_dec_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc12288; GFX9-NEXT: s_waitcnt vmcnt(0)12289; GFX9-NEXT: buffer_wbinvl1_vol12290; GFX9-NEXT: s_setpc_b64 s[30:31]12291 %gep = getelementptr i64, ptr addrspace(1) %out, i64 412292 %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !012293 ret i64 %result12294}12295 12296!0 = !{}12297